CONFIG: handle string splitting [zsh] (#2640)

- all file-handlers can now be created with a user-specified
  communicator and/or with specified io-ranks. This makes it possible
  to quickly create special file-handlers for subsets of ranks.

  For the uncollated file output this implies that the file-handler
  now has a communicator only with itself, which is correct since each
  rank is acting like it own IO master rank and doesn't coordinate
  effort with other ranks.

src/OpenFOAM/db/IOstreams/Fstreams/masterOFstream.C

@@ -92,21 +92,20 @@ void Foam::masterOFstream::commit()
 {
     if (Pstream::parRun())
     {
-        List<fileName> filePaths(Pstream::nProcs());
+        List<fileName> filePaths(Pstream::nProcs(comm_));
-        filePaths[Pstream::myProcNo()] = pathName_;
+        filePaths[Pstream::myProcNo(comm_)] = pathName_;
-        Pstream::gatherList(filePaths);
+        Pstream::gatherList(filePaths, UPstream::msgType(), comm_);
 
         bool uniform =
-            fileOperations::masterUncollatedFileOperation::uniformFile
+        (
-            (
+            Pstream::master(comm_) && fileOperation::uniformFile(filePaths)
-                filePaths
+        );
-            );
 
-        Pstream::broadcast(uniform);
+        Pstream::broadcast(uniform, comm_);
 
         if (uniform)
         {
-            if (Pstream::master() && valid_)
+            if (Pstream::master(comm_) && valid_)
             {
                 checkWrite(pathName_, this->str());
             }
@@ -115,13 +114,13 @@ void Foam::masterOFstream::commit()
             return;
         }
 
-        boolList procValid(UPstream::listGatherValues<bool>(valid_));
+        boolList procValid(UPstream::listGatherValues<bool>(valid_, comm_));
 
         // Different files
-        PstreamBuffers pBufs(Pstream::commsTypes::nonBlocking);
+        PstreamBuffers pBufs(comm_, Pstream::commsTypes::nonBlocking);
 
         // Send my buffer to master
-        if (!Pstream::master())
+        if (!Pstream::master(comm_))
         {
             UOPstream os(Pstream::masterNo(), pBufs);
             string s(this->str());
@@ -133,7 +132,7 @@ void Foam::masterOFstream::commit()
         labelList recvSizes;
         pBufs.finishedGathers(recvSizes);
 
-        if (Pstream::master())
+        if (Pstream::master(comm_))
         {
             // Write master data
             if (procValid[Pstream::masterNo()])
@@ -149,7 +148,7 @@ void Foam::masterOFstream::commit()
                 *std::max_element(recvSizes.cbegin(), recvSizes.cend())
             );
 
-            for (const int proci : Pstream::subProcs())
+            for (const int proci : Pstream::subProcs(comm_))
             {
                 UIPstream is(proci, pBufs);
 
@@ -179,6 +178,7 @@ void Foam::masterOFstream::commit()
 Foam::masterOFstream::masterOFstream
 (
     IOstreamOption::atomicType atomic,
+    const label comm,
     const fileName& pathName,
     IOstreamOption streamOpt,
     IOstreamOption::appendType append,
@@ -190,7 +190,8 @@ Foam::masterOFstream::masterOFstream
     atomic_(atomic),
     compression_(streamOpt.compression()),
     append_(append),
-    valid_(valid)
+    valid_(valid),
+    comm_(comm)
 {}
 
 

src/OpenFOAM/db/IOstreams/Fstreams/masterOFstream.H

@@ -30,6 +30,9 @@ Class
 Description
     Master-only drop-in replacement for OFstream.
 
+    Called on all processors (of the provided communicator). Sends files to
+    the master and writes them there.
+
 SourceFiles
     masterOFstream.C
 
@@ -39,6 +42,7 @@ SourceFiles
 #define Foam_masterOFstream_H
 
 #include "StringStream.H"
+#include "UPstream.H"
 
 // * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
 
@@ -70,6 +74,9 @@ class masterOFstream
         //- Should file be written (on this processor)
         const bool valid_;
 
+        //- Communicator
+        const label comm_;
+
 
     // Private Member Functions
 
@@ -92,6 +99,40 @@ public:
 
     // Constructors
 
+        //- Construct with specified atomic behaviour and communicator
+        //- from pathname, stream option, optional append
+        masterOFstream
+        (
+            IOstreamOption::atomicType atomic,
+            const label comm,
+            const fileName& pathname,
+            IOstreamOption streamOpt = IOstreamOption(),
+            IOstreamOption::appendType append = IOstreamOption::NON_APPEND,
+            const bool valid = true
+        );
+
+        //- Construct with specified communicator
+        //- from pathname, stream option, optional append
+        masterOFstream
+        (
+            const label comm,
+            const fileName& pathname,
+            IOstreamOption streamOpt = IOstreamOption(),
+            IOstreamOption::appendType append = IOstreamOption::NON_APPEND,
+            const bool valid = true
+        )
+        :
+            masterOFstream
+            (
+                IOstreamOption::NON_ATOMIC,
+                comm,
+                pathname,
+                streamOpt,
+                append,
+                valid
+            )
+        {}
+
         //- Construct with specified atomic behaviour (with worldComm)
         //- from pathname, stream option, optional append
         masterOFstream
@@ -101,7 +142,18 @@ public:
             IOstreamOption streamOpt = IOstreamOption(),
             IOstreamOption::appendType append = IOstreamOption::NON_APPEND,
             const bool valid = true
-        );
+        )
+        :
+            masterOFstream
+            (
+                atomic,
+                UPstream::worldComm,
+                pathname,
+                streamOpt,
+                append,
+                valid
+            )
+        {}
 
         //- Construct (with worldComm)
         //- from pathname, stream option, optional append
@@ -116,6 +168,7 @@ public:
             masterOFstream
             (
                 IOstreamOption::NON_ATOMIC,
+                UPstream::worldComm,
                 pathname,
                 streamOpt,
                 append,

src/OpenFOAM/db/Time/Time.C

@@ -397,7 +397,15 @@ void Foam::Time::setMonitoring(const bool forceProfiling)
     if (runTimeModifiable_)
     {
         // Monitor all files that controlDict depends on
-        fileHandler().addWatches(controlDict_, controlDict_.files());
+        auto& watchFiles = controlDict_.files();
+
+        // Files might have been set during token reading so only on master
+        // processor. Make sure we add them to all processors (though they
+        // are checked only on master - we just need to keep the state
+        // synchronised)
+        Pstream::broadcast(watchFiles, UPstream::worldComm);
+
+        fileHandler().addWatches(controlDict_, watchFiles);
     }
 
     // Clear dependent files - not needed now

src/OpenFOAM/db/dictionary/functionEntries/includeEntry/includeEntry.C

@@ -6,7 +6,7 @@
      \\/     M anipulation  |
 -------------------------------------------------------------------------------
     Copyright (C) 2011-2017 OpenFOAM Foundation
-    Copyright (C) 2018-2021 OpenCFD Ltd.
+    Copyright (C) 2018-2022 OpenCFD Ltd.
 -------------------------------------------------------------------------------
 License
     This file is part of OpenFOAM.
@@ -133,6 +133,15 @@ bool Foam::functionEntries::includeEntry::execute
     Istream& is
 )
 {
+    const dictionary& top = parentDict.topDict();
+    const auto* rioPtr = dynamic_cast<const regIOobject*>(&top);
+    const label oldComm
+    (
+        rioPtr && rioPtr->global()
+      ? fileHandler().comm(UPstream::worldComm)
+      : fileHandler().comm()
+    );
+
     const fileName rawName(is);
     const fileName fName(resolveFile(is.name().path(), rawName, parentDict));
 
@@ -148,20 +157,19 @@ bool Foam::functionEntries::includeEntry::execute
         }
 
         // Add watch on included file
-        const dictionary& top = parentDict.topDict();
+        if (rioPtr)
-        if (isA<regIOobject>(top))
         {
-            regIOobject& rio = const_cast<regIOobject&>
+            const_cast<regIOobject&>(*rioPtr).addWatch(fName);
-            (
-                dynamic_cast<const regIOobject&>(top)
-            );
-            rio.addWatch(fName);
         }
 
         parentDict.read(ifs);
+
+        fileHandler().comm(oldComm);
         return true;
     }
 
+    fileHandler().comm(oldComm);
+
     if (!mandatory)
     {
         return true; // Never fails if optional
@@ -185,6 +193,15 @@ bool Foam::functionEntries::includeEntry::execute
     Istream& is
 )
 {
+    const dictionary& top = parentDict.topDict();
+    const auto* rioPtr = dynamic_cast<const regIOobject*>(&top);
+    const label oldComm
+    (
+        rioPtr && rioPtr->global()
+      ? fileHandler().comm(UPstream::worldComm)
+      : fileHandler().comm()
+    );
+
     const fileName rawName(is);
     const fileName fName(resolveFile(is.name().path(), rawName, parentDict));
 
@@ -200,20 +217,19 @@ bool Foam::functionEntries::includeEntry::execute
         }
 
         // Add watch on included file
-        const dictionary& top = parentDict.topDict();
+        if (rioPtr)
-        if (isA<regIOobject>(top))
         {
-            regIOobject& rio = const_cast<regIOobject&>
+            const_cast<regIOobject&>(*rioPtr).addWatch(fName);
-            (
-                dynamic_cast<const regIOobject&>(top)
-            );
-            rio.addWatch(fName);
         }
 
         entry.read(parentDict, ifs);
+
+        fileHandler().comm(oldComm);
         return true;
     }
 
+    fileHandler().comm(oldComm);
+
     if (!mandatory)
     {
         return true; // Never fails if optional

src/OpenFOAM/db/dictionary/functionEntries/includeEtcEntry/includeEtcEntry.C

@@ -6,7 +6,7 @@
      \\/     M anipulation  |
 -------------------------------------------------------------------------------
     Copyright (C) 2015-2017 OpenFOAM Foundation
-    Copyright (C) 2019-2021 OpenCFD Ltd.
+    Copyright (C) 2019-2022 OpenCFD Ltd.
 -------------------------------------------------------------------------------
 License
     This file is part of OpenFOAM.
@@ -33,6 +33,8 @@ License
 #include "IFstream.H"
 #include "IOstreams.H"
 #include "fileOperation.H"
+#include "regIOobject.H"
+#include "UPstream.H"
 
 // * * * * * * * * * * * * * * Static Data Members * * * * * * * * * * * * * //
 
@@ -113,6 +115,16 @@ bool Foam::functionEntries::includeEtcEntry::execute
     Istream& is
 )
 {
+    const dictionary& top = parentDict.topDict();
+    const regIOobject* rioPtr = dynamic_cast<const regIOobject*>(&top);
+
+    const label oldComm
+    (
+        rioPtr && rioPtr->global()
+      ? fileHandler().comm(UPstream::worldComm)
+      : fileHandler().comm()
+    );
+
     const fileName rawName(is);
     const fileName fName(resolveEtcFile(rawName, parentDict));
 
@@ -127,9 +139,13 @@ bool Foam::functionEntries::includeEtcEntry::execute
             Info<< fName << nl;
         }
         parentDict.read(ifs);
+
+        fileHandler().comm(oldComm);
         return true;
     }
 
+    fileHandler().comm(oldComm);
+
     if (!mandatory)
     {
         return true; // Never fails if optional
@@ -153,6 +169,15 @@ bool Foam::functionEntries::includeEtcEntry::execute
     Istream& is
 )
 {
+    const dictionary& top = parentDict.topDict();
+    const regIOobject* rioPtr = dynamic_cast<const regIOobject*>(&top);
+    const label oldComm
+    (
+        rioPtr && rioPtr->global()
+      ? fileHandler().comm(UPstream::worldComm)
+      : fileHandler().comm()
+    );
+
     const fileName rawName(is);
     const fileName fName(resolveEtcFile(rawName, parentDict));
 
@@ -167,9 +192,13 @@ bool Foam::functionEntries::includeEtcEntry::execute
             Info<< fName << nl;
         }
         entry.read(parentDict, ifs);
+
+        fileHandler().comm(oldComm);
         return true;
     }
 
+    fileHandler().comm(oldComm);
+
     if (!mandatory)
     {
         return true; // Never fails if optional

src/OpenFOAM/global/argList/argList.C

@@ -1815,7 +1815,7 @@ Foam::argList::~argList()
     jobInfo.stop();     // Normal job termination
 
     // Delete file handler to flush any remaining IO
-    Foam::fileHandler(nullptr);
+    (void)Foam::fileHandler(nullptr);
 }
 
 

src/OpenFOAM/global/fileOperations/collatedFileOperation/collatedFileOperation.C

@@ -50,6 +50,12 @@ namespace fileOperations
         collatedFileOperation,
         word
     );
+    addToRunTimeSelectionTable
+    (
+        fileOperation,
+        collatedFileOperation,
+        comm
+    );
 
     float collatedFileOperation::maxThreadFileBufferSize
     (
@@ -107,45 +113,10 @@ void Foam::fileOperations::collatedFileOperation::printBanner
 
     if (printRanks)
     {
-        // Information about the ranks
+        masterUncollatedFileOperation::printRanks();
-        stringList hosts(Pstream::nProcs());
-        if (Pstream::master(comm_))
-        {
-            hosts[Pstream::myProcNo()] = hostName();
-        }
-        Pstream::gatherList(hosts);
-
-        DynamicList<label> offsetMaster(Pstream::nProcs());
-
-        forAll(hosts, ranki)
-        {
-            if (!hosts[ranki].empty())
-            {
-                offsetMaster.append(ranki);
-            }
-        }
-
-        if (offsetMaster.size() > 1)
-        {
-            DetailInfo
-                << "IO nodes:" << nl << '(' << nl;
-
-            offsetMaster.append(Pstream::nProcs());
-
-            for (label group = 1; group < offsetMaster.size(); ++group)
-            {
-                const label beg = offsetMaster[group-1];
-                const label end = offsetMaster[group];
-
-                DetailInfo
-                    << "    (" << hosts[beg].c_str() << ' '
-                    << (end-beg) << ')' << nl;
-            }
-            DetailInfo
-                << ')' << nl;
-        }
     }
 
+    //- fileModificationChecking already set by base class (masterUncollated)
     // if (IOobject::fileModificationChecking == IOobject::timeStampMaster)
     // {
     //     WarningInFunction
@@ -169,10 +140,10 @@ const
     {
         return Pstream::master(comm_);
     }
-    else if (ioRanks_.size())
+    else if (ioRanks().size())
     {
         // Found myself in IO rank
-        return ioRanks_.found(proci);
+        return ioRanks().found(proci);
     }
     else
     {
@@ -256,6 +227,38 @@ bool Foam::fileOperations::collatedFileOperation::appendObject
 }
 
 
+// * * * * * * * * * * * * * * * Local Functions * * * * * * * * * * * * * * //
+
+namespace Foam
+{
+
+// Construction helper
+static Tuple2<label, labelList> getCommPattern()
+{
+    // With useHost == false
+
+    Tuple2<label, labelList> commAndIORanks;
+
+    commAndIORanks.first() = UPstream::worldComm;
+    commAndIORanks.second() = fileOperation::getGlobalIORanks(false);
+
+    if (!commAndIORanks.second().empty())
+    {
+        // Needs its own communicator
+        commAndIORanks.first() =
+            UPstream::allocateCommunicator
+            (
+                UPstream::worldComm,
+                fileOperation::getGlobalSubRanks(false)
+            );
+    }
+
+    return commAndIORanks;
+}
+
+} // End namespace Foam
+
+
 // * * * * * * * * * * * * * * * * Constructors  * * * * * * * * * * * * * * //
 
 void Foam::fileOperations::collatedFileOperation::init(bool verbose)
@@ -276,21 +279,32 @@ Foam::fileOperations::collatedFileOperation::collatedFileOperation
 :
     masterUncollatedFileOperation
     (
-        (
+        getCommPattern(),
-            ioRanks().size()
+        false,  // distributedRoots
-          ? UPstream::allocateCommunicator
+        false   // verbose
-            (
-                UPstream::worldComm,
-                subRanks(Pstream::nProcs())
-            )
-          : UPstream::worldComm
-        ),
-        false
     ),
-    myComm_(comm_),
+    managedComm_(comm_),  // [sic] Managed here, or is worldComm
-    writer_(mag(maxThreadFileBufferSize), comm_),
+    writer_(mag(maxThreadFileBufferSize), comm_)
-    nProcs_(Pstream::nProcs()),
+{
-    ioRanks_(ioRanks())
+    init(verbose);
+}
+
+
+Foam::fileOperations::collatedFileOperation::collatedFileOperation
+(
+    const Tuple2<label, labelList>& commAndIORanks,
+    const bool distributedRoots,
+    bool verbose
+)
+:
+    masterUncollatedFileOperation
+    (
+        commAndIORanks,
+        distributedRoots,
+        false   // verbose
+    ),
+    managedComm_(-1),  // Externally managed
+    writer_(mag(maxThreadFileBufferSize), comm_)
 {
     init(verbose);
 }
@@ -299,16 +313,20 @@ Foam::fileOperations::collatedFileOperation::collatedFileOperation
 Foam::fileOperations::collatedFileOperation::collatedFileOperation
 (
     const label comm,
-    const labelList& ioRanks,
+    const labelUList& ioRanks,
-    const word& typeName,
+    const bool distributedRoots,
     bool verbose
 )
 :
-    masterUncollatedFileOperation(comm, false),
+    masterUncollatedFileOperation
-    myComm_(-1),
+    (
-    writer_(mag(maxThreadFileBufferSize), comm),
+        comm,
-    nProcs_(Pstream::nProcs()),
+        ioRanks,
-    ioRanks_(ioRanks)
+        distributedRoots,
+        false   // verbose
+    ),
+    managedComm_(-1),  // Externally managed
+    writer_(mag(maxThreadFileBufferSize), comm_)
 {
     init(verbose);
 }
@@ -321,9 +339,9 @@ Foam::fileOperations::collatedFileOperation::~collatedFileOperation()
     // Wait for any outstanding file operations
     flush();
 
-    if (myComm_ != -1 && myComm_ != UPstream::worldComm)
+    if (managedComm_ >= 0 && managedComm_ != UPstream::worldComm)
     {
-        UPstream::freeCommunicator(myComm_);
+        UPstream::freeCommunicator(managedComm_);
     }
 }
 
@@ -389,6 +407,7 @@ bool Foam::fileOperations::collatedFileOperation::writeObject
         // Note: currently still NON_ATOMIC (Dec-2022)
         masterOFstream os
         (
+            comm_,
             pathName,
             streamOpt,
             IOstreamOption::NON_APPEND,
@@ -420,7 +439,7 @@ bool Foam::fileOperations::collatedFileOperation::writeObject
         mkDir(path);
         fileName pathName(path/io.name());
 
-        if (io.global())
+        if (io.global() || io.globalObject())
         {
             if (debug)
             {
@@ -433,6 +452,7 @@ bool Foam::fileOperations::collatedFileOperation::writeObject
             // Note: currently still NON_ATOMIC (Dec-2022)
             masterOFstream os
             (
+                comm_,
                 pathName,
                 streamOpt,
                 IOstreamOption::NON_APPEND,
@@ -549,9 +569,10 @@ Foam::word Foam::fileOperations::collatedFileOperation::processorsDir
     {
         const List<int>& procs(UPstream::procID(comm_));
 
-        word procDir(processorsBaseDir+Foam::name(Pstream::nProcs()));
+        //word procDir(processorsBaseDir+Foam::name(Pstream::nProcs()));
+        word procDir(processorsBaseDir+Foam::name(nProcs_));
 
-        if (procs.size() != Pstream::nProcs())
+        if (procs.size() != nProcs_)
         {
             procDir +=
               + "_"
@@ -565,7 +586,7 @@ Foam::word Foam::fileOperations::collatedFileOperation::processorsDir
     {
         word procDir(processorsBaseDir+Foam::name(nProcs_));
 
-        if (ioRanks_.size())
+        if (ioRanks().size())
         {
             // Detect current processor number
             label proci = detectProcessorPath(fName);
@@ -575,7 +596,7 @@ Foam::word Foam::fileOperations::collatedFileOperation::processorsDir
                 // Find lowest io rank
                 label minProc = 0;
                 label maxProc = nProcs_-1;
-                for (const label ranki : ioRanks_)
+                for (const label ranki : ioRanks())
                 {
                     if (ranki >= nProcs_)
                     {
@@ -613,16 +634,4 @@ Foam::word Foam::fileOperations::collatedFileOperation::processorsDir
 }
 
 
-void Foam::fileOperations::collatedFileOperation::setNProcs(const label nProcs)
-{
-    nProcs_ = nProcs;
-
-    if (debug)
-    {
-        Pout<< "collatedFileOperation::setNProcs :"
-            << " Setting number of processors to " << nProcs_ << endl;
-    }
-}
-
-
 // ************************************************************************* //

src/OpenFOAM/global/fileOperations/collatedFileOperation/collatedFileOperation.H

@@ -6,7 +6,7 @@
      \\/     M anipulation  |
 -------------------------------------------------------------------------------
     Copyright (C) 2017 OpenFOAM Foundation
-    Copyright (C) 2019-2021 OpenCFD Ltd.
+    Copyright (C) 2019-2022 OpenCFD Ltd.
 -------------------------------------------------------------------------------
 License
     This file is part of OpenFOAM.
@@ -46,8 +46,8 @@ SourceFiles
 
 \*---------------------------------------------------------------------------*/
 
-#ifndef fileOperations_collatedFileOperation_H
+#ifndef Foam_fileOperations_collatedFileOperation_H
-#define fileOperations_collatedFileOperation_H
+#define Foam_fileOperations_collatedFileOperation_H
 
 #include "masterUncollatedFileOperation.H"
 #include "OFstreamCollator.H"
@@ -67,6 +67,12 @@ class collatedFileOperation
 :
     public masterUncollatedFileOperation
 {
+    // Private Data
+
+        //- Communicator allocated/managed by us
+        label managedComm_;
+
+
     // Private Member Functions
 
         //- Any initialisation steps after constructing
@@ -76,20 +82,9 @@ protected:
 
     // Protected Data
 
-        //- Any communicator allocated by me
-        const label myComm_;
-
         //- Threaded writer
         mutable OFstreamCollator writer_;
 
-        // For non-parallel operation
-
-            //- Number of processors (overall)
-            label nProcs_;
-
-            //- Ranks of IO handlers
-            const labelList ioRanks_;
-
 
     // Protected Member Functions
 
@@ -111,8 +106,8 @@ protected:
 
 public:
 
-        //- Runtime type information
+    //- Runtime type information
-        TypeName("collated");
+    TypeName("collated");
 
 
     // Static Data
@@ -128,12 +123,20 @@ public:
         //- Default construct
         explicit collatedFileOperation(bool verbose);
 
-        //- Construct from user communicator
+        //- Construct from communicator with specified io-ranks
+        collatedFileOperation
+        (
+            const Tuple2<label, labelList>& commAndIORanks,
+            const bool distributedRoots,
+            bool verbose
+        );
+
+        //- Construct from communicator with specified io-ranks
         collatedFileOperation
         (
             const label comm,
-            const labelList& ioRanks,
+            const labelUList& ioRanks,
-            const word& typeName,
+            const bool distributedRoots,
             bool verbose
         );
 
@@ -162,6 +165,7 @@ public:
                 const bool valid = true
             ) const;
 
+
         // Other
 
             //- Forcibly wait until all output done. Flush any cached data
@@ -172,10 +176,6 @@ public:
 
             //- Actual name of processors dir
             virtual word processorsDir(const fileName&) const;
-
-            //- Set number of processor directories/results.
-            //- Only used in decomposePar
-            virtual void setNProcs(const label nProcs);
 };
 
 

src/OpenFOAM/global/fileOperations/collatedFileOperation/hostCollatedFileOperation.C

@@ -28,7 +28,6 @@ License
 
 #include "hostCollatedFileOperation.H"
 #include "addToRunTimeSelectionTable.H"
-#include "bitSet.H"
 
 /* * * * * * * * * * * * * * * Static Member Data  * * * * * * * * * * * * * */
 
@@ -43,6 +42,12 @@ namespace fileOperations
         hostCollatedFileOperation,
         word
     );
+    addToRunTimeSelectionTable
+    (
+        fileOperation,
+        hostCollatedFileOperation,
+        comm
+    );
 
     // Register initialisation routine. Signals need for threaded mpi and
     // handles command line arguments
@@ -57,70 +62,6 @@ namespace fileOperations
 }
 
 
-// * * * * * * * * * * * * * Private Member Functions  * * * * * * * * * * * //
-
-Foam::labelList Foam::fileOperations::hostCollatedFileOperation::subRanks
-(
-    const label n
-)
-{
-    DynamicList<label> subRanks(64);
-
-    labelList mainRanks(fileOperation::ioRanks());
-    if (!mainRanks.empty())
-    {
-        if (!mainRanks.found(0))
-        {
-            FatalErrorInFunction
-                << "Rank 0 (master) should be in the IO ranks. Currently "
-                << mainRanks << nl
-                << exit(FatalError);
-        }
-
-        // The lowest numbered rank is the IO rank
-        const bitSet isIOrank(n, mainRanks);
-
-        for (label proci = Pstream::myProcNo(); proci >= 0; --proci)
-        {
-            if (isIOrank[proci])
-            {
-                // Found my master. Collect all processors with same master
-                subRanks.append(proci);
-                for
-                (
-                    label rank = proci+1;
-                    rank < n && !isIOrank[rank];
-                    ++rank
-                )
-                {
-                    subRanks.append(rank);
-                }
-                break;
-            }
-        }
-    }
-    else
-    {
-        // Normal operation: one lowest rank per hostname is the writer
-        const string myHostName(hostName());
-
-        stringList hosts(Pstream::nProcs());
-        hosts[Pstream::myProcNo()] = myHostName;
-        Pstream::allGatherList(hosts);
-
-        // Collect procs with same hostname
-        forAll(hosts, proci)
-        {
-            if (hosts[proci] == myHostName)
-            {
-                subRanks.append(proci);
-            }
-        }
-    }
-    return subRanks;
-}
-
-
 // * * * * * * * * * * * * * * * * Constructors  * * * * * * * * * * * * * * //
 
 void Foam::fileOperations::hostCollatedFileOperation::init(bool verbose)
@@ -144,24 +85,71 @@ Foam::fileOperations::hostCollatedFileOperation::hostCollatedFileOperation
         UPstream::allocateCommunicator
         (
             UPstream::worldComm,
-            subRanks(Pstream::nProcs())
+            fileOperation::getGlobalSubRanks(true)  // Host
         ),
-        (Pstream::parRun() ? labelList() : ioRanks()), // processor dirs
+        fileOperation::getGlobalIORanks(true),  // Host
-        typeName,
+        false,  // distributedRoots
-        false // verbose
+        false   // verbose
-    )
+    ),
+    managedComm_(comm_)
 {
     init(verbose);
 }
 
 
+Foam::fileOperations::hostCollatedFileOperation::hostCollatedFileOperation
+(
+    const Tuple2<label, labelList>& commAndIORanks,
+    const bool distributedRoots,
+    bool verbose
+)
+:
+    collatedFileOperation
+    (
+        commAndIORanks,
+        distributedRoots,
+        false   // verbose
+    ),
+    managedComm_(-1)  // Externally managed
+{
+    if (verbose && Foam::infoDetailLevel > 0)
+    {
+        this->printBanner(ioRanks_.size());
+    }
+}
+
+
+Foam::fileOperations::hostCollatedFileOperation::hostCollatedFileOperation
+(
+    const label comm,
+    const labelUList& ioRanks,
+    const bool distributedRoots,
+    bool verbose
+)
+:
+    collatedFileOperation
+    (
+        comm,
+        ioRanks,
+        distributedRoots,
+        false  // verbose
+    ),
+    managedComm_(-1)  // Externally managed
+{
+    if (verbose && Foam::infoDetailLevel > 0)
+    {
+        this->printBanner(ioRanks_.size());
+    }
+}
+
+
 // * * * * * * * * * * * * * * * * Destructor  * * * * * * * * * * * * * * * //
 
 Foam::fileOperations::hostCollatedFileOperation::~hostCollatedFileOperation()
 {
-    if (comm_ != -1 && comm_ != UPstream::worldComm)
+    if (managedComm_ >= 0 && managedComm_ != UPstream::worldComm)
     {
-        UPstream::freeCommunicator(comm_);
+        UPstream::freeCommunicator(managedComm_);
     }
 }
 

src/OpenFOAM/global/fileOperations/collatedFileOperation/hostCollatedFileOperation.H

@@ -6,7 +6,7 @@
      \\/     M anipulation  |
 -------------------------------------------------------------------------------
     Copyright (C) 2017-2018 OpenFOAM Foundation
-    Copyright (C) 2021 OpenCFD Ltd.
+    Copyright (C) 2021-2022 OpenCFD Ltd.
 -------------------------------------------------------------------------------
 License
     This file is part of OpenFOAM.
@@ -77,14 +77,17 @@ class hostCollatedFileOperation
 :
     public collatedFileOperation
 {
+    // Private Data
+
+        //- Communicator allocated/managed by us
+        label managedComm_;
+
+
     // Private Member Functions
 
         //- Any initialisation steps after constructing
         void init(bool verbose);
 
-        //- Get the list of processors part of this set
-        static labelList subRanks(const label n);
-
 public:
 
         //- Runtime type information
@@ -96,6 +99,23 @@ public:
         //- Default construct
         explicit hostCollatedFileOperation(const bool verbose);
 
+        //- Construct from communicator with specified io-ranks
+        hostCollatedFileOperation
+        (
+            const Tuple2<label, labelList>& commAndIORanks,
+            const bool distributedRoots,
+            bool verbose
+        );
+
+        //- Construct from communicator with specified io-ranks
+        hostCollatedFileOperation
+        (
+            const label comm,
+            const labelUList& ioRanks,
+            const bool distributedRoots,
+            bool verbose
+        );
+
 
     //- Destructor
     virtual ~hostCollatedFileOperation();

src/OpenFOAM/global/fileOperations/fileOperation/fileOperation.C

@@ -37,6 +37,7 @@ License
 #include "registerSwitch.H"
 #include "Time.H"
 #include "ITstream.H"
+#include <algorithm>
 #include <cerrno>
 #include <cinttypes>
 
@@ -46,6 +47,7 @@ namespace Foam
 {
     defineTypeNameAndDebug(fileOperation, 0);
     defineRunTimeSelectionTable(fileOperation, word);
+    defineRunTimeSelectionTable(fileOperation, comm);
 
     word fileOperation::defaultFileHandler
     (
@@ -83,6 +85,9 @@ Foam::word Foam::fileOperation::processorsBaseDir = "processors";
 
 Foam::autoPtr<Foam::fileOperation> Foam::fileOperation::fileHandlerPtr_;
 
+//- Caching (e.g. of time directories) - enabled by default
+int Foam::fileOperation::cacheLevel_(1);
+
 
 // * * * * * * * * * * * * * * * Local Functions * * * * * * * * * * * * * * //
 
@@ -221,7 +226,11 @@ void sortProcessorDirs(Foam::UList<Foam::fileOperation::dirIndex>& dirs)
 
 // * * * * * * * * * * * * * Static Member Functions * * * * * * * * * * * * //
 
-Foam::labelList Foam::fileOperation::ioRanks()
+Foam::labelList Foam::fileOperation::getGlobalIORanks
+(
+    const bitSet& useProc,
+    const bool useHost
+)
 {
     labelList ranks;
 
@@ -231,6 +240,278 @@ Foam::labelList Foam::fileOperation::ioRanks()
         is >> ranks;
     }
 
+    const label np = Pstream::nProcs(UPstream::worldComm);
+
+    bool needFilter = false;
+
+    if (ranks.size())
+    {
+        if (!ranks.found(0))
+        {
+            // Could also add silently, using append
+            FatalErrorInFunction
+                << "Rank 0 (master) should be in the IO ranks. Currently:" << nl
+                << "    " << flatOutput(ranks) << nl
+                << exit(FatalError);
+        }
+
+        // Cannot trust in user input.
+        // Sort and eliminate any duplicates
+
+        std::sort(ranks.begin(), ranks.end());
+
+        auto last = std::unique(ranks.begin(), ranks.end());
+
+        ranks.resize(static_cast<label>(last - ranks.begin()));
+
+        needFilter =
+        (
+            !useProc.empty()
+         && (useProc.size() != np || !useProc.all())
+        );
+
+        // Could also test everything first in the hope of
+        // avoiding unneeded work
+        //
+        /// if
+        /// (
+        ///     !useProc.empty()
+        ///  && (useProc.size() != np || !useProc.all())
+        /// )
+        /// {
+        ///     needFilter = false;
+        ///     for (const label ranki : ranks)
+        ///     {
+        ///         if (!useProc.test(ranki))
+        ///         {
+        ///             needFilter = true;
+        ///             break
+        ///         }
+        ///     }
+        /// }
+    }
+    else if (useHost)
+    {
+        // Use hostname
+        // Lowest rank per hostname is the IO rank
+
+        const label myProci = Pstream::myProcNo(UPstream::worldComm);
+
+        stringList hostNames(np);
+
+        // Most efficient to apply subset filter now,
+        // by simply not populating that hostName
+        if
+        (
+            // Always include master
+            Pstream::master(UPstream::worldComm)
+
+            // No filtering, or is included in subset
+         || (useProc.empty() || useProc.test(myProci))
+        )
+        {
+            hostNames[myProci] = hostName();
+        }
+
+        Pstream::gatherList
+        (
+            hostNames,
+            UPstream::msgType(),
+            UPstream::worldComm
+        );
+
+        if (Pstream::master(UPstream::worldComm))
+        {
+            DynamicList<label> hostRanks(np);
+
+            hostRanks.append(0);  // Always include master
+            label previ = 0;
+
+            for (label proci = 1; proci < hostNames.size(); ++proci)
+            {
+                if
+                (
+                    hostNames[proci].size()
+                 && hostNames[proci] != hostNames[previ]
+                )
+                {
+                    hostRanks.append(proci);
+                    previ = proci;
+                }
+            }
+
+            ranks.transfer(hostRanks);
+        }
+
+        Pstream::broadcast(ranks, UPstream::worldComm);
+    }
+
+
+    if (needFilter && ranks.size())
+    {
+        // The ranks are already sorted, no duplicate values
+        bitSet newRanks(ranks.last());
+
+        const label nRanks = ranks.size();
+
+        // Always include proc0
+        newRanks.set(0);
+
+        for
+        (
+            label idx = (ranks[0] == 0 ? 1 : 0);
+            idx < nRanks;
+            ++idx
+        )
+        {
+            label ranki = ranks[idx];
+
+            if (useProc.test(ranki))
+            {
+                // IO rank is also in processor subset
+                newRanks.set(ranki);
+            }
+            else
+            {
+                // IO rank not in processor subset.
+                // Find the next rank of this range that is
+
+                ranki = useProc.find_next(ranki);
+
+                if (ranki == -1)
+                {
+                    // No more processors for this IO group
+                    break;
+                }
+
+                // End of search, or in range
+                if ((idx == nRanks-1) || (ranki < ranks[idx+1]))
+                {
+                    newRanks.set(ranki);
+                }
+            }
+        }
+
+        ranks = newRanks.sortedToc();
+    }
+
+    return ranks;
+}
+
+
+Foam::labelList Foam::fileOperation::getGlobalSubRanks
+(
+    const bitSet& useProc,
+    const bool useHost
+)
+{
+    // Get IO ranks first
+    labelList ranks(fileOperation::getGlobalIORanks(useProc, useHost));
+
+    const label np = Pstream::nProcs(UPstream::worldComm);
+
+    const bool needFilter =
+    (
+        !useProc.empty()
+     && (useProc.size() != np || !useProc.all())
+    );
+
+
+    // Fast path - no IO ranks.
+    if (ranks.empty())
+    {
+        if (needFilter)
+        {
+            // Subset of ranks involved
+            return useProc.sortedToc();
+        }
+
+        // All ranks are involved
+        return identity(np);
+    }
+
+
+    // Build sub-ranks: the lowest numbered rank is the IO rank
+
+    // Note: the IO ranks have already been filtered/adjusted
+    //       to only include ranks that are also in useProc
+
+    const label myProci = Pstream::myProcNo(UPstream::worldComm);
+
+    if (needFilter && !useProc.test(myProci))
+    {
+        // I am not involved with any IO
+        return labelList();
+    }
+
+
+    labelRange subRange(0, np);
+
+    // Linear search for enclosing range
+    {
+        const label nIOranks = ranks.size();
+
+        // Starting with proc = 0, silently adds master (0) into IO ranks
+        label begIdx = 0;
+
+        for (label i = 0; i < nIOranks && (ranks[i] <= myProci); ++i)
+        {
+            begIdx = i;
+        }
+
+        // One-beyond end of our range (ie, where the next IO range begins)
+        const label begProc = ranks[begIdx];
+        const label endProc = (begIdx+1 < nIOranks) ? ranks[begIdx+1] : np;
+
+        subRange.reset(begProc, (endProc-begProc));
+    }
+
+    #if 0
+    // Use bitSet during search
+    {
+        // IO ranks are sorted, so last element is a good sizing estimate
+        const bitSet isIOrank(ranks.last(), ranks);
+
+        // Build sub-ranks: the lowest numbered rank is the IO rank
+        label begProc = 0;
+
+        for (label proci = myProci; proci >= 0; --proci)
+        {
+            if (isIOrank.test(proci))
+            {
+                // Found the lower rank used for IO
+                begProc = proci;
+                break;
+            }
+        }
+
+        // Find one-beyond end of our range (ie, where the next IO range begins)
+        label endProc = isIOrank.find_next(begProc);
+        if (endProc == -1) endProc = np;
+
+        subRange.reset(begProc, (endProc-begProc));
+    }
+    #endif
+
+    if (needFilter)
+    {
+        DynamicList<label> subRanks(subRange.size());
+
+        for (label proci : subRange)
+        {
+            if (useProc.test(proci))
+            {
+                subRanks.append(proci);
+            }
+        }
+
+        ranks.transfer(subRanks);
+    }
+    else
+    {
+        ranks = identity(subRange);
+    }
+
     return ranks;
 }
 
@@ -295,6 +576,42 @@ Foam::fileOperation::sortTimes
 }
 
 
+bool Foam::fileOperation::uniformFile(const fileNameList& names)
+{
+    if (names.empty())
+    {
+        return false;
+    }
+
+    const auto& object0 = names[0];
+
+    for (label i = 1; i < names.size(); ++i)
+    {
+        if (object0 != names[i])
+        {
+            return false;
+        }
+    }
+
+    return true;
+}
+
+
+bool Foam::fileOperation::uniformFile(const label comm, const fileName& name)
+{
+    if (!Pstream::parRun())
+    {
+        return true;
+    }
+
+    fileName masterName(name);
+
+    Pstream::broadcast(masterName, comm);
+
+    return returnReduceAnd((masterName == name), comm);
+}
+
+
 // * * * * * * * * * * * * Protected Member Functions  * * * * * * * * * * * //
 
 Foam::fileMonitor& Foam::fileOperation::monitor() const
@@ -415,11 +732,14 @@ Foam::fileOperation::lookupAndCacheProcessorsPath
     {
         const fileName procPath(path/pDir);
 
-        const auto iter = procsDirs_.cfind(procPath);
+        if (cacheLevel() > 0)
-
-        if (iter.found())
         {
-            return iter.val();
+            const auto iter = procsDirs_.cfind(procPath);
+
+            if (iter.found())
+            {
+                return iter.val();
+            }
         }
 
         DynamicList<dirIndex> procDirs;
@@ -449,7 +769,7 @@ Foam::fileOperation::lookupAndCacheProcessorsPath
             // Parallel and non-distributed
             // Read on master only and send to subProcs
 
-            if (Pstream::master(comm_))
+            if (Pstream::master(UPstream::worldComm))
             {
                 dirEntries = Foam::readDir(path, fileName::Type::DIRECTORY);
 
@@ -458,7 +778,7 @@ Foam::fileOperation::lookupAndCacheProcessorsPath
                     << " names to sub-processes" << endl;
             }
 
-            Pstream::broadcast(dirEntries, comm_);
+            Pstream::broadcast(dirEntries, UPstream::worldComm);
         }
         else
         {
@@ -619,7 +939,7 @@ Foam::fileOperation::lookupAndCacheProcessorsPath
             // Serial: use the number of decompositions (if found)
             if (nProcs)
             {
-                const_cast<fileOperation&>(*this).setNProcs(nProcs);
+                const_cast<fileOperation&>(*this).nProcs(nProcs);
             }
         }
 
@@ -628,10 +948,17 @@ Foam::fileOperation::lookupAndCacheProcessorsPath
 
         if (procDirsStatus & 2u)
         {
-            procsDirs_.insert(procPath, procDirs);
+            if (cacheLevel() > 0)
+            {
+                procsDirs_.insert(procPath, procDirs);
 
-            // Make sure to return a reference
+                // Make sure to return a reference
-            return procsDirs_[procPath];
+                return procsDirs_[procPath];
+            }
+            else
+            {
+                return refPtr<dirIndexList>::New(procDirs);
+            }
         }
     }
 
@@ -692,14 +1019,30 @@ bool Foam::fileOperation::exists(IOobject& io) const
 
 // * * * * * * * * * * * * * * * * Constructors  * * * * * * * * * * * * * * //
 
+Foam::fileOperation::fileOperation
+(
+    const Tuple2<label, labelList>& commAndIORanks,
+    const bool distributedRoots
+)
+:
+    comm_(commAndIORanks.first()),
+    nProcs_(Pstream::nProcs(UPstream::worldComm)),
+    distributed_(distributedRoots),
+    ioRanks_(commAndIORanks.second())
+{}
+
+
 Foam::fileOperation::fileOperation
 (
     const label comm,
+    const labelUList& ioRanks,
     const bool distributedRoots
 )
 :
     comm_(comm),
-    distributed_(distributedRoots)
+    nProcs_(Pstream::nProcs(UPstream::worldComm)),
+    distributed_(distributedRoots),
+    ioRanks_(ioRanks)
 {}
 
 
@@ -710,6 +1053,18 @@ Foam::fileOperation::New
     bool verbose
 )
 {
+    if (handlerType.empty())
+    {
+        if (fileOperation::defaultFileHandler.empty())
+        {
+            FatalErrorInFunction
+                << "defaultFileHandler name is undefined" << nl
+                << abort(FatalError);
+        }
+
+        return fileOperation::New(fileOperation::defaultFileHandler, verbose);
+    }
+
     DebugInFunction
         << "Constructing fileHandler" << endl;
 
@@ -729,16 +1084,65 @@ Foam::fileOperation::New
 }
 
 
-// * * * * * * * * * * * * * * * Member Functions  * * * * * * * * * * * * * //
+Foam::autoPtr<Foam::fileOperation>
-
+Foam::fileOperation::New
-bool Foam::fileOperation::distributed(bool on) const noexcept
+(
+    const word& handlerType,
+    const label comm,
+    const labelUList& ioRanks,
+    const bool distributedRoots,
+    bool verbose
+)
 {
-    bool old(distributed_);
+    if (handlerType.empty())
-    distributed_ = on;
+    {
-    return old;
+        if (fileOperation::defaultFileHandler.empty())
+        {
+            FatalErrorInFunction
+                << "defaultFileHandler name is undefined" << nl
+                << abort(FatalError);
+        }
+
+        return fileOperation::New
+        (
+            fileOperation::defaultFileHandler,
+            comm,
+            ioRanks,
+            distributedRoots,
+            verbose
+        );
+    }
+
+    DebugInFunction
+        << "Constructing fileHandler" << endl;
+
+    auto* ctorPtr = commConstructorTable(handlerType);
+
+    if (!ctorPtr)
+    {
+        FatalErrorInLookup
+        (
+            "fileHandler",
+            handlerType,
+            *commConstructorTablePtr_
+        ) << abort(FatalError);
+    }
+
+    return autoPtr<fileOperation>
+    (
+        ctorPtr
+        (
+            comm,
+            ioRanks,
+            distributedRoots,
+            verbose
+        )
+    );
 }
 
 
+// * * * * * * * * * * * * * * * Member Functions  * * * * * * * * * * * * * //
+
 Foam::fileName Foam::fileOperation::objectPath
 (
     const IOobject& io,
@@ -1209,8 +1613,18 @@ Foam::fileNameList Foam::fileOperation::readObjects
 }
 
 
-void Foam::fileOperation::setNProcs(const label nProcs)
+Foam::label Foam::fileOperation::nProcs(const label numProcs)
-{}
+{
+    if (debug)
+    {
+        Pout<< "fileOperation::nProcs :"
+            << " Setting number of processors to " << numProcs << endl;
+    }
+
+    label old(nProcs_);
+    nProcs_ = numProcs;
+    return old;
+}
 
 
 Foam::label Foam::fileOperation::nProcs
@@ -1273,6 +1687,23 @@ void Foam::fileOperation::flush() const
 }
 
 
+void Foam::fileOperation::sync()
+{
+    if (debug)
+    {
+        Pout<< "fileOperation::sync : parallel synchronisation"
+            << endl;
+    }
+
+    Pstream::broadcasts
+    (
+        UPstream::worldComm,
+        nProcs_,
+        procsDirs_
+    );
+}
+
+
 Foam::fileName Foam::fileOperation::processorsCasePath
 (
     const IOobject& io,
@@ -1511,37 +1942,56 @@ const Foam::fileOperation& Foam::fileHandler()
 {
     if (!fileOperation::fileHandlerPtr_)
     {
-        word handler(Foam::getEnv("FOAM_FILEHANDLER"));
+        word handlerType(Foam::getEnv("FOAM_FILEHANDLER"));
 
-        if (handler.empty())
+        if (handlerType.empty())
         {
-            handler = fileOperation::defaultFileHandler;
+            handlerType = fileOperation::defaultFileHandler;
         }
 
-        fileOperation::fileHandlerPtr_ = fileOperation::New(handler, true);
+        fileOperation::fileHandlerPtr_ = fileOperation::New(handlerType, true);
     }
 
     return *fileOperation::fileHandlerPtr_;
 }
 
 
+Foam::autoPtr<Foam::fileOperation>
+Foam::fileHandler(std::nullptr_t)
+{
+    return autoPtr<fileOperation>(fileOperation::fileHandlerPtr_.release());
+}
+
+
 Foam::autoPtr<Foam::fileOperation>
 Foam::fileHandler(autoPtr<fileOperation>&& newHandler)
 {
+    // - do nothing if newHandler is empty. Does not delete current
+    // - do nothing if newHandler is identical to current handler
+
+    // Change ownership as atomic operations
+
+    // If newHandler and current handler are actually identical, we
+    // have a bit problem somewhere else since this means that the pointer
+    // is managed is done in two places!
+    // Should flag as a FatalError (in the future), but there may still be
+    // some place where we would like to fake shared pointers?
+
+    // TBD: add a flush() operation on the old handler first,
+    // instead of waiting for it to be run on destruction?
+
+    autoPtr<fileOperation> old;
+
     if
     (
-        newHandler
+        newHandler.get() != nullptr
-     && fileOperation::fileHandlerPtr_
+     && newHandler.get() != fileOperation::fileHandlerPtr_.get()
-     && newHandler->type() == fileOperation::fileHandlerPtr_->type()
     )
     {
-        return nullptr;  // No change
+        old.reset(newHandler.release());
+        old.swap(fileOperation::fileHandlerPtr_);
     }
 
-    autoPtr<fileOperation> old(std::move(fileOperation::fileHandlerPtr_));
-
-    fileOperation::fileHandlerPtr_ = std::move(newHandler);
-
     return old;
 }
 

src/OpenFOAM/global/fileOperations/fileOperation/fileOperation.H

@@ -43,11 +43,12 @@ Description
 
 #include "ISstream.H"
 #include "Ostream.H"
-#include "autoPtr.H"
+#include "fileMonitor.H"
 #include "fileNameList.H"
 #include "instantList.H"
-#include "fileMonitor.H"
+#include "autoPtr.H"
 #include "refPtr.H"
+#include "bitSet.H"
 #include "Enum.H"
 #include "Tuple2.H"
 
@@ -108,14 +109,29 @@ public:
 
 protected:
 
+    // Protected Static Data
+
+        //- Cache level (eg, for caching time directories). Default: 1
+        static int cacheLevel_;
+
+
     // Protected Data
 
         //- Communicator to use
-        const label comm_;
+        mutable label comm_;
+
+        //- Overall number of processors.
+        //  Used to synthesise processor directory naming:
+        //  - parallel: Pstream::nProcs()
+        //  - non-parallel: detected from processor dir naming ('processorsNN')
+        label nProcs_;
 
         //- Distributed roots (parallel run)
         mutable bool distributed_;
 
+        //- Ranks (in comm_) of IO handlers
+        const labelList ioRanks_;
+
         //- Detected processors directories
         mutable HashTable<dirIndexList> procsDirs_;
 
@@ -128,9 +144,6 @@ protected:
         //- Get or create fileMonitor singleton
         fileMonitor& monitor() const;
 
-        //- Retrieve list of IO ranks from FOAM_IORANKS env variable
-        static labelList ioRanks();
-
         //- Merge two times
         static void mergeTimes
         (
@@ -189,13 +202,26 @@ public:
 
     // Constructors
 
-        //- Construct from communicator, optionally with distributed roots
+        //- Construct from communicator,
+        //- optionally with specified io-ranks and/or distributed roots
         explicit fileOperation
         (
             const label comm,
+            const labelUList& ioRanks = labelUList::null(),
             const bool distributedRoots = false
         );
 
+        //- Construct from communicator with specified io-ranks
+        explicit fileOperation
+        (
+            const Tuple2<label, labelList>& commAndIORanks,
+            const bool distributedRoots = false
+        );
+
+
+        //- Clone fileHandler
+        /// virtual autoPtr<fileOperation> clone() const = 0;
+
 
     // Declare run-time constructor selection table
 
@@ -210,16 +236,42 @@ public:
             (verbose)
         );
 
+        declareRunTimeSelectionTable
+        (
+            autoPtr,
+            fileOperation,
+            comm,
+            (
+                const label comm,
+                const labelUList& ioRanks,
+                const bool distributedRoots,
+                bool verbose
+            ),
+            (comm, ioRanks, distributedRoots, verbose)
+        );
+
 
     // Selectors
 
-        //- Select fileHandler-type
+        //- Select fileHandler-type.
+        //- Uses defaultFileHandler if the handlerType is empty.
         static autoPtr<fileOperation> New
         (
             const word& handlerType,
             bool verbose = false
         );
 
+        //- Select fileHandler-type.
+        //- Uses defaultFileHandler if the handlerType is empty.
+        static autoPtr<fileOperation> New
+        (
+            const word& handlerType,
+            const label comm,
+            const labelUList& ioRanks,
+            const bool distributedRoots,
+            bool verbose
+        );
+
 
     //- Destructor
     virtual ~fileOperation() = default;
@@ -227,6 +279,20 @@ public:
 
    // Static Functions
 
+        //- Return cache level
+        static int cacheLevel() noexcept
+        {
+            return cacheLevel_;
+        }
+
+        //- Set cache level (0 = off). \return the previous value
+        static int cacheLevel(int level) noexcept
+        {
+            int old(cacheLevel_);
+            cacheLevel_ = level;
+            return old;
+        }
+
         //- Sort directory entries according to time value,
         //  with "constant" appearing first (if it exists)
         static instantList sortTimes
@@ -235,19 +301,55 @@ public:
             const word& constantName = "constant"
         );
 
+        //- True if the file names are identical. False on an empty list
+        static bool uniformFile(const fileNameList& names);
+
+        //- True if the file name is identical on all ranks
+        static bool uniformFile(const label comm, const fileName& name);
+
 
     // Member Functions
 
+    // Characteristics
+
+        //- Communicator to use
+        label comm() const noexcept
+        {
+            return comm_;
+        }
+
+        //- Set communicator to use [mutable]. Negative values are a no-op.
+        //  \return old value
+        label comm(label communicator) const noexcept
+        {
+            label old(comm_);
+            if (communicator > 0) comm_ = communicator;
+            return old;
+        }
+
         //- Distributed roots (parallel run)
         bool distributed() const noexcept
         {
             return distributed_;
         }
 
-        //- Set distributed roots on/off (mutable)
+        //- Set distributed roots on/off [mutable]
         //  \return old value
-        bool distributed(bool on) const noexcept;
+        bool distributed(bool on) const noexcept
+        {
+            bool old(distributed_);
+            distributed_ = on;
+            return old;
+        }
 
+        //- Retrieve list of IO ranks (in the local communicator)
+        virtual const labelList& ioRanks() const
+        {
+            return ioRanks_;
+        }
+
+
+    // Member Functions
 
         // OSSpecific equivalents
 
@@ -531,12 +633,28 @@ public:
                 return processorsBaseDir;
             }
 
-            //- Set number of processor directories/results. Only used in
+            //- Overall number of processors,
-            //  decomposePar
+            //- eg, detected from directories/results.
-            virtual void setNProcs(const label nProcs);
+            virtual label nProcs() const
+            {
+                return nProcs_;
+            }
 
-            //- Get number of processor directories/results. Used for e.g.
+            //- Set number of processor directories/results.
-            //  reconstructPar, argList checking
+            //  Used to cache format of e.g. processorsDDD.
+            //  Returns old number of processors.
+            //  Only used in decomposePar
+            virtual label nProcs(const label numProcs);
+
+            //- Set number of processor directories/results.
+            //  Only used in decomposePar
+            virtual label setNProcs(const label numProcs)
+            {
+                 return nProcs(numProcs);
+            }
+
+            //- Get number of processor directories/results.
+            //  Used for e.g. reconstructPar, argList checking
             virtual label nProcs
             (
                 const fileName& dir,
@@ -564,6 +682,9 @@ public:
             //- Forcibly wait until all output done. Flush any cached data
             virtual void flush() const;
 
+            //- Forcibly parallel sync
+            virtual void sync();
+
             //- Generate path (like io.path) from root+casename with any
             //  'processorXXX' replaced by procDir (usually 'processsors')
             fileName processorsCasePath
@@ -603,6 +724,40 @@ public:
 
             //- Detect processor number from '/aa/bb/processorDDD/cc'
             static label detectProcessorPath(const fileName& objPath);
+
+
+    // Rank selection/sub-selection
+
+        //- Get the list of global IO ranks from FOAM_IORANKS env variable,
+        //- subsetted by a list of processor ranks,
+        //- with optional fallback by hostname
+        static labelList getGlobalIORanks
+        (
+            const bitSet& useProc = bitSet::null(),
+            const bool useHost = false
+        );
+
+        //- Get the list of global IO ranks from FOAM_IORANKS env variable,
+        //- no subsetting, with optional fallback by hostname
+        static labelList getGlobalIORanks(const bool useHost)
+        {
+            return fileOperation::getGlobalIORanks(bitSet::null(), useHost);
+        }
+
+        //- Get the list of global ranks that share the same IO rank
+        //- as myProcNo, subjected to subsetting by a list of processor ranks
+        static labelList getGlobalSubRanks
+        (
+            const bitSet& useProc = bitSet::null(),
+            const bool useHost = false
+        );
+
+        //- Get the list of global ranks that share the same IO rank.
+        //- as myProcNo.  Uses getGlobalIORanks
+        static labelList getGlobalSubRanks(const bool useHost)
+        {
+            return fileOperation::getGlobalSubRanks(bitSet::null(), useHost);
+        }
 };
 
 
@@ -628,11 +783,21 @@ inline Ostream& operator<<(Ostream& os, const fileOperation::pathType b)
 
 // Note: defined in fileOperation.C
 
-//- Get current file handler
+//- Return the current file handler.
+//- Will create the default file handler if necessary.
 const fileOperation& fileHandler();
 
-//- Replace, reset file handler.
+//- Delete current file handler.
-//  \return old handler on change, null otherwise
+//  \returns the old handler
+autoPtr<fileOperation> fileHandler(std::nullptr_t);
+
+//- Replace the current file handler.
+//  The following are considered no-ops:
+//  - an empty/invalid newHandler does \b not delete,
+//    use fileHandler(std::nullptr_t) - ie, a literal \c nullptr for that
+//  - if new handler and current handler are identical (same pointer).
+//  .
+//  \returns the old handler (on change), nullptr otherwise
 autoPtr<fileOperation> fileHandler(autoPtr<fileOperation>&& newHandler);
 
 

src/OpenFOAM/global/fileOperations/masterUncollatedFileOperation/masterUncollatedFileOperation.H

@@ -88,6 +88,12 @@ class masterUncollatedFileOperation
 :
     public fileOperation
 {
+    // Private Data
+
+        //- Communicator allocated/managed by us
+        label managedComm_;
+
+
     // Private Member Functions
 
         //- Any initialisation steps after constructing
@@ -97,9 +103,6 @@ protected:
 
     // Protected Data
 
-        //- Any communicator allocated by me
-        const label myComm_;
-
         //- Cached times for a given directory
         mutable HashPtrTable<DynamicList<instant>> times_;
 
@@ -388,9 +391,6 @@ protected:
 
     // Private Member Functions
 
-        //- Get the list of processors that are part of this communicator
-        static labelList subRanks(const label n);
-
         template<class Type>
         Type scatterList(const UList<Type>&, const int, const label comm) const;
 
@@ -432,6 +432,7 @@ protected:
             const bool checkGlobal,
             const bool isFile,
             const IOobject& io,
+            const dirIndexList& pDirs,
             const bool search,
             pathType& searchType,
             word& processorsDir,
@@ -470,6 +471,8 @@ protected:
         //  without parent searchign and instance searching
         bool exists(const dirIndexList&, IOobject& io) const;
 
+        //- Helper: output which ranks are IO
+        void printRanks() const;
 
 public:
 
@@ -490,8 +493,22 @@ public:
         //- Default construct
         explicit masterUncollatedFileOperation(bool verbose);
 
-        //- Construct from communicator
+        //- Construct from communicator with specified io-ranks
-        masterUncollatedFileOperation(const label comm, bool verbose);
+        masterUncollatedFileOperation
+        (
+            const Tuple2<label, labelList>& commAndIORanks,
+            const bool distributedRoots,
+            bool verbose
+        );
+
+        //- Construct from communicator with specified io-ranks
+        masterUncollatedFileOperation
+        (
+            const label comm,
+            const labelUList& ioRanks,
+            const bool distributedRoots,
+            bool verbose
+        );
 
 
     //- Destructor
@@ -744,9 +761,6 @@ public:
 
         // Other
 
-            //- Same file?
-            static bool uniformFile(const fileNameList&);
-
             //- Get sorted list of times
             virtual instantList findTimes(const fileName&, const word&) const;
 
@@ -767,6 +781,9 @@ public:
             //- Forcibly wait until all output done. Flush any cached data
             virtual void flush() const;
 
+            //- Forcibly parallel sync
+            virtual void sync();
+
             //- Return cached times
             const HashPtrTable<DynamicList<instant>>& times() const noexcept
             {

src/OpenFOAM/global/fileOperations/uncollatedFileOperation/hostUncollatedFileOperation.C

@@ -0,0 +1,155 @@
+/*---------------------------------------------------------------------------*\
+  =========                 |
+  \\      /  F ield         | OpenFOAM: The Open Source CFD Toolbox
+   \\    /   O peration     |
+    \\  /    A nd           | www.openfoam.com
+     \\/     M anipulation  |
+-------------------------------------------------------------------------------
+    Copyright (C) 2022 OpenCFD Ltd.
+-------------------------------------------------------------------------------
+License
+    This file is part of OpenFOAM.
+
+    OpenFOAM is free software: you can redistribute it and/or modify it
+    under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    OpenFOAM is distributed in the hope that it will be useful, but WITHOUT
+    ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+    FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+    for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with OpenFOAM.  If not, see <http://www.gnu.org/licenses/>.
+
+\*---------------------------------------------------------------------------*/
+
+#include "hostUncollatedFileOperation.H"
+#include "addToRunTimeSelectionTable.H"
+
+/* * * * * * * * * * * * * * * Static Member Data  * * * * * * * * * * * * * */
+
+namespace Foam
+{
+namespace fileOperations
+{
+    defineTypeNameAndDebug(hostUncollatedFileOperation, 0);
+    addToRunTimeSelectionTable
+    (
+        fileOperation,
+        hostUncollatedFileOperation,
+        word
+    );
+    addToRunTimeSelectionTable
+    (
+        fileOperation,
+        hostUncollatedFileOperation,
+        comm
+    );
+
+    // Register initialisation routine. Signals need for threaded mpi and
+    // handles command line arguments
+    addNamedToRunTimeSelectionTable
+    (
+        fileOperationInitialise,
+        hostUncollatedFileOperationInitialise,
+        word,
+        hostUncollated
+    );
+}
+}
+
+
+// * * * * * * * * * * * * * * * * Constructors  * * * * * * * * * * * * * * //
+
+void Foam::fileOperations::hostUncollatedFileOperation::init(bool verbose)
+{
+    verbose = (verbose && Foam::infoDetailLevel > 0);
+
+    if (verbose)
+    {
+        DetailInfo
+            << "I/O    : " << this->type() << nl;
+
+        if (ioRanks().size())
+        {
+            masterUncollatedFileOperation::printRanks();
+        }
+    }
+}
+
+
+Foam::fileOperations::hostUncollatedFileOperation::hostUncollatedFileOperation
+(
+    bool verbose
+)
+:
+    masterUncollatedFileOperation
+    (
+        UPstream::allocateCommunicator
+        (
+            UPstream::worldComm,
+            fileOperation::getGlobalSubRanks(true)  // Host
+        ),
+        fileOperation::getGlobalIORanks(true),  // Host
+        false,  // distributedRoots
+        false   // verbose
+    ),
+    managedComm_(comm_)
+{
+    init(verbose);
+}
+
+
+Foam::fileOperations::hostUncollatedFileOperation::hostUncollatedFileOperation
+(
+    const Tuple2<label, labelList>& commAndIORanks,
+    const bool distributedRoots,
+    bool verbose
+)
+:
+    masterUncollatedFileOperation
+    (
+        commAndIORanks,
+        distributedRoots,
+        false   // verbose
+    ),
+    managedComm_(-1)  // Externally managed
+{
+    init(verbose);
+}
+
+
+Foam::fileOperations::hostUncollatedFileOperation::
+hostUncollatedFileOperation
+(
+    const label comm,
+    const labelUList& ioRanks,
+    const bool distributedRoots,
+    bool verbose
+)
+:
+    masterUncollatedFileOperation(comm, ioRanks, distributedRoots, false),
+    managedComm_(comm_)
+{
+    init(verbose);
+}
+
+
+// * * * * * * * * * * * * * * * * Destructor  * * * * * * * * * * * * * * * //
+
+Foam::fileOperations::hostUncollatedFileOperation::
+~hostUncollatedFileOperation()
+{
+    // Wait for any outstanding file operations
+    flush();
+
+    if (myComm_ != -1 && myComm_ != UPstream::worldComm)
+    {
+        UPstream::freeCommunicator(comm_);
+    }
+}
+
+
+// ************************************************************************* //

src/OpenFOAM/global/fileOperations/uncollatedFileOperation/hostUncollatedFileOperation.H

@@ -0,0 +1,132 @@
+/*---------------------------------------------------------------------------*\
+  =========                 |
+  \\      /  F ield         | OpenFOAM: The Open Source CFD Toolbox
+   \\    /   O peration     |
+    \\  /    A nd           | www.openfoam.com
+     \\/     M anipulation  |
+-------------------------------------------------------------------------------
+    Copyright (C) 2022 OpenCFD Ltd.
+-------------------------------------------------------------------------------
+License
+    This file is part of OpenFOAM.
+
+    OpenFOAM is free software: you can redistribute it and/or modify it
+    under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    OpenFOAM is distributed in the hope that it will be useful, but WITHOUT
+    ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+    FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+    for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with OpenFOAM.  If not, see <http://www.gnu.org/licenses/>.
+
+Class
+    Foam::fileOperations::hostUncollatedFileOperation
+
+Description
+    Version of masterUncollated with host-based IO ranks
+
+\*---------------------------------------------------------------------------*/
+
+#ifndef Foam_fileOperations_hostUncollatedFileOperation_H
+#define Foam_fileOperations_hostUncollatedFileOperation_H
+
+#include "masterUncollatedFileOperation.H"
+
+// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
+
+namespace Foam
+{
+namespace fileOperations
+{
+
+/*---------------------------------------------------------------------------*\
+                 Class hostUncollatedFileOperation Declaration
+\*---------------------------------------------------------------------------*/
+
+class hostUncollatedFileOperation
+:
+    public masterUncollatedFileOperation
+{
+    // Private Data
+
+        //- Communicator allocated/managed by us
+        label managedComm_;
+
+
+    // Private Member Functions
+
+        //- Any initialisation steps after constructing
+        void init(bool verbose);
+
+public:
+
+        //- Runtime type information
+        TypeName("hostUncollated");
+
+
+    // Constructors
+
+        //- Default construct
+        explicit hostUncollatedFileOperation(bool verbose);
+
+        //- Construct from communicator with specified io-ranks
+        hostUncollatedFileOperation
+        (
+            const Tuple2<label, labelList>& commAndIORanks,
+            const bool distributedRoots,
+            bool verbose
+        );
+
+        //- Construct from communicator with specified io-ranks
+        hostUncollatedFileOperation
+        (
+            const label comm,
+            const labelUList& ioRanks,
+            const bool distributedRoots,
+            bool verbose
+        );
+
+
+    //- Destructor
+    virtual ~hostUncollatedFileOperation();
+};
+
+
+/*---------------------------------------------------------------------------*\
+            Class hostUncollatedFileOperationInitialise Declaration
+\*---------------------------------------------------------------------------*/
+
+class hostUncollatedFileOperationInitialise
+:
+    public masterUncollatedFileOperationInitialise
+{
+public:
+
+    // Constructors
+
+        //- Construct from components
+        hostUncollatedFileOperationInitialise(int& argc, char**& argv)
+        :
+            masterUncollatedFileOperationInitialise(argc, argv)
+        {}
+
+
+    //- Destructor
+    virtual ~hostUncollatedFileOperationInitialise() = default;
+};
+
+
+// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
+
+} // End namespace fileOperations
+} // End namespace Foam
+
+// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
+
+#endif
+
+// ************************************************************************* //

src/OpenFOAM/global/fileOperations/uncollatedFileOperation/uncollatedFileOperation.C

@@ -42,6 +42,12 @@ namespace fileOperations
 {
     defineTypeNameAndDebug(uncollatedFileOperation, 0);
     addToRunTimeSelectionTable(fileOperation, uncollatedFileOperation, word);
+    addToRunTimeSelectionTable
+    (
+        fileOperation,
+        uncollatedFileOperation,
+        comm
+    );
 
     // Mark as not needing threaded mpi
     addNamedToRunTimeSelectionTable
@@ -159,7 +165,7 @@ Foam::fileName Foam::fileOperations::uncollatedFileOperation::filePathInfo
         }
     }
 
-    return fileName::null;
+    return fileName();
 }
 
 
@@ -174,6 +180,46 @@ Foam::fileOperations::uncollatedFileOperation::lookupProcessorsPath
 }
 
 
+// * * * * * * * * * * * * * * * Local Functions * * * * * * * * * * * * * * //
+
+namespace Foam
+{
+
+// Construction helper
+static Tuple2<label, labelList> getCommPattern()
+{
+    // With useHost == false
+
+    Tuple2<label, labelList> commAndIORanks;
+
+    if (fileOperation::getGlobalIORanks(false).empty())
+    {
+        // No specified IO ranks
+        // Master of communicator is the one that writes
+        commAndIORanks.first() =
+            UPstream::allocateCommunicator
+            (
+                UPstream::worldComm,
+                labelList(Foam::one{}, Pstream::myProcNo(UPstream::worldComm))
+            );
+    }
+    else
+    {
+        // Group by IO ranks
+        commAndIORanks.first() =
+            UPstream::allocateCommunicator
+            (
+                UPstream::worldComm,
+                fileOperation::getGlobalSubRanks(false)
+            );
+    }
+
+    return commAndIORanks;
+}
+
+} // End namespace Foam
+
+
 // * * * * * * * * * * * * * * * * Constructors  * * * * * * * * * * * * * * //
 
 void Foam::fileOperations::uncollatedFileOperation::init(bool verbose)
@@ -193,12 +239,59 @@ Foam::fileOperations::uncollatedFileOperation::uncollatedFileOperation
     bool verbose
 )
 :
-    fileOperation(Pstream::worldComm)
+    fileOperation
+    (
+        getCommPattern()
+    ),
+    managedComm_(comm_)
 {
     init(verbose);
 }
 
 
+Foam::fileOperations::uncollatedFileOperation::uncollatedFileOperation
+(
+    const Tuple2<label, labelList>& commAndIORanks,
+    const bool distributedRoots,
+    bool verbose
+)
+:
+    fileOperation(commAndIORanks, distributedRoots),
+    managedComm_(-1)  // Externally managed
+{
+    init(verbose);
+}
+
+
+Foam::fileOperations::uncollatedFileOperation::uncollatedFileOperation
+(
+    const label comm,
+    const labelUList& ioRanks,
+    const bool distributedRoots,
+    bool verbose
+)
+:
+    fileOperation(comm, ioRanks, distributedRoots),
+    managedComm_(-1)  // Externally managed
+{
+    init(verbose);
+}
+
+
+// * * * * * * * * * * * * * * * * Destructor  * * * * * * * * * * * * * * * //
+
+Foam::fileOperations::uncollatedFileOperation::~uncollatedFileOperation()
+{
+    // Wait for any outstanding file operations
+    flush();
+
+    if (managedComm_ >= 0 && managedComm_ != UPstream::worldComm)
+    {
+        UPstream::freeCommunicator(managedComm_);
+    }
+}
+
+
 // * * * * * * * * * * * * * Filesystem Operations * * * * * * * * * * * * * //
 
 bool Foam::fileOperations::uncollatedFileOperation::mkDir

src/OpenFOAM/global/fileOperations/uncollatedFileOperation/uncollatedFileOperation.H

@@ -53,6 +53,12 @@ class uncollatedFileOperation
 :
     public fileOperation
 {
+    // Private Data
+
+        //- Communicator allocated/managed by us
+        label managedComm_;
+
+
     // Private Member Functions
 
         //- Any initialisation steps after constructing
@@ -84,8 +90,8 @@ protected:
 
 public:
 
-        //- Runtime type information
+    //- Runtime type information
-        TypeName("uncollated");
+    TypeName("uncollated");
 
 
     // Constructors
@@ -93,9 +99,26 @@ public:
         //- Default construct
         explicit uncollatedFileOperation(bool verbose);
 
+        //- Construct from communicator with specified io-ranks
+        uncollatedFileOperation
+        (
+            const Tuple2<label, labelList>& commAndIORanks,
+            const bool distributedRoots,
+            bool verbose
+        );
+
+        //- Construct from communicator with specified io-ranks
+        uncollatedFileOperation
+        (
+            const label comm,
+            const labelUList& ioRanks,
+            const bool distributedRoots,
+            bool verbose
+        );
+
 
     //- Destructor
-    virtual ~uncollatedFileOperation() = default;
+    virtual ~uncollatedFileOperation();
 
 
     // Member Functions

wmake/scripts/sysFunctions

@@ -5,7 +5,7 @@
 #   \\  /    A nd           | www.openfoam.com
 #    \\/     M anipulation  |
 #------------------------------------------------------------------------------
-#     Copyright (C) 2018-2020 OpenCFD Ltd.
+#     Copyright (C) 2018-2022 OpenCFD Ltd.
 #------------------------------------------------------------------------------
 # License
 #     This file is part of OpenFOAM, distributed under GPL-3.0-or-later.
@@ -212,7 +212,7 @@ then
     findLibrary()
     {
         local prefixDir localDir searchDir searchName
-        local file ext
+        local file found ext zshsplit
 
         searchDir=true
 
@@ -264,6 +264,13 @@ then
 
             ## echo "search: $# $@" 1>&2
 
+            # Split extLibraries on space, but zsh does not like that
+            if [ -n "$ZSH_VERSION" ]
+            then
+                case "$-" in (*y*) zshsplit=y;; esac
+                setopt shwordsplit
+            fi
+
             for searchDir in "$@"
             do
                 [ -n "$searchDir" ] || continue
@@ -272,8 +279,8 @@ then
                     file="$prefixDir/$searchDir/$searchName$ext"
                     if [ -f "$file" ] && [ -r "$file" ]
                     then
-                        echo "$file"  # Found
+                        found="$file"  # Found
-                        return 0
+                        break 2
                     fi
                 done
             done
@@ -281,6 +288,13 @@ then
         else
             # Directed search
 
+            # Split extLibraries on space, but zsh does not like that
+            if [ -n "$ZSH_VERSION" ]
+            then
+                case "$-" in (*y*) zshsplit=y;; esac
+                setopt shwordsplit
+            fi
+
             for file
             do
                 [ -n "$file" ] || continue
@@ -288,13 +302,27 @@ then
                 do
                     if [ -f "$file$ext" ] && [ -r "$file$ext" ]
                     then
-                        echo "$file$ext"  # Found
+                        found="$file$ext"  # Found
-                        return 0
+                        break 2
                     fi
                 done
             done
         fi
 
+        # Restore word splitting (zsh)
+
+        # Restore case "$-" in (*y*) zshsplit=y;; esac
+        if [ -n "$ZSH_VERSION" ] && [ -z "$zshsplit" ]
+        then
+            unsetopt shwordsplit
+        fi
+
+        if [ -n "$found" ]
+        then
+            echo "$found"
+            return 0
+        fi
+
         return 2
     }