ENH: hostCollated: revert to collated naming if running on single proc

Can be done since processorsDirs now synced.

applications/utilities/parallelProcessing/redistributePar/loadOrCreateMesh.C

@@ -6,7 +6,7 @@
      \\/     M anipulation  |
 -------------------------------------------------------------------------------
     Copyright (C) 2012-2017 OpenFOAM Foundation
-    Copyright (C) 2015-2022 OpenCFD Ltd.
+    Copyright (C) 2015-2023 OpenCFD Ltd.
 -------------------------------------------------------------------------------
 License
     This file is part of OpenFOAM.
@@ -30,9 +30,104 @@ License
 #include "faMesh.H"
 #include "Pstream.H"
 #include "OSspecific.H"
+#include "decomposedBlockData.H"
 
 // * * * * * * * * * * * * * * * Global Functions  * * * * * * * * * * * * * //
 
+bool Foam::checkFileExistence(const fileName& fName)
+{
+    // Trimmed-down version of lookupAndCacheProcessorsPath
+    // with Foam::exists() check. No caching.
+
+    // Check for two conditions:
+    // - file has to exist
+    // - if collated the entry has to exist inside the file
+
+    // Note: bypass fileOperation::filePath(IOobject&) since has problems
+    //       with going to a different number of processors
+    //       (in collated format). Use file-based searching instead
+
+    const auto& handler = Foam::fileHandler();
+    typedef fileOperation::procRangeType procRangeType;
+
+    fileName path, pDir, local;
+    procRangeType group;
+    label numProcs;
+    const label proci =
+        fileOperation::splitProcessorPath
+        (fName, path, pDir, local, group, numProcs);
+
+    bool found = false;
+
+    if (proci != -1)
+    {
+        // Read all directories to see any beginning with processor
+        // ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+        const fileNameList dirEntries
+        (
+            handler.readDir(path, fileName::Type::DIRECTORY)
+        );
+
+        // Extract info from processorN or processorsNN
+        // - highest processor number
+        // - directory+offset containing data for proci
+
+        // label nProcs = 0;
+        for (const fileName& dirN : dirEntries)
+        {
+            // Analyse directory name
+            fileName rp, rd, rl;
+            label rNum;
+            const label readProci =
+                fileOperation::splitProcessorPath
+                (dirN, rp, rd, rl, group, rNum);
+
+            if (proci == readProci)
+            {
+                // Found "processorN"
+                if (Foam::exists(path/dirN/local))
+                {
+                    found = true;
+                    break;
+                }
+            }
+            else if (rNum != -1)
+            {
+                // "processorsNN" or "processorsNN_start-end"
+                if (group.empty())
+                {
+                    // "processorsNN"
+                    if (proci < rNum && Foam::exists(path/dirN/local))
+                    {
+                        found = true;
+                        break;
+                    }
+                }
+                else if (group.contains(proci))
+                {
+                    // "processorsNN_start-end"
+                    // - save the local proc offset
+
+                    if (Foam::exists(path/dirN/local))
+                    {
+                        found = true;
+                        break;
+                    }
+                }
+            }
+        }
+    }
+
+    if (!found)
+    {
+        found = Foam::exists(fName);
+    }
+
+    return found;
+}
+
+
 Foam::boolList Foam::haveMeshFile
 (
     const Time& runTime,
@@ -41,18 +136,83 @@ Foam::boolList Foam::haveMeshFile
     const bool verbose
 )
 {
+    #if 0
+
+    // Simple directory scanning - too fragile
+    bool found = checkFileExistence(runTime.path()/meshPath/meshFile);
+
+    #else
+
+    // Trimmed-down version of lookupAndCacheProcessorsPath
+    // with Foam::exists() check. No caching.
+
+    // Check for two conditions:
+    // - file has to exist
+    // - if collated the entry has to exist inside the file
+
+    // Note: bypass fileOperation::filePath(IOobject&) since has problems
+    //       with going to a different number of processors
+    //       (in collated format). Use file-based searching instead
+
+    const auto& handler = Foam::fileHandler();
+    typedef fileOperation::procRangeType procRangeType;
+
+    const fileName fName
+    (
+        handler.filePath(runTime.path()/meshPath/meshFile)
+    );
+
+    bool found = handler.isFile(fName);
+    if (returnReduceAnd(found)) // worldComm
+    {
+        autoPtr<ISstream> isPtr(fileHandler().NewIFstream(fName));
+        if (isPtr && isPtr->good())
+        {
+            auto& is = *isPtr;
+
+            IOobject io(meshFile, meshPath, runTime);
+            io.readHeader(is);
+
+            if (decomposedBlockData::isCollatedType(io))
+            {
+                fileName path, pDir, local;
+                procRangeType group;
+                label numProcs;
+                label proci = fileOperation::splitProcessorPath
+                (
+                    fName,
+                    path,
+                    pDir,
+                    local,
+                    group,
+                    numProcs
+                );
+
+                label myBlockNumber = 0;
+                if (proci == -1 && group.empty())
+                {
+                    // 'processorsXXX' format so contains all ranks
+                    // according to worldComm
+                    myBlockNumber = UPstream::myProcNo(UPstream::worldComm);
+                }
+                else
+                {
+                    // 'processorsXXX_n-m' format so check for the
+                    // relative rank
+                    myBlockNumber = UPstream::myProcNo(fileHandler().comm());
+                }
+
+                // Check if block for the local rank is inside file
+                found = decomposedBlockData::hasBlock(is, myBlockNumber);
+            }
+        }
+    }
+    #endif
+
+    // Globally consistent information about who has a mesh
     boolList haveFileOnProc
     (
-        UPstream::listGatherValues<bool>
-        (
-            fileHandler().isFile
-            (
-                fileHandler().filePath
-                (
-                    runTime.path()/meshPath/meshFile
-                )
-            )
-        )
+        UPstream::allGatherValues<bool>(found, UPstream::worldComm)
     );
 
     if (verbose)
@@ -62,7 +222,6 @@ Foam::boolList Foam::haveMeshFile
             << "    " << flatOutput(haveFileOnProc) << nl << endl;
     }
 
-    Pstream::broadcast(haveFileOnProc);
     return haveFileOnProc;
 }
 
@@ -107,38 +266,52 @@ void Foam::removeProcAddressing(const polyMesh& mesh)
 }
 
 
-void Foam::removeEmptyDir(const fileName& path)
+void Foam::masterMeshInstance
+(
+    const IOobject& io,
+    fileName& facesInstance,
+    fileName& pointsInstance
+)
 {
-    // Remove directory: silent, emptyOnly
-    Foam::rmDir(path, true, true);
-}
+    const fileName meshSubDir
+    (
+        polyMesh::regionName(io.name()) / polyMesh::meshSubDir
+    );
 
-
-void Foam::removeEmptyDirs(const fileName& meshPath)
-{
-    // Delete resulting directory if empty
-    fileName path(meshPath);
-    path.clean();
-
-    // Do subdirectories
+    if (UPstream::master())
     {
-        const fileNameList dirs
+        const bool oldParRun = UPstream::parRun(false);
+        const label oldNumProcs = fileHandler().nProcs();
+        const int oldCache = fileOperation::cacheLevel(0);
+
+        facesInstance = io.time().findInstance
         (
-            Foam::readDir
-            (
-                path,
-                fileName::DIRECTORY,
-                false,                  // filterGz
-                false                   // followLink
-            )
+            meshSubDir,
+            "faces",
+            IOobjectOption::MUST_READ
         );
-        for (const auto& dir : dirs)
+        pointsInstance = io.time().findInstance
+        (
+            meshSubDir,
+            "points",
+            IOobjectOption::MUST_READ
+        );
+
+        fileOperation::cacheLevel(oldCache);
+        if (oldParRun)
         {
-            removeEmptyDirs(path/dir);
+            const_cast<fileOperation&>(fileHandler()).nProcs(oldNumProcs);
         }
+        UPstream::parRun(oldParRun);
     }
 
-    removeEmptyDir(path);
+    // Broadcast information to all
+    Pstream::broadcasts
+    (
+        UPstream::worldComm,
+        facesInstance,
+        pointsInstance
+    );
 }
 
 

applications/utilities/parallelProcessing/redistributePar/loadOrCreateMesh.H

@@ -6,7 +6,7 @@
      \\/     M anipulation  |
 -------------------------------------------------------------------------------
     Copyright (C) 2012 OpenFOAM Foundation
-    Copyright (C) 2022 OpenCFD Ltd.
+    Copyright (C) 2022-2023 OpenCFD Ltd.
 -------------------------------------------------------------------------------
 License
     This file is part of OpenFOAM.
@@ -48,6 +48,9 @@ namespace Foam
 // Forward Declarations
 class faMesh;
 
+//- Check for availability of given file
+bool checkFileExistence(const fileName& fName);
+
 //- Check for availability of specified mesh file (default: "faces")
 boolList haveMeshFile
 (
@@ -64,11 +67,13 @@ void removeProcAddressing(const faMesh& mesh);
 //- Remove procAddressing
 void removeProcAddressing(const polyMesh& mesh);
 
-//- Remove empty directory
-void removeEmptyDir(const fileName& path);
-
-//- Remove empty directories from bottom up
-void removeEmptyDirs(const fileName& path);
+//- Determine master faces instance
+void masterMeshInstance
+(
+    const IOobject& io,
+    fileName& facesInstance,
+    fileName& pointsInstance
+);
 
 }
 

applications/utilities/parallelProcessing/redistributePar/parFaFieldDistributorCache.C

@@ -64,6 +64,7 @@ void Foam::parFaFieldDistributorCache::read
     const bool decompose,  // i.e. read from undecomposed case
 
     const boolList& areaMeshOnProc,
+    refPtr<fileOperation>& readHandler,
     const fileName& areaMeshInstance,
     faMesh& mesh
 )
@@ -76,10 +77,16 @@ void Foam::parFaFieldDistributorCache::read
     // Missing an area mesh somewhere?
     if (areaMeshOnProc.found(false))
     {
+        const bool oldParRun = UPstream::parRun(false);
+        const int oldCache = fileOperation::cacheLevel(0);
+
         // A zero-sized mesh with boundaries.
         // This is used to create zero-sized fields.
         subsetterPtr.reset(new faMeshSubset(mesh, zero{}));
 
+        fileOperation::cacheLevel(oldCache);
+        UPstream::parRun(oldParRun);  // Restore parallel state
+
         // Deregister from polyMesh ...
         auto& obr = const_cast<objectRegistry&>
         (
@@ -92,14 +99,26 @@ void Foam::parFaFieldDistributorCache::read
         obr.checkOut("faSolution");
     }
 
-
     // Get original objects (before incrementing time!)
-    if (Pstream::master() && decompose)
+    if (UPstream::master() && decompose)
     {
         runTime.caseName() = baseRunTime.caseName();
         runTime.processorCase(false);
     }
-    IOobjectList objects(mesh.mesh(), runTime.timeName());
+
+    IOobjectList objects;   //(mesh.mesh(), runTime.timeName());
+
+    if (readHandler)
+    {
+        auto oldHandler = fileOperation::fileHandler(readHandler);
+        const auto oldComm = UPstream::commWorld(fileHandler().comm());
+
+        objects = IOobjectList(mesh.mesh(), runTime.timeName());
+        readHandler = fileOperation::fileHandler(oldHandler);
+        UPstream::commWorld(oldComm);
+    }
+
+
     if (Pstream::master() && decompose)
     {
         runTime.caseName() = proc0CaseName;
@@ -116,11 +135,13 @@ void Foam::parFaFieldDistributorCache::read
         runTime.processorCase(false);
     }
 
+
     #undef  doFieldReading
     #define doFieldReading(Storage)                                   \
     fieldsDistributor::readFields                                     \
     (                                                                 \
-        areaMeshOnProc, mesh, subsetterPtr, objects, Storage,         \
+        areaMeshOnProc, readHandler, mesh, subsetterPtr, objects,     \
+        Storage,                                                      \
         true  /* (deregister field) */                                \
     );
 

applications/utilities/parallelProcessing/redistributePar/parFaFieldDistributorCache.H

@@ -105,6 +105,7 @@ public:
             const bool decompose,  // i.e. read from undecomposed case
 
             const boolList& areaMeshOnProc,
+            refPtr<fileOperation>& readHandler,
             const fileName& areaMeshInstance,
             faMesh& mesh
         );

applications/utilities/parallelProcessing/redistributePar/parLagrangianDistributor.C

@@ -52,7 +52,7 @@ Foam::parLagrangianDistributor::parLagrangianDistributor
     const mapDistribute& cellMap = distMap_.cellMap();
 
     // Get destination processors and cells
-    destinationProcID_ = labelList(tgtMesh_.nCells(), Pstream::myProcNo());
+    destinationProcID_ = labelList(tgtMesh_.nCells(), UPstream::myProcNo());
     cellMap.reverseDistribute(nSrcCells, destinationProcID_);
 
     destinationCell_ = identity(tgtMesh_.nCells());
@@ -67,22 +67,40 @@ Foam::parLagrangianDistributor::parLagrangianDistributor
 void Foam::parLagrangianDistributor::findClouds
 (
     const fvMesh& mesh,
-    wordList& cloudNames,
-    List<wordList>& objectNames
+    wordList& cloudNames,       // All cloud names on any processor
+    boolList& haveClouds,       // Per cloud name, whether my processor has it
+    List<wordList>& objectNames // Per cloud name, the field name
 )
 {
-    fileNameList localCloudDirs
+    const IOobject io
     (
-        readDir
+        cloud::prefix,
+        mesh.time().timeName(),
+        mesh,
+        IOobject::MUST_READ,
+        IOobject::NO_WRITE
+    );
+
+    // Using the fileHandler:
+    // - use fileHandler to synthesise correct processor directory
+    // - cannot use readObjects since assumes all processors have same
+    //   files (i.e. it only checks processor0)
+    const fileNameList localCloudDirs
+    (
+        fileHandler().readDir
         (
-            mesh.time().timePath()
-          / mesh.dbDir()
-          / cloud::prefix,
+            fileHandler().objectPath
+            (
+                io,
+                word::null  // typeName: not used currently
+            ),
             fileName::DIRECTORY
         )
     );
 
-    cloudNames.setSize(localCloudDirs.size());
+
+    // Copy out the local cloud names (and fileName -> word)
+    cloudNames.resize_nocopy(localCloudDirs.size());
     forAll(localCloudDirs, i)
     {
         cloudNames[i] = localCloudDirs[i];
@@ -92,18 +110,34 @@ void Foam::parLagrangianDistributor::findClouds
     Pstream::combineReduce(cloudNames, ListOps::uniqueEqOp<word>());
     Foam::sort(cloudNames);  // Consistent order
 
-    objectNames.clear();
+
+    // See which of the global cloudNames I have
+    haveClouds.resize_nocopy(cloudNames.size());
+    haveClouds = false;
+
+    for (const fileName& localCloudName : localCloudDirs)
+    {
+        const label cloudi = cloudNames.find(localCloudName);
+        if (cloudi >= 0)
+        {
+            haveClouds[cloudi] = true;
+        }
+    }
+
+    // Collect fields per cloud
     objectNames.resize(cloudNames.size());
 
     for (const fileName& localCloudName : localCloudDirs)
     {
         // Do local scan for valid cloud objects
+        const bool oldParRun = UPstream::parRun(false);
         IOobjectList localObjs
         (
             mesh,
             mesh.time().timeName(),
             cloud::prefix/localCloudName
         );
+        UPstream::parRun(oldParRun);
 
         bool isCloud = false;
         if (localObjs.erase("coordinates"))
@@ -141,8 +175,7 @@ Foam::parLagrangianDistributor::distributeLagrangianPositions
 ) const
 {
     //Debug(lpi.size());
-
-    const label oldLpi = lpi.size();
+    //const label oldLpi = lpi.size();
 
     labelListList sendMap;
 
@@ -154,7 +187,7 @@ Foam::parLagrangianDistributor::distributeLagrangianPositions
         // neighbour processors
         List<IDLList<passivePositionParticle>> particleTransferLists
         (
-            Pstream::nProcs()
+            UPstream::nProcs()
         );
 
         // Per particle the destination processor
@@ -233,50 +266,50 @@ Foam::parLagrangianDistributor::distributeLagrangianPositions
             }
         }
 
-        if (lagrangianPositions.size())
+        const bool writeOnProc = lagrangianPositions.size();
+
+        //if (writeOnProc)
         {
             // Write coordinates file
             IOPosition<passivePositionParticleCloud>
             (
                 lagrangianPositions
-            ).write();
+            ).write(writeOnProc);
 
-            // Optionally write positions file in v1706 format and earlier
+            // Optionally Write positions file in v1706 format and earlier
             if (particle::writeLagrangianPositions)
             {
                 IOPosition<passivePositionParticleCloud>
                 (
                      lagrangianPositions,
                      cloud::geometryType::POSITIONS
-                 ).write();
+                 ).write(writeOnProc);
             }
         }
-        else if (oldLpi)
-        {
-            // When running with -overwrite it should also delete the old
-            // files. Below works but is not optimal.
-
-            // Remove any existing coordinates
-            const fileName oldCoords
-            (
-                IOPosition<passivePositionParticleCloud>
-                (
-                    lagrangianPositions
-                ).objectPath()
-            );
-            Foam::rm(oldCoords);
-
-            // Remove any existing positions
-            const fileName oldPos
-            (
-                IOPosition<passivePositionParticleCloud>
-                (
-                    lagrangianPositions,
-                    cloud::geometryType::POSITIONS
-                ).objectPath()
-            );
-            Foam::rm(oldPos);
-        }
+        //else if (!writeOnProc && oldLpi)
+        //{
+        //    // When running with -overwrite it should also delete the old
+        //    // files. Below works but is not optimal.
+        //
+        //    // Remove any existing coordinates
+        //    Foam::rm
+        //    (
+        //        IOPosition<passivePositionParticleCloud>
+        //        (
+        //            lagrangianPositions
+        //        ).objectPath()
+        //    );
+        //
+        //    // Remove any existing positions
+        //    Foam::rm
+        //    (
+        //        IOPosition<passivePositionParticleCloud>
+        //        (
+        //            lagrangianPositions,
+        //            cloud::geometryType::POSITIONS
+        //        ).objectPath()
+        //    );
+        //}
 
         // Restore cloud name
         lpi.rename(cloudName);

applications/utilities/parallelProcessing/redistributePar/parLagrangianDistributor.H

@@ -6,7 +6,7 @@
      \\/     M anipulation  |
 -------------------------------------------------------------------------------
     Copyright (C) 2015 OpenFOAM Foundation
-    Copyright (C) 2018-2022 OpenCFD Ltd.
+    Copyright (C) 2018-2023 OpenCFD Ltd.
 -------------------------------------------------------------------------------
 License
     This file is part of OpenFOAM.
@@ -114,7 +114,14 @@ public:
         static void findClouds
         (
             const fvMesh&,
+
+            //! All cloud names on any processor
             wordList& cloudNames,
+
+            //! Per cloud name, whether my processor has it
+            boolList& haveClouds,
+
+            //! Per cloud nmae, the field names
             List<wordList>& objectNames
         );
 
@@ -131,6 +138,7 @@ public:
         static label readFields
         (
             const passivePositionParticleCloud& cloud,
+            const bool haveCloud,
             const IOobjectList& objects,
             const wordRes& selectedFields = wordRes()
         );
@@ -139,18 +147,11 @@ public:
         static label readAllFields
         (
             const passivePositionParticleCloud& cloud,
+            const bool haveCloud,
             const IOobjectList& objects,
             const wordRes& selectedFields = wordRes()
         );
 
-        //- Read and store all fields for known cloud field types
-        //  Uses the current cloud instance to obtain the IOobjectList
-        static label readAllFields
-        (
-            const passivePositionParticleCloud& cloud,
-            const wordRes& selectedFields = wordRes()
-        );
-
 
     // Member Functions
 
@@ -171,6 +172,7 @@ public:
         (
             const mapDistributeBase& lagrangianMap,
             const word& cloudName,
+            const bool haveCloud,
             const IOobjectList& cloudObjs,
             const wordRes& selectedFields
         ) const;
@@ -189,6 +191,7 @@ public:
         (
             const mapDistributeBase& map,
             const word& cloudName,
+            const bool haveCloud,
             const IOobjectList& objects,
             const wordRes& selectedFields = wordRes()
         ) const;
@@ -199,11 +202,14 @@ public:
         (
             const mapDistributeBase& map,
             const word& cloudName,
+            const bool haveCloud,
             const IOobjectList& objects,
             const wordRes& selectedFields = wordRes()
         ) const;
 
-        //- Redistribute and write stored lagrangian fields
+        //- Redistribute and write stored lagrangian fields.
+        //  Note: does no reading so no need to check for existence
+        //  of lagrangian files
         template<class Container>
         label distributeStoredFields
         (

applications/utilities/parallelProcessing/redistributePar/parLagrangianDistributorFields.C

@@ -5,7 +5,7 @@
     \\  /    A nd           | www.openfoam.com
      \\/     M anipulation  |
 -------------------------------------------------------------------------------
-    Copyright (C) 2022 OpenCFD Ltd.
+    Copyright (C) 2022-2023 OpenCFD Ltd.
 -------------------------------------------------------------------------------
 License
     This file is part of OpenFOAM.
@@ -33,6 +33,7 @@ License
 Foam::label Foam::parLagrangianDistributor::readAllFields
 (
     const passivePositionParticleCloud& cloud,
+    const bool haveCloud,
     const IOobjectList& cloudObjs,
     const wordRes& selectedFields
 )
@@ -48,6 +49,7 @@ Foam::label Foam::parLagrangianDistributor::readAllFields
             <IOField<Type>>                                                   \
             (                                                                 \
                 cloud,                                                        \
+                haveCloud,                                                    \
                 cloudObjs,                                                    \
                 selectedFields                                                \
             );                                                                \
@@ -56,6 +58,7 @@ Foam::label Foam::parLagrangianDistributor::readAllFields
             <IOField<Field<Type>>>                                            \
             (                                                                 \
                 cloud,                                                        \
+                haveCloud,                                                    \
                 cloudObjs,                                                    \
                 selectedFields                                                \
             );                                                                \
@@ -64,6 +67,7 @@ Foam::label Foam::parLagrangianDistributor::readAllFields
             <CompactIOField<Field<Type>, Type>>                               \
             (                                                                 \
                 cloud,                                                        \
+                haveCloud,                                                    \
                 cloudObjs,                                                    \
                 selectedFields                                                \
             );                                                                \
@@ -84,21 +88,22 @@ Foam::label Foam::parLagrangianDistributor::readAllFields
 }
 
 
-Foam::label Foam::parLagrangianDistributor::readAllFields
-(
-    const passivePositionParticleCloud& cloud,
-    const wordRes& selectedFields
-)
-{
-    IOobjectList cloudObjs(cloud, cloud.time().timeName());
-    return readAllFields(cloud, cloudObjs, selectedFields);
-}
+//Foam::label Foam::parLagrangianDistributor::readAllFields
+//(
+//    const passivePositionParticleCloud& cloud,
+//    const wordRes& selectedFields
+//)
+//{
+//    IOobjectList cloudObjs(cloud, cloud.time().timeName());
+//    return readAllFields(cloud, cloudObjs, selectedFields);
+//}
 
 
 Foam::label Foam::parLagrangianDistributor::distributeAllFields
 (
     const mapDistributeBase& lagrangianMap,
     const word& cloudName,
+    const bool haveCloud,
     const IOobjectList& cloudObjs,
     const wordRes& selectedFields
 ) const
@@ -114,6 +119,7 @@ Foam::label Foam::parLagrangianDistributor::distributeAllFields
             (                                                                 \
                 lagrangianMap,                                                \
                 cloudName,                                                    \
+                haveCloud,                                                    \
                 cloudObjs,                                                    \
                 selectedFields                                                \
             );                                                                \
@@ -122,6 +128,7 @@ Foam::label Foam::parLagrangianDistributor::distributeAllFields
             (                                                                 \
                 lagrangianMap,                                                \
                 cloudName,                                                    \
+                haveCloud,                                                    \
                 cloudObjs,                                                    \
                 selectedFields                                                \
             );                                                                \

applications/utilities/parallelProcessing/redistributePar/parLagrangianDistributorTemplates.C

@@ -6,7 +6,7 @@
      \\/     M anipulation  |
 -------------------------------------------------------------------------------
     Copyright (C) 2015 OpenFOAM Foundation
-    Copyright (C) 2018-2022 OpenCFD Ltd.
+    Copyright (C) 2018-2023 OpenCFD Ltd.
 -------------------------------------------------------------------------------
 License
     This file is part of OpenFOAM.
@@ -64,90 +64,86 @@ Foam::label Foam::parLagrangianDistributor::distributeFields
 (
     const mapDistributeBase& map,
     const word& cloudName,
+    const bool haveCloud,
     const IOobjectList& objects,
     const wordRes& selectedFields
 ) const
 {
-    typedef IOField<Type> fieldType;
+    typedef IOField<Type> Container;
 
     const wordList fieldNames
     (
-        filterObjects<fieldType>
+        filterObjects<IOField<Type>>
         (
             objects,
             selectedFields
         )
     );
 
+    // Read if present
+    IOobject srcIOobject
+    (
+        "none",
+        srcMesh_.time().timeName(),
+        cloud::prefix/cloudName,
+        srcMesh_,
+        IOobject::LAZY_READ,
+        IOobject::NO_WRITE,
+        IOobject::NO_REGISTER
+    );
 
-    bool reconstruct = false;
+    IOobject tgtIOobject
+    (
+        "none",
+        tgtMesh_.time().timeName(),
+        cloud::prefix/cloudName,
+        tgtMesh_,
+        IOobject::NO_READ,
+        IOobject::NO_WRITE,
+        IOobject::NO_REGISTER
+    );
+
+    //bool reconstruct = false;
 
     label nFields = 0;
     for (const word& objectName : fieldNames)
     {
-        if (!nFields)
-        {
-            // Performing an all-to-one (reconstruct)?
-            reconstruct =
-                returnReduceAnd(!map.constructSize() || Pstream::master());
-        }
-
         if (verbose_)
         {
             if (!nFields)
             {
                 Info<< "    Distributing lagrangian "
-                    << fieldType::typeName << "s\n" << nl;
+                    << Container::typeName << "s\n" << nl;
             }
             Info<< "        " <<  objectName << nl;
         }
         ++nFields;
 
-        // Read if present
-        IOField<Type> field
-        (
-            IOobject
-            (
-                objectName,
-                srcMesh_.time().timeName(),
-                cloud::prefix/cloudName,
-                srcMesh_,
-                IOobject::READ_IF_PRESENT,
-                IOobject::NO_WRITE,
-                IOobject::NO_REGISTER
-            ),
-            label(0)
-        );
+        srcIOobject.resetHeader(objectName);
+        tgtIOobject.resetHeader(objectName);
 
+        // Read if present (ie, haveCloud means readOnProc)
+        Container field(srcIOobject, haveCloud);
+
+        // Distribute
         map.distribute(field);
 
+        const bool writeOnProc = field.size();
 
-        const IOobject fieldIO
-        (
-            objectName,
-            tgtMesh_.time().timeName(),
-            cloud::prefix/cloudName,
-            tgtMesh_,
-            IOobject::NO_READ,
-            IOobject::NO_WRITE,
-            IOobject::NO_REGISTER
-        );
+        // Write
+        Container(tgtIOobject, std::move(field)).write(writeOnProc);
 
-        if (field.size())
-        {
-            IOField<Type>(fieldIO, std::move(field)).write();
-        }
-        else if (!reconstruct)
-        {
-            // When running with -overwrite it should also delete the old
-            // files. Below works but is not optimal.
-
-            const fileName fldName(fieldIO.objectPath());
-            Foam::rm(fldName);
-        }
+        //if (!writeOnProc && !reconstruct)
+        //{
+        //    // When running with -overwrite it should also delete the old
+        //    // files. Below works but is not optimal.
+        //
+        //    Foam::rm(tgtIOobject.objectPath());
+        //}
     }
 
     if (nFields && verbose_) Info<< endl;
+
     return nFields;
 }
 
@@ -157,25 +153,27 @@ Foam::label Foam::parLagrangianDistributor::distributeFieldFields
 (
     const mapDistributeBase& map,
     const word& cloudName,
+    const bool haveCloud,
     const IOobjectList& objects,
     const wordRes& selectedFields
 ) const
 {
-    typedef CompactIOField<Field<Type>, Type> fieldType;
+    typedef CompactIOField<Field<Type>, Type> Container;
 
     DynamicList<word> fieldNames;
 
-    fieldNames.append
+    // CompactIOField Field names
+    fieldNames.push_back
     (
-        filterObjects<fieldType>
+        filterObjects<CompactIOField<Field<Type>, Type>>
         (
             objects,
             selectedFields
         )
     );
 
-    // Append IOField Field names
-    fieldNames.append
+    // IOField Field names
+    fieldNames.push_back
     (
         filterObjects<IOField<Field<Type>>>
         (
@@ -184,76 +182,66 @@ Foam::label Foam::parLagrangianDistributor::distributeFieldFields
         )
     );
 
-    bool reconstruct = false;
+    // Read if present
+    IOobject srcIOobject
+    (
+        "none",
+        srcMesh_.time().timeName(),
+        cloud::prefix/cloudName,
+        srcMesh_,
+        IOobject::LAZY_READ,
+        IOobject::NO_WRITE,
+        IOobject::NO_REGISTER
+    );
+
+    IOobject tgtIOobject
+    (
+        "none",
+        tgtMesh_.time().timeName(),
+        cloud::prefix/cloudName,
+        tgtMesh_,
+        IOobject::NO_READ,
+        IOobject::NO_WRITE,
+        IOobject::NO_REGISTER
+    );
+
+    //bool reconstruct = false;
 
     label nFields = 0;
     for (const word& objectName : fieldNames)
     {
-        if (!nFields)
-        {
-            // Performing an all-to-one (reconstruct)?
-            reconstruct =
-                returnReduceAnd(!map.constructSize() || Pstream::master());
-        }
-
         if (verbose_)
         {
             if (!nFields)
             {
                 Info<< "    Distributing lagrangian "
-                    << fieldType::typeName << "s\n" << nl;
+                    << Container::typeName << "s\n" << nl;
             }
             Info<< "        " <<  objectName << nl;
         }
         ++nFields;
 
-        // Read if present
-        CompactIOField<Field<Type>, Type> field
-        (
-            IOobject
-            (
-                objectName,
-                srcMesh_.time().timeName(),
-                cloud::prefix/cloudName,
-                srcMesh_,
-                IOobject::READ_IF_PRESENT,
-                IOobject::NO_WRITE,
-                IOobject::NO_REGISTER
-            ),
-            label(0)
-        );
+        srcIOobject.resetHeader(objectName);
+        tgtIOobject.resetHeader(objectName);
+
+        // Read if present (ie, haveCloud means readOnProc)
+        Container field(srcIOobject, haveCloud);
 
         // Distribute
         map.distribute(field);
 
+        const bool writeOnProc = field.size();
+
         // Write
-        const IOobject fieldIO
-        (
-            objectName,
-            tgtMesh_.time().timeName(),
-            cloud::prefix/cloudName,
-            tgtMesh_,
-            IOobject::NO_READ,
-            IOobject::NO_WRITE,
-            IOobject::NO_REGISTER
-        );
+        Container(tgtIOobject, std::move(field)).write(writeOnProc);
 
-        if (field.size())
-        {
-            CompactIOField<Field<Type>, Type>
-            (
-                fieldIO,
-                std::move(field)
-            ).write();
-        }
-        else if (!reconstruct)
-        {
-            // When running with -overwrite it should also delete the old
-            // files. Below works but is not optimal.
-
-            const fileName fldName(fieldIO.objectPath());
-            Foam::rm(fldName);
-        }
+        //if (!writeOnProc && !reconstruct)
+        //{
+        //    // When running with -overwrite it should also delete the old
+        //    // files. Below works but is not optimal.
+        //
+        //    Foam::rm(tgtIOobject.objectPath());
+        //}
     }
 
     if (nFields && verbose_) Info<< endl;
@@ -265,12 +253,11 @@ template<class Container>
 Foam::label Foam::parLagrangianDistributor::readFields
 (
     const passivePositionParticleCloud& cloud,
+    const bool haveCloud,
     const IOobjectList& objects,
     const wordRes& selectedFields
 )
 {
-    const word fieldClassName(Container::typeName);
-
     const wordList fieldNames
     (
         filterObjects<Container>
@@ -280,6 +267,17 @@ Foam::label Foam::parLagrangianDistributor::readFields
         )
     );
 
+    // Read if present
+    IOobject readIO
+    (
+        "none",
+        cloud.time().timeName(),
+        cloud,
+        IOobject::LAZY_READ,
+        IOobject::NO_WRITE,
+        IOobject::REGISTER
+    );
+
     label nFields = 0;
     for (const word& objectName : fieldNames)
     {
@@ -294,20 +292,10 @@ Foam::label Foam::parLagrangianDistributor::readFields
         }
         ++nFields;
 
-        // Read if present
-        Container* fieldPtr = new Container
-        (
-            IOobject
-            (
-                objectName,
-                cloud.time().timeName(),
-                cloud,
-                IOobject::LAZY_READ,
-                IOobject::NO_WRITE,
-                IOobject::REGISTER
-            ),
-            label(0)
-        );
+        readIO.resetHeader(objectName);
+
+        // Read if present (ie, haveCloud means readOnProc)
+        Container* fieldPtr = new Container(readIO, haveCloud);
 
         fieldPtr->store();
     }
@@ -329,19 +317,27 @@ Foam::label Foam::parLagrangianDistributor::distributeStoredFields
         cloud.lookupClass<Container>()
     );
 
-    bool reconstruct = false;
+    // Parallel-consistent names
+    const wordList fieldNames(fields.sortedToc());
+
+    IOobject writeIO
+    (
+        "none",
+        tgtMesh_.time().timeName(),
+        cloud::prefix/cloud.name(),
+        tgtMesh_,
+        IOobject::NO_READ,
+        IOobject::NO_WRITE,
+        IOobject::NO_REGISTER
+    );
+
+    //bool reconstruct = false;
 
     label nFields = 0;
-    forAllIters(fields, iter)
-    {
-        Container& field = *(iter.val());
 
-        if (!nFields)
-        {
-            // Performing an all-to-one (reconstruct)?
-            reconstruct =
-                returnReduceAnd(!map.constructSize() || Pstream::master());
-        }
+    for (const word& fieldName : fieldNames)
+    {
+        Container& field = *(fields[fieldName]);
 
         if (verbose_)
         {
@@ -356,29 +352,20 @@ Foam::label Foam::parLagrangianDistributor::distributeStoredFields
 
         map.distribute(field);
 
-        const IOobject fieldIO
-        (
-            field.name(),
-            tgtMesh_.time().timeName(),
-            cloud::prefix/cloud.name(),
-            tgtMesh_,
-            IOobject::NO_READ,
-            IOobject::NO_WRITE,
-            IOobject::NO_REGISTER
-        );
+        writeIO.resetHeader(field.name());
 
-        if (field.size())
-        {
-            Container(fieldIO, std::move(field)).write();
-        }
-        else if (!reconstruct)
-        {
-            // When running with -overwrite it should also delete the old
-            // files. Below works but is not optimal.
+        const bool writeOnProc = field.size();
 
-            const fileName fldName(fieldIO.objectPath());
-            Foam::rm(fldName);
-        }
+        // Write
+        Container(writeIO, std::move(field)).write(writeOnProc);
+
+        //if (!writeOnProc && !reconstruct)
+        //{
+        //    // When running with -overwrite it should also delete the old
+        //    // files. Below works but is not optimal.
+        //
+        //    Foam::rm(writeIO.objectPath());
+        //}
     }
 
     if (nFields && verbose_) Info<< endl;

applications/utilities/parallelProcessing/redistributePar/redistributeLagrangian.H

@@ -6,7 +6,7 @@
      \\/     M anipulation  |
 -------------------------------------------------------------------------------
     Copyright (C) 2011-2017 OpenFOAM Foundation
-    Copyright (C) 2015-2022 OpenCFD Ltd.
+    Copyright (C) 2015-2023 OpenCFD Ltd.
 -------------------------------------------------------------------------------
 License
     This file is part of OpenFOAM.
@@ -48,6 +48,7 @@ readLagrangian
 (
     const fvMesh& mesh,
     const wordList& cloudNames,
+    const boolUList& haveClouds,
     const wordRes& selectedFields
 )
 {
@@ -78,11 +79,10 @@ readLagrangian
 
         IOobjectList cloudObjs(clouds[i], clouds[i].time().timeName());
 
-        //Pout<< "Found cloud objects:" << cloudObjs.names() << endl;
-
         parLagrangianDistributor::readAllFields
         (
             clouds[i],
+            haveClouds[i],
             cloudObjs,
             selectedFields
         );
@@ -101,11 +101,19 @@ readLagrangian
 )
 {
     wordList cloudNames;
+    boolList haveClouds;
     List<wordList> fieldNames;
-    // Find all cloudNames on all processors
-    parLagrangianDistributor::findClouds(mesh, cloudNames, fieldNames);
 
-    return readLagrangian(mesh, cloudNames, selectedFields);
+    // Find all cloudNames on all processors
+    parLagrangianDistributor::findClouds
+    (
+        mesh,
+        cloudNames,
+        haveClouds,
+        fieldNames
+    );
+
+    return readLagrangian(mesh, cloudNames, haveClouds, selectedFields);
 }
 
 
@@ -121,9 +129,17 @@ void reconstructLagrangian
     // Clouds (note: might not be present on all processors)
 
     wordList cloudNames;
+    boolList haveClouds;
     List<wordList> fieldNames;
+
     // Find all cloudNames on all processors
-    parLagrangianDistributor::findClouds(mesh, cloudNames, fieldNames);
+    parLagrangianDistributor::findClouds
+    (
+        mesh,
+        cloudNames,
+        haveClouds,
+        fieldNames
+    );
 
     if (cloudNames.empty())
     {
@@ -142,13 +158,16 @@ void reconstructLagrangian
                 baseMesh,
                 mesh.nCells(),      // range of cell indices in clouds
                 distMap
+                //writeHandler        // Which processors to write
             )
         );
     }
     const auto& distributor = *distributorPtr;
 
-    for (const word& cloudName : cloudNames)
+    forAll(cloudNames, cloudi)
     {
+        const word& cloudName = cloudNames[cloudi];
+
         Info<< "Reconstructing lagrangian fields for cloud "
             << cloudName << nl << endl;
 
@@ -169,6 +188,7 @@ void reconstructLagrangian
         (
             lagrangianMapPtr(),
             cloudName,
+            haveClouds[cloudi],
             cloudObjs,
             selectedFields
         );

src/OpenFOAM/db/Time/Time.C

@@ -199,7 +199,8 @@ void Foam::Time::setControls()
     //       alternative processorsDDD directories naming
     //       - do not look for compressed (is directory)
 
-    if (fileHandler().filePath(timePath(), false).empty())
+    const fileName timeDir(fileHandler().filePath(timePath(), false));
+    if (returnReduceAnd(timeDir.empty()))
     {
         const int oldPrecision = precision_;
         int requiredPrecision = -1;
@@ -222,7 +223,8 @@ void Foam::Time::setControls()
             oldTime = std::move(newTime);
 
             // Does a time directory exist with the new format?
-            if (!fileHandler().filePath(timePath(), false).empty())
+            const fileName timeDir(fileHandler().filePath(timePath(), false));
+            if (returnReduceOr(!timeDir.empty()))
             {
                 requiredPrecision = precision_;
             }

src/OpenFOAM/db/dictionary/functionEntries/codeStream/codeStream.C

@@ -196,7 +196,17 @@ Foam::functionEntries::codeStream::getFunction
     //  (flag set by regIOobject::read, baseIOdictionary constructor)
     if (!masterOnly && returnReduceOr(lib == nullptr))
     {
-        // Broadcast distributed...
+        // Broadcast to distributed masters
+        if (fileHandler().distributed())
+        {
+            fileHandler().broadcastCopy
+            (
+                UPstream::worldComm,
+                UPstream::master(fileHandler().comm()),
+                libPath,
+                libPath
+            );
+        }
 
         dynamicCode::waitForFile(libPath, context.dict());
     }

src/OpenFOAM/db/dictionary/functionEntries/includeEntry/includeEntry.C

@@ -6,7 +6,7 @@
      \\/     M anipulation  |
 -------------------------------------------------------------------------------
     Copyright (C) 2011-2017 OpenFOAM Foundation
-    Copyright (C) 2018-2021 OpenCFD Ltd.
+    Copyright (C) 2018-2022 OpenCFD Ltd.
 -------------------------------------------------------------------------------
 License
     This file is part of OpenFOAM.
@@ -27,12 +27,13 @@ License
 \*---------------------------------------------------------------------------*/
 
 #include "includeEntry.H"
-#include "addToMemberFunctionSelectionTable.H"
 #include "stringOps.H"
 #include "IFstream.H"
 #include "IOstreams.H"
-#include "Time.H"
+#include "UPstream.H"
 #include "fileOperation.H"
+#include "regIOobject.H"
+#include "addToMemberFunctionSelectionTable.H"
 
 // * * * * * * * * * * * * * * Static Data Members * * * * * * * * * * * * * //
 
@@ -133,6 +134,15 @@ bool Foam::functionEntries::includeEntry::execute
     Istream& is
 )
 {
+    const auto* rioPtr = isA<regIOobject>(parentDict.topDict());
+
+    const label oldComm
+    (
+        rioPtr && rioPtr->global()
+      ? fileHandler().comm(UPstream::worldComm)
+      : fileHandler().comm()
+    );
+
     const fileName rawName(is);
     const fileName fName(resolveFile(is.name().path(), rawName, parentDict));
 
@@ -148,20 +158,19 @@ bool Foam::functionEntries::includeEntry::execute
         }
 
         // Add watch on included file
-        const dictionary& top = parentDict.topDict();
-        if (isA<regIOobject>(top))
+        if (rioPtr)
         {
-            regIOobject& rio = const_cast<regIOobject&>
-            (
-                dynamic_cast<const regIOobject&>(top)
-            );
-            rio.addWatch(fName);
+            const_cast<regIOobject&>(*rioPtr).addWatch(fName);
         }
 
         parentDict.read(ifs);
+
+        fileHandler().comm(oldComm);
         return true;
     }
 
+    fileHandler().comm(oldComm);
+
     if (!mandatory)
     {
         return true; // Never fails if optional
@@ -185,6 +194,15 @@ bool Foam::functionEntries::includeEntry::execute
     Istream& is
 )
 {
+    const auto* rioPtr = isA<regIOobject>(parentDict.topDict());
+
+    const label oldComm
+    (
+        rioPtr && rioPtr->global()
+      ? fileHandler().comm(UPstream::worldComm)
+      : fileHandler().comm()
+    );
+
     const fileName rawName(is);
     const fileName fName(resolveFile(is.name().path(), rawName, parentDict));
 
@@ -200,20 +218,19 @@ bool Foam::functionEntries::includeEntry::execute
         }
 
         // Add watch on included file
-        const dictionary& top = parentDict.topDict();
-        if (isA<regIOobject>(top))
+        if (rioPtr)
         {
-            regIOobject& rio = const_cast<regIOobject&>
-            (
-                dynamic_cast<const regIOobject&>(top)
-            );
-            rio.addWatch(fName);
+            const_cast<regIOobject&>(*rioPtr).addWatch(fName);
         }
 
         entry.read(parentDict, ifs);
+
+        fileHandler().comm(oldComm);
         return true;
     }
 
+    fileHandler().comm(oldComm);
+
     if (!mandatory)
     {
         return true; // Never fails if optional

src/OpenFOAM/db/dictionary/functionEntries/includeEtcEntry/includeEtcEntry.C

@@ -6,7 +6,7 @@
      \\/     M anipulation  |
 -------------------------------------------------------------------------------
     Copyright (C) 2015-2017 OpenFOAM Foundation
-    Copyright (C) 2019-2021 OpenCFD Ltd.
+    Copyright (C) 2019-2022 OpenCFD Ltd.
 -------------------------------------------------------------------------------
 License
     This file is part of OpenFOAM.
@@ -27,12 +27,14 @@ License
 \*---------------------------------------------------------------------------*/
 
 #include "includeEtcEntry.H"
-#include "addToMemberFunctionSelectionTable.H"
 #include "etcFiles.H"
 #include "stringOps.H"
 #include "IFstream.H"
 #include "IOstreams.H"
+#include "UPstream.H"
 #include "fileOperation.H"
+#include "regIOobject.H"
+#include "addToMemberFunctionSelectionTable.H"
 
 // * * * * * * * * * * * * * * Static Data Members * * * * * * * * * * * * * //
 
@@ -113,6 +115,15 @@ bool Foam::functionEntries::includeEtcEntry::execute
     Istream& is
 )
 {
+    const regIOobject* rioPtr = isA<regIOobject>(parentDict.topDict());
+
+    const label oldComm
+    (
+        rioPtr && rioPtr->global()
+      ? fileHandler().comm(UPstream::worldComm)
+      : fileHandler().comm()
+    );
+
     const fileName rawName(is);
     const fileName fName(resolveEtcFile(rawName, parentDict));
 
@@ -127,9 +138,13 @@ bool Foam::functionEntries::includeEtcEntry::execute
             Info<< fName << nl;
         }
         parentDict.read(ifs);
+
+        fileHandler().comm(oldComm);
         return true;
     }
 
+    fileHandler().comm(oldComm);
+
     if (!mandatory)
     {
         return true; // Never fails if optional
@@ -153,6 +168,15 @@ bool Foam::functionEntries::includeEtcEntry::execute
     Istream& is
 )
 {
+    const regIOobject* rioPtr = isA<regIOobject>(parentDict.topDict());
+
+    const label oldComm
+    (
+        rioPtr && rioPtr->global()
+      ? fileHandler().comm(UPstream::worldComm)
+      : fileHandler().comm()
+    );
+
     const fileName rawName(is);
     const fileName fName(resolveEtcFile(rawName, parentDict));
 
@@ -167,9 +191,13 @@ bool Foam::functionEntries::includeEtcEntry::execute
             Info<< fName << nl;
         }
         entry.read(parentDict, ifs);
+
+        fileHandler().comm(oldComm);
         return true;
     }
 
+    fileHandler().comm(oldComm);
+
     if (!mandatory)
     {
         return true; // Never fails if optional

src/OpenFOAM/db/dynamicLibrary/codedBase/codedBase.C

@@ -254,9 +254,21 @@ void Foam::codedBase::createLibrary
         UPstream::barrier(UPstream::worldComm);
     }
 
-    // Broadcast distributed...
+    const fileName libPath = dynCode.libPath();
 
-    dynamicCode::waitForFile(dynCode.libPath(), context.dict());
+    // Broadcast to distributed masters
+    if (fileHandler().distributed())
+    {
+        fileHandler().broadcastCopy
+        (
+            UPstream::worldComm,
+            UPstream::master(fileHandler().comm()),
+            libPath,
+            libPath
+        );
+    }
+
+    dynamicCode::waitForFile(libPath, context.dict());
 }
 
 

src/OpenFOAM/db/regIOobject/regIOobject.C

@@ -52,7 +52,6 @@ Foam::regIOobject::regIOobject(const IOobject& io, const bool isTimeObject)
     IOobject(io),
     registered_(false),
     ownedByRegistry_(false),
-    watchIndices_(),
     eventNo_(isTimeObject ? 0 : db().getEvent()), // No event for top-level Time
     metaDataPtr_(nullptr),
     isPtr_(nullptr)
@@ -70,8 +69,9 @@ Foam::regIOobject::regIOobject(const regIOobject& rio)
     IOobject(rio),
     registered_(false),
     ownedByRegistry_(false),
-    watchIndices_(rio.watchIndices_),
     eventNo_(db().getEvent()),
+    watchFiles_(rio.watchFiles_),
+    watchIndices_(rio.watchIndices_),
     metaDataPtr_(rio.metaDataPtr_.clone()),
     isPtr_(nullptr)
 {
@@ -84,7 +84,6 @@ Foam::regIOobject::regIOobject(const regIOobject& rio, bool registerCopy)
     IOobject(rio),
     registered_(false),
     ownedByRegistry_(false),
-    watchIndices_(),
     eventNo_(db().getEvent()),
     metaDataPtr_(rio.metaDataPtr_.clone()),
     isPtr_(nullptr)
@@ -111,7 +110,6 @@ Foam::regIOobject::regIOobject
     IOobject(newName, rio.instance(), rio.local(), rio.db()),
     registered_(false),
     ownedByRegistry_(false),
-    watchIndices_(),
     eventNo_(db().getEvent()),
     metaDataPtr_(rio.metaDataPtr_.clone()),
     isPtr_(nullptr)
@@ -135,7 +133,6 @@ Foam::regIOobject::regIOobject
     IOobject(io),
     registered_(false),
     ownedByRegistry_(false),
-    watchIndices_(),
     eventNo_(db().getEvent()),
     metaDataPtr_(rio.metaDataPtr_.clone()),
     isPtr_(nullptr)
@@ -234,6 +231,7 @@ bool Foam::regIOobject::checkOut()
         fileHandler().removeWatch(watchIndices_[i]);
     }
     watchIndices_.clear();
+    watchFiles_.clear();
 
     if (registered_)
     {
@@ -253,17 +251,23 @@ Foam::label Foam::regIOobject::addWatch(const fileName& f)
     if
     (
         registered_
-     && readOpt() == IOobject::MUST_READ_IF_MODIFIED
+     && readOpt() == IOobjectOption::READ_MODIFIED
      && time().runTimeModifiable()
     )
     {
-        index = fileHandler().findWatch(watchIndices_, f);
+        //- 1. Directly add to fileHandler
+        //index = fileHandler().findWatch(watchIndices_, f);
+        //
+        //if (index == -1)
+        //{
+        //    index = watchIndices_.size();
+        //    watchIndices_.push_back(fileHandler().addWatch(f));
+        //}
 
-        if (index == -1)
-        {
-            index = watchIndices_.size();
-            watchIndices_.append(fileHandler().addWatch(f));
-        }
+        //- 2. Delay adding; add to list and handle in addWatch() later on
+        //     Note: what do we return?
+        index = watchFiles_.size();
+        watchFiles_.push_back(f);
     }
 
     return index;
@@ -275,7 +279,7 @@ void Foam::regIOobject::addWatch()
     if
     (
         registered_
-     && readOpt() == IOobject::MUST_READ_IF_MODIFIED
+     && readOpt() == IOobjectOption::READ_MODIFIED
      && time().runTimeModifiable()
     )
     {
@@ -307,37 +311,80 @@ void Foam::regIOobject::addWatch()
             )
         );
 
-        if (masterOnly && Pstream::parRun())
+        if (masterOnly && UPstream::parRun())
         {
-            // Get master watched files
-            fileNameList watchFiles;
-            if (Pstream::master())
+            // Get all files watched on master, and broadcast at once
+            fileNameList filesToWatch;
+            if (UPstream::master())
             {
-                watchFiles.resize(watchIndices_.size());
+                const bool oldParRun = UPstream::parRun(false);
+
+                filesToWatch.resize(watchIndices_.size());
                 forAll(watchIndices_, i)
                 {
-                    watchFiles[i] = fileHandler().getFile(watchIndices_[i]);
+                    filesToWatch[i] = fileHandler().getFile(watchIndices_[i]);
                 }
-            }
-            Pstream::broadcast(watchFiles);
 
-            if (!Pstream::master())
+                UPstream::parRun(oldParRun);
+            }
+            Pstream::broadcast(filesToWatch);
+
+
+            // Add master files in same order
+            if (!UPstream::master())
             {
-                // unregister current ones
+                const bool oldParRun = UPstream::parRun(false);
+
+                // Unregister current watched indices
                 forAllReverse(watchIndices_, i)
                 {
                     fileHandler().removeWatch(watchIndices_[i]);
                 }
 
+                // Insert the ones from master, in master order
                 watchIndices_.clear();
-                forAll(watchFiles, i)
+                for (const auto& file : filesToWatch)
                 {
-                    watchIndices_.append(fileHandler().addWatch(watchFiles[i]));
+                    watchIndices_.push_back(fileHandler().addWatch(file));
                 }
-            }
-        }
 
-        watchIndices_.append(fileHandler().addWatch(f));
+                UPstream::parRun(oldParRun);
+            }
+
+
+            // Files that were explicitly added via addWatch(const fileName&)
+            // (e.g. through #include)
+            for (const auto& file : watchFiles_)
+            {
+                watchIndices_.push_back(fileHandler().addWatch(file));
+            }
+
+            // Append the local file
+            watchIndices_.push_back(fileHandler().addWatch(f));
+        }
+        else
+        {
+            DynamicList<fileName> filesToWatch
+            (
+                watchIndices_.size()+watchFiles_.size()+1
+            );
+
+            // Get existing watched files from fileHandler
+            for (const label index : watchIndices_)
+            {
+                filesToWatch.push_back(fileHandler().getFile(index));
+            }
+
+            // The files explicitly added from addWatch(const fileName&)
+            // (e.g. through #include)
+            filesToWatch.push_back(std::move(watchFiles_));
+
+            // The local file
+            filesToWatch.push_back(f);
+
+            // Re-do all watches
+            fileHandler().addWatches(*this, filesToWatch);
+        }
     }
 }
 

src/OpenFOAM/db/regIOobject/regIOobject.H

@@ -43,6 +43,7 @@ SourceFiles
 
 #include "IOobject.H"
 #include "OSspecific.H"
+#include "DynamicList.H"
 
 // * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
 
@@ -94,12 +95,15 @@ private:
         //- Is this object owned by the registry
         bool ownedByRegistry_;
 
-        //- List of modification watch indices
-        mutable labelList watchIndices_;
-
-        //- eventNo of last update
+        //- The eventNo of last update
         label eventNo_;
 
+        //- List of additional files to watch
+        mutable DynamicList<fileName> watchFiles_;
+
+        //- List of modification watch indices
+        mutable DynamicList<label> watchIndices_;
+
         //- Dictionary for any meta-data
         autoPtr<dictionary> metaDataPtr_;
 

src/OpenFOAM/db/regIOobject/regIOobjectRead.C

@@ -58,11 +58,11 @@ bool Foam::regIOobject::readHeaderOk
     {
         if (masterOnly)
         {
-            if (Pstream::master())
+            if (UPstream::master())
             {
-                const bool oldParRun = Pstream::parRun(false);
+                const bool oldParRun = UPstream::parRun(false);
                 isHeaderOk = headerOk();
-                Pstream::parRun(oldParRun);
+                UPstream::parRun(oldParRun);
             }
             Pstream::broadcast(isHeaderOk);
         }

src/OpenFOAM/db/regIOobject/regIOobjectWrite.C

@@ -105,7 +105,7 @@ bool Foam::regIOobject::writeObject
     );
 
     bool osGood = false;
-    if (!masterOnly || Pstream::master())
+    if (!masterOnly || UPstream::master())
     {
         osGood = fileHandler().writeObject(*this, streamOpt, writeOnProc);
     }

src/OpenFOAM/global/argList/argList.C

@@ -1526,6 +1526,9 @@ void Foam::argList::parse
                 // also have to protect the actual dictionary parsing since
                 // it might trigger file access (e.g. #include, #codeStream)
                 const bool oldParRun = Pstream::parRun(false);
+                // Note: non-parallel running might update
+                // fileOperation::nProcs() so store & restore below
+                const label nOldProcs = fileHandler().nProcs();
 
                 autoPtr<ISstream> dictStream
                 (
@@ -1584,6 +1587,7 @@ void Foam::argList::parse
                 }
 
                 Pstream::parRun(oldParRun);  // Restore parallel state
+                const_cast<fileOperation&>(fileHandler()).nProcs(nOldProcs);
 
                 if (Pstream::nProcs() == 1)
                 {
@@ -1763,16 +1767,148 @@ void Foam::argList::parse
     }
 
     // If needed, adjust fileHandler for distributed roots
-    if (runControl_.distributed())
+    if (runControl_.distributed() && fileOperation::fileHandlerPtr_)
     {
-        if (fileOperation::fileHandlerPtr_)
+        fileOperation::fileHandlerPtr_->distributed(true);
+
+        const labelList& ranks = fileHandler().ioRanks();
+
+        if (runControl_.parRun() && ranks.size())
         {
-            fileOperation::fileHandlerPtr_->distributed(true);
+            // Detect processor directories both on local proc and on
+            // (world) master proc. If the local proc doesn't have them
+            // but the master has it will attempt to copy them.
+
+            // Expected local directory name
+            const fileName procDir
+            (
+                rootPath_
+              / globalCase_
+              / ("processor" + Foam::name(UPstream::myProcNo()))
+            );
+
+            // Try and find my local directory using the fileHandler. This
+            // will check the local disk on the IO rank
+            // (since running distributed)
+            fileNameList pathDirs(UPstream::nProcs());
+            auto& pathDir = pathDirs[UPstream::myProcNo()];
+            pathDir = fileHandler().filePath(procDir, false);
+
+            if (returnReduceOr(pathDir.empty()))
+            {
+                // There is at least one processor that cannot find
+                // the processor directory. Look for it on the master.
+                // E.g. decomposed into 4 processors, two roots:
+                //      processors4_0-1/
+                //      processors4_2-3/
+                // So:
+                // - processor0 reads the same disk as processor0
+                // - processor2 needs the whole directory sent over
+                // - processor3 reads the same disk as processor2
+                if (UPstream::master() && bannerEnabled())
+                {
+                    Info<< "I/O    :"
+                        << " distributed - copying missing directories"
+                        << nl;
+                }
+
+                // Collect all wanted directories (or empty). Note: could
+                // just collect missing ones ...
+
+                Pstream::gatherList(pathDirs);
+                fileName masterRootPath(rootPath_);
+                Pstream::broadcast(masterRootPath);
+
+                List<fileNameList> rankToDirs(UPstream::nProcs());
+                if (UPstream::master())
+                {
+                    const bool oldParRun = Pstream::parRun(false);
+                    // Note: non-parallel running might update
+                    // fileOperation::nProcs() so store & restore below
+                    const label nOldProcs = fileHandler().nProcs();
+
+                    label rank = 0;
+                    for (label proci = 1; proci < pathDirs.size(); ++proci)
+                    {
+                        if (ranks.contains(proci))
+                        {
+                            rank = proci;
+                        }
+
+                        if (pathDirs[proci].empty())
+                        {
+                            // Synthesise corresponding name on the master
+                            // processor
+                            const fileName procDir
+                            (
+                                rootPath_
+                              / globalCase_
+                              / ("processor" + Foam::name(proci))
+                            );
+                            const fileName foundDir
+                            (
+                                fileHandler().filePath(procDir, false)
+                            );
+
+                            if
+                            (
+                                !foundDir.empty()
+                             && !rankToDirs[rank].contains(foundDir)
+                            )
+                            {
+                                rankToDirs[rank].push_back(foundDir);
+                            }
+                        }
+                    }
+
+                    UPstream::parRun(oldParRun);
+                    const_cast<fileOperation&>(fileHandler()).nProcs(nOldProcs);
+                }
+                Pstream::broadcast(rankToDirs);
+
+                // Copy missing directories on all the IOranks.
+                // Note: instead of passing 'writeOnProc' flag we could create
+                // communicator just between master and IOrank, but that is
+                // also expensive.
+
+                forAll(rankToDirs, proci)
+                {
+                    // Am I the reponsible IOrank for this processor
+                    const bool amIO = (UPstream::myProcNo() == proci);
+
+                    // Construct equivalent directory on proci
+                    for (const auto& srcDir : rankToDirs[proci])
+                    {
+                        const fileName tgtDir
+                        (
+                            rootPath_
+                          / srcDir.relative(masterRootPath)
+                        );
+
+                        if (amIO)
+                        {
+                            // I am the IO rank
+                            Pout<< "On rank " << proci << nl
+                                << "    copying : " << srcDir << nl
+                                << "    to      : " << tgtDir << endl;
+                        }
+
+                        fileHandler().broadcastCopy
+                        (
+                            UPstream::worldComm,
+                            amIO,
+                            tgtDir,
+                            tgtDir
+                        );
+                    }
+                }
+            }
         }
     }
 
+
     // Keep/discard sub-process host/root information for reporting:
-    if (Pstream::master() && runControl_.parRun())
+    if (UPstream::master() && runControl_.parRun())
     {
         if (!writeHostsSwitch)
         {
@@ -1785,7 +1921,7 @@ void Foam::argList::parse
         }
     }
 
-    if (Pstream::master() && bannerEnabled())
+    if (UPstream::master() && bannerEnabled())
     {
         Info<< "Case   : " << (rootPath_/globalCase_).c_str() << nl
             << "nProcs : " << nProcs << nl;

src/OpenFOAM/global/fileOperations/collatedFileOperation/collatedFileOperation.C

@@ -113,43 +113,7 @@ void Foam::fileOperations::collatedFileOperation::printBanner
 
     if (withRanks)
     {
-        // Information about the ranks
-        stringList hosts(Pstream::nProcs());
-        if (Pstream::master(comm_))
-        {
-            hosts[Pstream::myProcNo()] = hostName();
-        }
-        Pstream::gatherList(hosts);
-
-        DynamicList<label> offsetMaster(Pstream::nProcs());
-
-        forAll(hosts, ranki)
-        {
-            if (!hosts[ranki].empty())
-            {
-                offsetMaster.append(ranki);
-            }
-        }
-
-        if (offsetMaster.size() > 1)
-        {
-            DetailInfo
-                << "IO nodes:" << nl << '(' << nl;
-
-            offsetMaster.append(Pstream::nProcs());
-
-            for (label group = 1; group < offsetMaster.size(); ++group)
-            {
-                const label beg = offsetMaster[group-1];
-                const label end = offsetMaster[group];
-
-                DetailInfo
-                    << "    (" << hosts[beg].c_str() << ' '
-                    << (end-beg) << ')' << nl;
-            }
-            DetailInfo
-                << ')' << nl;
-        }
+        fileOperation::printRanks();
     }
 
     //- fileModificationChecking already set by base class (masterUncollated)
@@ -386,8 +350,10 @@ bool Foam::fileOperations::collatedFileOperation::writeObject
 
     if (inst.isAbsolute() || !tm.processorCase())
     {
-        mkDir(io.path());
-        fileName pathName(io.objectPath());
+        // Note: delay mkdir to masterOFstream so it does not get created
+        //       if not needed (e.g. when running distributed)
+
+        const fileName pathName(io.objectPath());
 
         if (debug)
         {
@@ -427,10 +393,12 @@ bool Foam::fileOperations::collatedFileOperation::writeObject
     else
     {
         // Construct the equivalent processors/ directory
-        fileName path(processorsPath(io, inst, processorsDir(io)));
+        const fileName path(processorsPath(io, inst, processorsDir(io)));
 
-        mkDir(path);
-        fileName pathName(path/io.name());
+        // Note: delay mkdir to masterOFstream so it does not get created
+        //       if not needed (e.g. when running distributed)
+
+        const fileName pathName(path/io.name());
 
         if (io.global() || io.globalObject())
         {
@@ -480,6 +448,7 @@ bool Foam::fileOperations::collatedFileOperation::writeObject
                     << " appending to " << pathName << endl;
             }
 
+            mkDir(path);
             return appendObject(io, pathName, streamOpt);
         }
         else
@@ -604,11 +573,16 @@ Foam::word Foam::fileOperations::collatedFileOperation::processorsDir
                         break;
                     }
                 }
-                procDir +=
-                  + "_"
-                  + Foam::name(minProc)
-                  + "-"
-                  + Foam::name(maxProc);
+
+                // Add range if not all processors
+                if (maxProc-minProc+1 != nProcs_)
+                {
+                    procDir +=
+                      + "_"
+                      + Foam::name(minProc)
+                      + "-"
+                      + Foam::name(maxProc);
+                }
             }
         }
 

src/OpenFOAM/global/fileOperations/fileOperation/fileOperation.C

@@ -80,6 +80,7 @@ Foam::word Foam::fileOperation::processorsBaseDir = "processors";
 //- Caching (e.g. of time directories) - enabled by default
 int Foam::fileOperation::cacheLevel_(1);
 
+int Foam::fileOperation::nProcsFilter_(-1);
 
 // * * * * * * * * * * * * * * * Local Functions * * * * * * * * * * * * * * //
 
@@ -179,8 +180,6 @@ static bool parseProcsNumRange
 } // End anonymous namespace
 
 
-#if 0
-
 // Sorting of processor directories
 #include "stringOpsSort.H"
 namespace
@@ -214,7 +213,6 @@ void sortProcessorDirs(Foam::UList<Foam::fileOperation::dirIndex>& dirs)
 }
 
 } // End anonymous namespace
-#endif
 
 
 // * * * * * * * * * * * * * Static Member Functions * * * * * * * * * * * * //
@@ -431,6 +429,21 @@ Foam::fileOperation::lookupAndCacheProcessorsPath
     //             if directory status gets synchronised
     // - distributed() : different processors have different roots
     // - fileModificationChecking : (uncollated only) do IO on master only
+    // - nProcsFilter_ : if set to
+    //       0 : accept any directory (e.g. for redistributePar where we don't
+    //           know yet number of read/write procs)
+    //      -1 : accept only processorsDDD where DDD is nProcs(worldComm)
+    //      >0 : accept the exact mentioned number of prcessors
+
+
+    // Collated : check whether/how to filter processorsXXX directory names
+    const label targetNProcs 
+    (
+        (UPstream::parRun() && nProcsFilter_ < 0)
+      ? UPstream::nProcs(UPstream::worldComm)
+      : nProcsFilter_
+    );
+
 
     fileName path, pDir, local;
     procRangeType group;
@@ -528,6 +541,19 @@ Foam::fileOperation::lookupAndCacheProcessorsPath
             }
             else if (rNum != -1)
             {
+                if (targetNProcs > 1 && (targetNProcs != rNum))
+                {
+                    // Current directory can never contain wanted proc
+                    //Pout<< "For fName:" << fName
+                    //    << "Ignoring directory " << dirN
+                    //    << " since parsed rNum:" << rNum
+                    //    << " targetNProcs:" << targetNProcs
+                    //   << endl;
+                    //error::printStack(Pout);
+
+                    continue;
+                }
+
                 // "processorsNN" or "processorsNN_start-end"
                 nProcs = max(nProcs, rNum);
 
@@ -535,7 +561,7 @@ Foam::fileOperation::lookupAndCacheProcessorsPath
                 {
                     // "processorsNN"
 
-                    if (proci < rNum)
+                    if (proci < rNum || (nProcsFilter_ == 0))
                     {
                         // And it is also in range.
                         // Eg for "processors4": 3 is ok, 10 is not
@@ -544,7 +570,7 @@ Foam::fileOperation::lookupAndCacheProcessorsPath
                         pathTypeIdx.second() = proci;
                     }
                 }
-                else if (group.contains(proci))
+                else if (group.contains(proci) || (nProcsFilter_ == 0))
                 {
                     // "processorsNN_start-end"
                     // - save the local proc offset
@@ -560,6 +586,10 @@ Foam::fileOperation::lookupAndCacheProcessorsPath
             }
         }
 
+        // Sort processor directory names (natural order)
+        sortProcessorDirs(procDirs);
+
+
         // Global check of empty/exists.
         // 1 : empty directory
         // 2 : non-empty directory
@@ -571,7 +601,7 @@ Foam::fileOperation::lookupAndCacheProcessorsPath
         if (debug)
         {
             Pout<< "fileOperation::lookupProcessorsPath " << procPath
-                << " detected:" << procDirs << endl;
+                << " detected:" << flatOutput(procDirs) << endl;
         }
 
         if (UPstream::parRun() && (!distributed() || syncPar))
@@ -654,9 +684,6 @@ Foam::fileOperation::lookupAndCacheProcessorsPath
             }
         }
 
-        // Sort processor directory names (natural order)
-        /// sortProcessorDirs(procDirs);
-
         if (procDirsStatus & 2u)
         {
             if (cacheLevel() > 0)
@@ -680,7 +707,6 @@ Foam::fileOperation::lookupAndCacheProcessorsPath
 Foam::refPtr<Foam::fileOperation::dirIndexList>
 Foam::fileOperation::lookupProcessorsPath(const fileName& fName) const
 {
-    // Use parallel synchronisation
     return lookupAndCacheProcessorsPath(fName, true);
 }
 
@@ -844,6 +870,7 @@ Foam::fileName Foam::fileOperation::filePath
     }
 
     // Give preference to processors variant
+    fileName foundName;
     if (proci != -1)
     {
         // Get all processor directories
@@ -859,26 +886,40 @@ Foam::fileName Foam::fileOperation::filePath
                 {
                     Pout<< "fileOperation::filePath : " << collatedName << endl;
                 }
-                return collatedName;
+                foundName = collatedName;
             }
         }
     }
 
-    if (exists(fName, checkGzip, followLink))
+    //if (returnReduceOr(foundName.empty()))    // worldComm
+    if (foundName.empty())
+    {
+        // There is at least one processor that cannot find the processors
+        // directory. Re-do with straight supplied filename
+        if (exists(fName, checkGzip, followLink))
+        {
+            if (foundName.empty())
+            {
+                foundName = fName;
+            }
+        }
+    }
+
+    if (!foundName.empty())
     {
         if (debug)
         {
-            Pout<< "fileOperation::filePath : " << fName << endl;
+            Pout<< "fileOperation::filePath : " << foundName << endl;
         }
-        return fName;
     }
-
-    if (debug)
+    else
     {
-        Pout<< "fileOperation::filePath : Not found" << endl;
+        if (debug)
+        {
+            Pout<< "fileOperation::filePath : Not found" << endl;
+        }
     }
-
-    return fileName();
+    return foundName;
 }
 
 

src/OpenFOAM/global/fileOperations/fileOperation/fileOperation.H

@@ -150,6 +150,13 @@ private:
         //- Storage of the dummy file handler (demand-driven)
         static refPtr<fileOperation> dummyHandlerPtr_;
 
+        //- Filtering of processorsDDD directories (in parallel):
+        //      - >0 : only accept processorsDDD (default)
+        //      -  0 : accept anything (e.g. when detecting nprocs from
+        //             directory naming)
+        //      - -1 : use nProcs(UPstream::worldComm)
+        static int nProcsFilter_;
+
 
 protected:
 
@@ -452,6 +459,21 @@ public:
             return old;
         }
 
+        //- Return collated 'processorsDDD' filtering
+        static int nProcsFilter() noexcept
+        {
+            return nProcsFilter_;
+        }
+
+        //- Set collated 'processorsDDD' filtering (0 = off).
+        //  \return the previous value
+        static int nProcsFilter(int level) noexcept
+        {
+            int old(nProcsFilter_);
+            nProcsFilter_ = level;
+            return old;
+        }
+
         //- Sort directory entries according to time value,
         //  with "constant" appearing first (if it exists)
         static instantList sortTimes

src/OpenFOAM/global/fileOperations/fileOperation/fileOperationNew.C

@@ -373,6 +373,11 @@ Foam::fileOperation::New_impl
         // Allocate new handler with same type and similar IO ranks
         // but with different sub-ranks (and communicator)
 
+
+        // Temporarily override world comm to get through the initialisation
+        // (e.g. fileOperation::printRanks() which uses worldComm)
+        const label oldWorldComm = UPstream::commWorld(commAndIORanks.first());
+
         newHandler = fileOperation::New
         (
             origHandler.type(),
@@ -381,9 +386,13 @@ Foam::fileOperation::New_impl
             verbose
         );
 
+        UPstream::commWorld(oldWorldComm);
+
         if (newHandler)
         {
-            newHandler->nProcs(origHandler.nProcs());
+            // Make sure that the output format uses the correct number of
+            // 'active' ranks (instead of that of the origHandler)
+            newHandler->nProcs(subProcs.size());
             newHandler->storeComm();
         }
     }
@@ -400,9 +409,9 @@ Foam::fileOperation::New
     bool verbose
 )
 {
-    labelList subProcs = getSelectedProcs(useProc);
+    const labelList subProcs = getSelectedProcs(useProc);
 
-    return fileOperation::New_impl(origHandler, subProcs, verbose);
+    return New_impl(origHandler, subProcs, verbose);
 }
 
 
@@ -414,9 +423,9 @@ Foam::fileOperation::New
     bool verbose
 )
 {
-    labelList subProcs = getSelectedProcs(useProc);
+    const labelList subProcs = getSelectedProcs(useProc);
 
-    return fileOperation::New_impl(origHandler, subProcs, verbose);
+    return New_impl(origHandler, subProcs, verbose);
 }
 
 

src/OpenFOAM/global/fileOperations/fileOperation/fileOperationRanks.C

@@ -100,8 +100,6 @@ Foam::labelRange Foam::fileOperation::subRanks(const labelUList& mainIOranks)
         }
     }
 
-    Pout<< "subRanks: " << labelRange(begProc, (endProc-begProc)) << endl;
-
     return labelRange(begProc, (endProc-begProc));
 }
 

src/OpenFOAM/global/fileOperations/masterUncollatedFileOperation/masterUncollatedFileOperation.C

@@ -735,6 +735,23 @@ masterUncollatedFileOperation
     managedComm_(getManagedComm(comm_))  // Possibly locally allocated
 {
     init(verbose);
+
+    if (comm_ == -1)
+    {
+        FatalErrorInFunction<< "Problem comm_:" << comm_ << exit(FatalError);
+    }
+    if (UPstream::nProcs(comm_) == -1)
+    {
+        FatalErrorInFunction<< "Problem comm_:" << comm_
+            << " nProcs:" << UPstream::nProcs(comm_)
+            << exit(FatalError);
+    }
+    if (UPstream::myProcNo(comm_) == -1)
+    {
+        FatalErrorInFunction<< "Problem comm_:" << comm_
+            << " myProcNo:" << UPstream::myProcNo(comm_)
+            << exit(FatalError);
+    }
 }
 
 
@@ -750,6 +767,23 @@ masterUncollatedFileOperation
     managedComm_(-1)  // Externally managed
 {
     init(verbose);
+
+    if (comm_ == -1)
+    {
+        FatalErrorInFunction<< "Problem comm_:" << comm_ << exit(FatalError);
+    }
+    if (UPstream::nProcs(comm_) == -1)
+    {
+        FatalErrorInFunction<< "Problem comm_:" << comm_
+            << " nProcs:" << UPstream::nProcs(comm_)
+            << exit(FatalError);
+    }
+    if (UPstream::myProcNo(comm_) == -1)
+    {
+        FatalErrorInFunction<< "Problem comm_:" << comm_
+            << " myProcNo:" << UPstream::myProcNo(comm_)
+            << exit(FatalError);
+    }
 }
 
 
@@ -1095,6 +1129,7 @@ Foam::fileName Foam::fileOperations::masterUncollatedFileOperation::filePath
     {
         const bool oldParRun = UPstream::parRun(false);
         const int oldCache = fileOperation::cacheLevel(0);
+        const label oldNProcs = fileOperation::nProcs();
 
         // All masters search locally. Note that global objects might
         // fail (except on master). This gets handled later on (in PARENTOBJECT)
@@ -1111,6 +1146,7 @@ Foam::fileName Foam::fileOperations::masterUncollatedFileOperation::filePath
                 newInstancePath
             );
 
+        const_cast<masterUncollatedFileOperation&>(*this).nProcs(oldNProcs);
         fileOperation::cacheLevel(oldCache);
         UPstream::parRun(oldParRun);
 
@@ -1251,6 +1287,7 @@ Foam::fileName Foam::fileOperations::masterUncollatedFileOperation::dirPath
     {
         const bool oldParRun = UPstream::parRun(false);
         const int oldCache = fileOperation::cacheLevel(0);
+        const label oldNProcs = fileOperation::nProcs();
 
         objPath = filePathInfo
         (
@@ -1264,6 +1301,7 @@ Foam::fileName Foam::fileOperations::masterUncollatedFileOperation::dirPath
             newInstancePath
         );
 
+        const_cast<masterUncollatedFileOperation&>(*this).nProcs(oldNProcs);
         fileOperation::cacheLevel(oldCache);
         UPstream::parRun(oldParRun);
 
@@ -1453,11 +1491,13 @@ Foam::fileOperations::masterUncollatedFileOperation::findInstance
     {
         const bool oldParRun = UPstream::parRun(false);
         const int oldCache = fileOperation::cacheLevel(0);
+        const label oldNProcs = fileOperation::nProcs();
 
         if (exists(pDirs, io))
         {
             foundInstance = io.instance();
         }
+        const_cast<masterUncollatedFileOperation&>(*this).nProcs(oldNProcs);
         fileOperation::cacheLevel(oldCache);
         UPstream::parRun(oldParRun);
     }
@@ -1492,6 +1532,7 @@ Foam::fileOperations::masterUncollatedFileOperation::findInstance
     {
         const bool oldParRun = UPstream::parRun(false);
         const int oldCache = fileOperation::cacheLevel(0);
+        const label oldNProcs = fileOperation::nProcs();
 
         label instIndex = ts.size()-1;
 
@@ -1602,6 +1643,7 @@ Foam::fileOperations::masterUncollatedFileOperation::findInstance
             }
         }
 
+        const_cast<masterUncollatedFileOperation&>(*this).nProcs(oldNProcs);
         fileOperation::cacheLevel(oldCache);
         UPstream::parRun(oldParRun);  // Restore parallel state
     }
@@ -1677,6 +1719,7 @@ Foam::fileOperations::masterUncollatedFileOperation::readObjects
         // (through call to filePath which triggers parallel )
         const bool oldParRun = UPstream::parRun(false);
         const int oldCache = fileOperation::cacheLevel(0);
+        const label oldNProcs = fileOperation::nProcs();
 
         //- Use non-time searching version
         objectNames = fileOperation::readObjects
@@ -1720,6 +1763,7 @@ Foam::fileOperations::masterUncollatedFileOperation::readObjects
             }
         }
 
+        const_cast<masterUncollatedFileOperation&>(*this).nProcs(oldNProcs);
         fileOperation::cacheLevel(oldCache);
         UPstream::parRun(oldParRun);  // Restore parallel state
     }
@@ -2164,11 +2208,13 @@ bool Foam::fileOperations::masterUncollatedFileOperation::read
             // Do master-only reading always.
             const bool oldParRun = UPstream::parRun(false);
             const int oldCache = fileOperation::cacheLevel(0);
+            const label oldNProcs = fileOperation::nProcs();
 
             auto& is = io.readStream(typeName);
             ok = io.readData(is);
             io.close();
 
+            const_cast<masterUncollatedFileOperation&>(*this).nProcs(oldNProcs);
             fileOperation::cacheLevel(oldCache);
             UPstream::parRun(oldParRun);  // Restore parallel state
         }
@@ -2298,9 +2344,11 @@ Foam::instantList Foam::fileOperations::masterUncollatedFileOperation::findTimes
             // Do master-only reading always.
             const bool oldParRun = UPstream::parRun(false);
             const int oldCache = fileOperation::cacheLevel(0);
+            const label oldNProcs = fileOperation::nProcs();
 
             times = fileOperation::findTimes(directory, constantName);
 
+            const_cast<masterUncollatedFileOperation&>(*this).nProcs(oldNProcs);
             fileOperation::cacheLevel(oldCache);
             UPstream::parRun(oldParRun);  // Restore parallel state
         }
@@ -2702,30 +2750,46 @@ void Foam::fileOperations::masterUncollatedFileOperation::addWatches
 {
     const labelList& watchIndices = rio.watchIndices();
 
+    // Do on master and distribute effect to subprocs such that after
+    // all have consistent numbering & files
+
     DynamicList<label> newWatchIndices;
-    labelHashSet removedWatches(watchIndices);
-
-    for (const fileName& f : files)
+    if (UPstream::master())
     {
-        const label index = findWatch(watchIndices, f);
+        // Switch off comms inside findWatch/addWatch etc.
+        const bool oldParRun = UPstream::parRun(false);
+        const int oldCache = fileOperation::cacheLevel(0);
+        const label oldNProcs = fileOperation::nProcs();
 
-        if (index == -1)
-        {
-            newWatchIndices.append(addWatch(f));
-        }
-        else
-        {
-            // Existing watch
-            newWatchIndices.append(watchIndices[index]);
-            removedWatches.erase(index);
-        }
-    }
+        labelHashSet removedWatches(watchIndices);
 
-    // Remove any unused watches
-    for (const label index : removedWatches)
-    {
-        removeWatch(watchIndices[index]);
+        for (const fileName& f : files)
+        {
+            const label index = findWatch(watchIndices, f);
+
+            if (index == -1)
+            {
+                newWatchIndices.push_back(addWatch(f));
+            }
+            else
+            {
+                // Existing watch
+                newWatchIndices.push_back(watchIndices[index]);
+                removedWatches.erase(index);
+            }
+        }
+
+        // Remove any unused watches
+        for (const label index : removedWatches)
+        {
+            removeWatch(watchIndices[index]);
+        }
+
+        const_cast<masterUncollatedFileOperation&>(*this).nProcs(oldNProcs);
+        fileOperation::cacheLevel(oldCache);
+        UPstream::parRun(oldParRun);
     }
+    Pstream::broadcast(newWatchIndices);
 
     rio.watchIndices() = newWatchIndices;
 }

src/OpenFOAM/global/fileOperations/uncollatedFileOperation/uncollatedFileOperation.C

@@ -121,12 +121,11 @@ Foam::fileName Foam::fileOperations::uncollatedFileOperation::filePathInfo
             // Check if parallel "procesors" directory
             if (io.time().processorCase())
             {
-                refPtr<dirIndexList> pDirs
+                const refPtr<dirIndexList> pDirs
                 (
-                    fileOperation::lookupAndCacheProcessorsPath
+                    lookupProcessorsPath
                     (
-                        io.objectPath(),
-                        false // No additional parallel synchronisation
+                        io.objectPath()
                     )
                 );
 

src/OpenFOAM/global/fileOperations/uncollatedFileOperation/uncollatedFileOperation.H

@@ -85,7 +85,7 @@ protected:
         //  \return empty fileName if not found.
         virtual refPtr<dirIndexList> lookupProcessorsPath
         (
-            const fileName&
+            const fileName& objectPath
         ) const;
 
 

src/finiteVolume/fvMesh/fvMeshTools/fvMeshTools.C

@@ -760,12 +760,19 @@ Foam::fvMeshTools::loadOrCreateMeshImpl
         const label oldNumProcs = fileHandler().nProcs();
         const int oldCache = fileOperation::cacheLevel(0);
 
+        const fileName facesInstance = io.time().findInstance
+        (
+            meshSubDir,
+            "faces",
+            IOobject::MUST_READ
+        );
+
         patchEntries = polyBoundaryMeshEntries
         (
             IOobject
             (
                 "boundary",
-                io.instance(),
+                facesInstance,
                 meshSubDir,
                 io.db(),
                 IOobject::MUST_READ,
@@ -794,25 +801,6 @@ Foam::fvMeshTools::loadOrCreateMeshImpl
         // Non-null reference when a mesh exists on given processor
         haveLocalMesh = (*readHandlerPtr).good();
     }
-    else
-    {
-        // No file handler.
-        // Check for presence of the "faces" file,
-        // but for 'decompose', only need mesh on master.
-
-        haveLocalMesh =
-        (
-            decompose
-          ? UPstream::master()
-          : fileHandler().isFile
-            (
-                fileHandler().filePath
-                (
-                    io.time().path()/io.instance()/meshSubDir/"faces"
-                )
-            )
-        );
-    }
 
     // Globally consistent information about who has a mesh
     boolList haveMesh
@@ -820,6 +808,11 @@ Foam::fvMeshTools::loadOrCreateMeshImpl
         UPstream::allGatherValues<bool>(haveLocalMesh)
     );
 
+    // Make sure all have the same number of processors
+    label masterNProcs = fileHandler().nProcs();
+    Pstream::broadcast(masterNProcs);
+    const_cast<fileOperation&>(fileHandler()).nProcs(masterNProcs);
+
 
     autoPtr<fvMesh> meshPtr;
 
@@ -960,6 +953,7 @@ Foam::fvMeshTools::loadOrCreateMeshImpl
             )
         )
         {
+            const_cast<fileOperation&>(fileHandler()).nProcs(UPstream::nProcs(oldWorldComm));
             // Can use the handler communicator as is.
             UPstream::commWorld(fileHandler().comm());
         }

src/finiteVolume/fvMesh/fvMeshTools/fvMeshTools.H

@@ -220,6 +220,7 @@ public:
         const fvMesh& procMesh,
         const mapDistributePolyMesh& map,
         const bool decompose,
+        const fileName& writeHandlerInstance,
         refPtr<fileOperation>& writeHandler
     );
 };

src/finiteVolume/fvMesh/fvMeshTools/fvMeshToolsProcAddr.C

@@ -303,6 +303,7 @@ void Foam::fvMeshTools::writeProcAddressing
     const fvMesh& mesh,
     const mapDistributePolyMesh& map,
     const bool decompose,
+    const fileName& writeHandlerInstance,
     refPtr<fileOperation>& writeHandler
 )
 {
@@ -317,7 +318,7 @@ void Foam::fvMeshTools::writeProcAddressing
     // been done independently (as a registered object)
     IOobject ioAddr
     (
-        "procAddressing",
+        "proc-addressing",
         mesh.facesInstance(),
         polyMesh::meshSubDir,
         mesh.thisDb(),
@@ -421,23 +422,43 @@ void Foam::fvMeshTools::writeProcAddressing
         );
     }
 
-    auto oldHandler = fileOperation::fileHandler(writeHandler);
 
-    const bool cellOk = cellMap.write();
-    const bool faceOk = faceMap.write();
-    const bool pointOk = pointMap.write();
-    const bool patchOk = patchMap.write();
+    // Switch to using the correct
+    // - fileHandler
+    // - instance
+    // to write to the original mesh/time in the original format. Clunky!
+    // Bypass regIOobject writing to avoid taking over the current time
+    // as instance so instead of e.g. 'celllMap.write()' directly call
+    // the chosen file-handler.
 
-    writeHandler = fileOperation::fileHandler(oldHandler);
-
-    if (!cellOk || !faceOk || !pointOk || !patchOk)
+    const auto& tm = cellMap.time();
+    const IOstreamOption opt(tm.writeFormat(), tm.writeCompression());
     {
-        WarningInFunction
-            << "Failed to write some of "
-            << cellMap.objectRelPath() << ", "
-            << faceMap.objectRelPath() << ", "
-            << pointMap.objectRelPath() << ", "
-            << patchMap.objectRelPath() << endl;
+        auto oldHandler = fileOperation::fileHandler(writeHandler);
+
+        cellMap.instance() = writeHandlerInstance;
+        const bool cellOk = fileHandler().writeObject(cellMap, opt, true);
+
+        faceMap.instance() = writeHandlerInstance;
+        const bool faceOk = fileHandler().writeObject(faceMap, opt, true);
+
+        pointMap.instance() = writeHandlerInstance;
+        const bool pointOk = fileHandler().writeObject(pointMap, opt, true);
+
+        patchMap.instance() = writeHandlerInstance;
+        const bool patchOk = fileHandler().writeObject(patchMap, opt, true);
+
+        writeHandler = fileOperation::fileHandler(oldHandler);
+
+        if (!cellOk || !faceOk || !pointOk || !patchOk)
+        {
+            WarningInFunction
+                << "Failed to write some of "
+                << cellMap.objectRelPath() << ", "
+                << faceMap.objectRelPath() << ", "
+                << pointMap.objectRelPath() << ", "
+                << patchMap.objectRelPath() << endl;
+        }
     }
 }
 

tutorials/IO/fileHandler/Allrun

@@ -21,11 +21,14 @@ application=$(getApplication)
 
 #- Test writing collated format
 runApplication decomposePar -fileHandler collated
+#runParallel redistributePar -decompose -fileHandler collated
 runParallel $application -fileHandler collated
 runApplication reconstructPar -latestTime -fileHandler collated
+#runParallel -s reconstruct redistributePar -reconstruct -latestTime -fileHandler collated
 
 #- Test writing uncollated format
 runApplication -s uncollated decomposePar -fileHandler uncollated -force
+#runParallel -s uncollated redistributePar -decompose -fileHandler uncollated
 runParallel -s uncollated $application -fileHandler uncollated
 
 

tutorials/IO/fileHandler/Test-distributed

@@ -46,7 +46,10 @@ CASE_ROOTS
 
 #export FOAM_ABORT=true
 
-runParallel -s decompose redistributePar -decompose -case test-distribute/machineA/testcase
+runParallel -s decompose redistributePar \
+    -decompose \
+    -case test-distribute/machineA/testcase \
+    -fileHandler masterUncollated
 
 runParallel checkMesh -case test-distribute/machineA/testcase
 

tutorials/IO/fileHandler_dynamicCode/0/U

@@ -0,0 +1,41 @@
+/*--------------------------------*- C++ -*----------------------------------*\
+| =========                 |                                                 |
+| \\      /  F ield         | OpenFOAM: The Open Source CFD Toolbox           |
+|  \\    /   O peration     | Version:  v2306                                 |
+|   \\  /    A nd           | Website:  www.openfoam.com                      |
+|    \\/     M anipulation  |                                                 |
+\*---------------------------------------------------------------------------*/
+FoamFile
+{
+    version     2.0;
+    format      ascii;
+    class       volVectorField;
+    object      U;
+}
+// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
+
+dimensions      [0 1 -1 0 0 0 0];
+
+internalField   uniform (0 0 0);
+
+boundaryField
+{
+    movingWall
+    {
+        type            fixedValue;
+        value           uniform (1 0 0);
+    }
+
+    fixedWalls
+    {
+        type            noSlip;
+    }
+
+    frontAndBack
+    {
+        type            empty;
+    }
+}
+
+
+// ************************************************************************* //

tutorials/IO/fileHandler_dynamicCode/0/p

@@ -0,0 +1,40 @@
+/*--------------------------------*- C++ -*----------------------------------*\
+| =========                 |                                                 |
+| \\      /  F ield         | OpenFOAM: The Open Source CFD Toolbox           |
+|  \\    /   O peration     | Version:  v2306                                 |
+|   \\  /    A nd           | Website:  www.openfoam.com                      |
+|    \\/     M anipulation  |                                                 |
+\*---------------------------------------------------------------------------*/
+FoamFile
+{
+    version     2.0;
+    format      ascii;
+    class       volScalarField;
+    object      p;
+}
+// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
+
+dimensions      [0 2 -2 0 0 0 0];
+
+internalField   uniform 0;
+
+boundaryField
+{
+    movingWall
+    {
+        type            zeroGradient;
+    }
+
+    fixedWalls
+    {
+        type            zeroGradient;
+    }
+
+    frontAndBack
+    {
+        type            empty;
+    }
+}
+
+
+// ************************************************************************* //

tutorials/IO/fileHandler_dynamicCode/constant/transportProperties

@@ -0,0 +1,20 @@
+/*--------------------------------*- C++ -*----------------------------------*\
+| =========                 |                                                 |
+| \\      /  F ield         | OpenFOAM: The Open Source CFD Toolbox           |
+|  \\    /   O peration     | Version:  v2306                                 |
+|   \\  /    A nd           | Website:  www.openfoam.com                      |
+|    \\/     M anipulation  |                                                 |
+\*---------------------------------------------------------------------------*/
+FoamFile
+{
+    version     2.0;
+    format      ascii;
+    class       dictionary;
+    object      transportProperties;
+}
+// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
+
+nu              0.01;
+
+
+// ************************************************************************* //

tutorials/IO/fileHandler_dynamicCode/system/blockMeshDict

@@ -0,0 +1,72 @@
+/*--------------------------------*- C++ -*----------------------------------*\
+| =========                 |                                                 |
+| \\      /  F ield         | OpenFOAM: The Open Source CFD Toolbox           |
+|  \\    /   O peration     | Version:  v2306                                 |
+|   \\  /    A nd           | Website:  www.openfoam.com                      |
+|    \\/     M anipulation  |                                                 |
+\*---------------------------------------------------------------------------*/
+FoamFile
+{
+    version     2.0;
+    format      ascii;
+    class       dictionary;
+    object      blockMeshDict;
+}
+// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
+
+scale   0.1;
+
+vertices
+(
+    (0 0 0)
+    (1 0 0)
+    (1 1 0)
+    (0 1 0)
+    (0 0 0.1)
+    (1 0 0.1)
+    (1 1 0.1)
+    (0 1 0.1)
+);
+
+blocks
+(
+    hex (0 1 2 3 4 5 6 7) (5 5 1) simpleGrading (1 1 1)
+);
+
+edges
+(
+);
+
+boundary
+(
+    movingWall
+    {
+        type wall;
+        faces
+        (
+            (3 7 6 2)
+        );
+    }
+    fixedWalls
+    {
+        type wall;
+        faces
+        (
+            (0 4 7 3)
+            (2 6 5 1)
+            (1 5 4 0)
+        );
+    }
+    frontAndBack
+    {
+        type empty;
+        faces
+        (
+            (0 3 2 1)
+            (4 5 6 7)
+        );
+    }
+);
+
+
+// ************************************************************************* //

tutorials/IO/fileHandler_dynamicCode/system/controlDict

@@ -0,0 +1,111 @@
+/*--------------------------------*- C++ -*----------------------------------*\
+| =========                 |                                                 |
+| \\      /  F ield         | OpenFOAM: The Open Source CFD Toolbox           |
+|  \\    /   O peration     | Version:  v2306                                 |
+|   \\  /    A nd           | Website:  www.openfoam.com                      |
+|    \\/     M anipulation  |                                                 |
+\*---------------------------------------------------------------------------*/
+FoamFile
+{
+    version     2.0;
+    format      ascii;
+    class       dictionary;
+    object      controlDict;
+}
+// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
+
+DebugSwitches
+{
+//    OSspecific  1;
+//    POSIX       2;
+//    codeStream  1;
+//    codedBase   1;
+//    masterUncollated    1;
+//    hostCollated      1;
+//    collated        1;
+//    fileMonitor 1;
+}
+
+OptimisationSwitches
+{
+    fileModificationSkew    1;
+}
+
+//- Set deltaT
+#include "solverControls"
+
+
+application     icoFoam;
+
+startFrom       startTime;
+
+startTime       0;
+
+stopAt          endTime;
+//stopAt          #codeStream
+//{
+//    code
+//    #{
+//        os  << "nextWrite;" << nl;
+//    #};
+//};
+
+
+endTime         0.04;
+
+//deltaT          0.005;
+
+writeControl    timeStep;
+
+writeInterval   20;
+
+purgeWrite      0;
+
+writeFormat     ascii;
+
+writePrecision  6;
+
+writeCompression off;
+
+timeFormat      general;
+
+timePrecision   6;
+
+runTimeModifiable true;
+
+functions
+{
+    fileUpdate
+    {
+        type                timeActivatedFileUpdate;
+        libs                (utilityFunctionObjects);
+        writeControl        timeStep;
+        writeInterval       1;
+        fileToUpdate        "<system>/solverControls";
+
+        timeVsFile
+        (
+            (-1     "<system>/solverControls.0" )
+            ( 0.02  "<system>/solverControls.5" )
+        );
+    }
+
+    // Wait a bit to make runTimeModifiable work.
+    //  - see also fileModificationSkew
+    //  - could also directly use system FO
+    sleep
+    {
+        type                coded;
+        libs                (utilityFunctionObjects);
+        name                sleep;
+        writeControl        timeStep;
+
+        codeExecute
+        #{
+            sleep(1);
+        #};
+    }
+}
+
+
+// ************************************************************************* //

tutorials/IO/fileHandler_dynamicCode/system/decomposeParDict

@@ -0,0 +1,27 @@
+/*--------------------------------*- C++ -*----------------------------------*\
+| =========                 |                                                 |
+| \\      /  F ield         | OpenFOAM: The Open Source CFD Toolbox           |
+|  \\    /   O peration     | Version:  v2306                                 |
+|   \\  /    A nd           | Website:  www.openfoam.com                      |
+|    \\/     M anipulation  |                                                 |
+\*---------------------------------------------------------------------------*/
+FoamFile
+{
+    version     2.0;
+    format      ascii;
+    class       dictionary;
+    object      decomposeParDict;
+}
+// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
+
+numberOfSubdomains  5;
+
+method  hierarchical;
+
+coeffs
+{
+    n   (5 1 1);
+}
+
+
+// ************************************************************************* //

tutorials/IO/fileHandler_dynamicCode/system/fvSchemes

@@ -0,0 +1,50 @@
+/*--------------------------------*- C++ -*----------------------------------*\
+| =========                 |                                                 |
+| \\      /  F ield         | OpenFOAM: The Open Source CFD Toolbox           |
+|  \\    /   O peration     | Version:  v2306                                 |
+|   \\  /    A nd           | Website:  www.openfoam.com                      |
+|    \\/     M anipulation  |                                                 |
+\*---------------------------------------------------------------------------*/
+FoamFile
+{
+    version     2.0;
+    format      ascii;
+    class       dictionary;
+    object      fvSchemes;
+}
+// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
+
+ddtSchemes
+{
+    default         Euler;
+}
+
+gradSchemes
+{
+    default         Gauss linear;
+    grad(p)         Gauss linear;
+}
+
+divSchemes
+{
+    default         none;
+    div(phi,U)      Gauss linear;
+}
+
+laplacianSchemes
+{
+    default         Gauss linear orthogonal;
+}
+
+interpolationSchemes
+{
+    default         linear;
+}
+
+snGradSchemes
+{
+    default         orthogonal;
+}
+
+
+// ************************************************************************* //

tutorials/IO/fileHandler_dynamicCode/system/fvSolution

@@ -0,0 +1,51 @@
+/*--------------------------------*- C++ -*----------------------------------*\
+| =========                 |                                                 |
+| \\      /  F ield         | OpenFOAM: The Open Source CFD Toolbox           |
+|  \\    /   O peration     | Version:  v2306                                 |
+|   \\  /    A nd           | Website:  www.openfoam.com                      |
+|    \\/     M anipulation  |                                                 |
+\*---------------------------------------------------------------------------*/
+FoamFile
+{
+    version     2.0;
+    format      ascii;
+    class       dictionary;
+    object      fvSolution;
+}
+// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
+
+solvers
+{
+    p
+    {
+        solver          PCG;
+        preconditioner  DIC;
+        tolerance       1e-06;
+        relTol          0.05;
+    }
+
+    pFinal
+    {
+        $p;
+        relTol          0;
+    }
+
+    U
+    {
+        solver          smoothSolver;
+        smoother        symGaussSeidel;
+        tolerance       1e-05;
+        relTol          0;
+    }
+}
+
+PISO
+{
+    nCorrectors     2;
+    nNonOrthogonalCorrectors 0;
+    pRefCell        0;
+    pRefValue       0;
+}
+
+
+// ************************************************************************* //

tutorials/IO/fileHandler_dynamicCode/system/solverControls

@@ -0,0 +1,11 @@
+/*--------------------------------*- C++ -*----------------------------------*\
+| =========                 |                                                 |
+| \\      /  F ield         | OpenFOAM: The Open Source CFD Toolbox           |
+|  \\    /   O peration     | Version:  v2306                                 |
+|   \\  /    A nd           | Website:  www.openfoam.com                      |
+|    \\/     M anipulation  |                                                 |
+\*---------------------------------------------------------------------------*/
+
+deltaT          0.001;
+
+// ************************************************************************* //

tutorials/IO/fileHandler_dynamicCode/system/solverControls.0

@@ -0,0 +1,11 @@
+/*--------------------------------*- C++ -*----------------------------------*\
+| =========                 |                                                 |
+| \\      /  F ield         | OpenFOAM: The Open Source CFD Toolbox           |
+|  \\    /   O peration     | Version:  v2306                                 |
+|   \\  /    A nd           | Website:  www.openfoam.com                      |
+|    \\/     M anipulation  |                                                 |
+\*---------------------------------------------------------------------------*/
+
+deltaT          0.005;
+
+// ************************************************************************* //

tutorials/IO/fileHandler_dynamicCode/system/solverControls.5

@@ -0,0 +1,11 @@
+/*--------------------------------*- C++ -*----------------------------------*\
+| =========                 |                                                 |
+| \\      /  F ield         | OpenFOAM: The Open Source CFD Toolbox           |
+|  \\    /   O peration     | Version:  v2306                                 |
+|   \\  /    A nd           | Website:  www.openfoam.com                      |
+|    \\/     M anipulation  |                                                 |
+\*---------------------------------------------------------------------------*/
+
+deltaT          0.001;
+
+// ************************************************************************* //

tutorials/heatTransfer/buoyantBoussinesqSimpleFoam/iglooWithFridges/system/decomposeParDict

@@ -14,8 +14,6 @@ FoamFile
 }
 // * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
 
-numberOfSubdomains 2;
-
 method             scotch;