From bcd873ccfe267c7e356474e2894d9897effbdeb2 Mon Sep 17 00:00:00 2001
From: mattijs <mattijs>
Date: Fri, 12 May 2023 18:43:32 +0200
Subject: [PATCH 01/12] ENH: Update redistribute clouds with
 readOnProc/writeOnProc

- when reading, detect all clouds on all processors and uses this when
  reading fields. Similarly, when writing it uses writeOnProc to skip
  clouds that are empty on any particular processor.

Co-authored-by: Mark Olesen <>
---
 .../parLagrangianDistributor.C                | 115 ++++--
 .../parLagrangianDistributor.H                |  26 +-
 .../parLagrangianDistributorFields.C          |  27 +-
 .../parLagrangianDistributorTemplates.C       | 367 +++++++++---------
 .../redistributePar/redistributeLagrangian.H  |  36 +-
 5 files changed, 309 insertions(+), 262 deletions(-)

diff --git a/applications/utilities/parallelProcessing/redistributePar/parLagrangianDistributor.C b/applications/utilities/parallelProcessing/redistributePar/parLagrangianDistributor.C
index b269a43a3c..626cb9c642 100644
--- a/applications/utilities/parallelProcessing/redistributePar/parLagrangianDistributor.C
+++ b/applications/utilities/parallelProcessing/redistributePar/parLagrangianDistributor.C
@@ -52,7 +52,7 @@ Foam::parLagrangianDistributor::parLagrangianDistributor
     const mapDistribute& cellMap = distMap_.cellMap();
 
     // Get destination processors and cells
-    destinationProcID_ = labelList(tgtMesh_.nCells(), Pstream::myProcNo());
+    destinationProcID_ = labelList(tgtMesh_.nCells(), UPstream::myProcNo());
     cellMap.reverseDistribute(nSrcCells, destinationProcID_);
 
     destinationCell_ = identity(tgtMesh_.nCells());
@@ -67,21 +67,39 @@ Foam::parLagrangianDistributor::parLagrangianDistributor
 void Foam::parLagrangianDistributor::findClouds
 (
     const fvMesh& mesh,
-    wordList& cloudNames,
-    List<wordList>& objectNames
+    wordList& cloudNames,       // All cloud names on any processor
+    boolList& haveClouds,       // Per cloud name, whether my processor has it
+    List<wordList>& objectNames // Per cloud name, the field name
 )
 {
-    fileNameList localCloudDirs
+    const IOobject io
     (
-        readDir
+        cloud::prefix,
+        mesh.time().timeName(),
+        mesh,
+        IOobjectOption::MUST_READ,
+        IOobjectOption::NO_WRITE
+    );
+
+    // Using the fileHandler:
+    // - use fileHandler to synthesise correct processor directory
+    // - cannot use readObjects since assumes all processors have same
+    //   files (i.e. it only checks processor0)
+    const fileNameList localCloudDirs
+    (
+        fileHandler().readDir
         (
-            mesh.time().timePath()
-          / mesh.dbDir()
-          / cloud::prefix,
+            fileHandler().objectPath
+            (
+                io,
+                word::null  // typeName: not used currently
+            ),
             fileName::DIRECTORY
         )
     );
 
+
+    // Copy out the local cloud names (and fileName -> word)
     cloudNames.resize_nocopy(localCloudDirs.size());
     forAll(localCloudDirs, i)
     {
@@ -92,18 +110,34 @@ void Foam::parLagrangianDistributor::findClouds
     Pstream::combineReduce(cloudNames, ListOps::uniqueEqOp<word>());
     Foam::sort(cloudNames);  // Consistent order
 
-    objectNames.clear();
+
+    // See which of the global cloudNames I have
+    haveClouds.resize_nocopy(cloudNames.size());
+    haveClouds = false;
+
+    for (const fileName& localCloudName : localCloudDirs)
+    {
+        const label cloudi = cloudNames.find(localCloudName);
+        if (cloudi >= 0)
+        {
+            haveClouds[cloudi] = true;
+        }
+    }
+
+    // Collect fields per cloud
     objectNames.resize(cloudNames.size());
 
     for (const fileName& localCloudName : localCloudDirs)
     {
         // Do local scan for valid cloud objects
+        const bool oldParRun = UPstream::parRun(false);
         IOobjectList localObjs
         (
             mesh,
             mesh.time().timeName(),
             cloud::prefix/localCloudName
         );
+        UPstream::parRun(oldParRun);
 
         bool isCloud = false;
         if (localObjs.erase("coordinates"))
@@ -141,8 +175,7 @@ Foam::parLagrangianDistributor::distributeLagrangianPositions
 ) const
 {
     //Debug(lpi.size());
-
-    const label oldLpi = lpi.size();
+    //const label oldLpi = lpi.size();
 
     labelListList sendMap;
 
@@ -154,7 +187,7 @@ Foam::parLagrangianDistributor::distributeLagrangianPositions
         // neighbour processors
         List<IDLList<passivePositionParticle>> particleTransferLists
         (
-            Pstream::nProcs()
+            UPstream::nProcs()
         );
 
         // Per particle the destination processor
@@ -233,13 +266,15 @@ Foam::parLagrangianDistributor::distributeLagrangianPositions
             }
         }
 
-        if (lagrangianPositions.size())
+        const bool writeOnProc = lagrangianPositions.size();
+
+        //if (writeOnProc)
         {
             // Write coordinates file
             IOPosition<passivePositionParticleCloud>
             (
                 lagrangianPositions
-            ).write();
+            ).write(writeOnProc);
 
             // Optionally write positions file in v1706 format and earlier
             if (particle::writeLagrangianPositions)
@@ -248,35 +283,33 @@ Foam::parLagrangianDistributor::distributeLagrangianPositions
                 (
                      lagrangianPositions,
                      cloud::geometryType::POSITIONS
-                 ).write();
+                 ).write(writeOnProc);
             }
         }
-        else if (oldLpi)
-        {
-            // When running with -overwrite it should also delete the old
-            // files. Below works but is not optimal.
-
-            // Remove any existing coordinates
-            const fileName oldCoords
-            (
-                IOPosition<passivePositionParticleCloud>
-                (
-                    lagrangianPositions
-                ).objectPath()
-            );
-            Foam::rm(oldCoords);
-
-            // Remove any existing positions
-            const fileName oldPos
-            (
-                IOPosition<passivePositionParticleCloud>
-                (
-                    lagrangianPositions,
-                    cloud::geometryType::POSITIONS
-                ).objectPath()
-            );
-            Foam::rm(oldPos);
-        }
+        //else if (!writeOnProc && oldLpi)
+        //{
+        //    // When running with -overwrite it should also delete the old
+        //    // files. Below works but is not optimal.
+        //
+        //    // Remove any existing coordinates
+        //    Foam::rm
+        //    (
+        //        IOPosition<passivePositionParticleCloud>
+        //        (
+        //            lagrangianPositions
+        //        ).objectPath()
+        //    );
+        //
+        //    // Remove any existing positions
+        //    Foam::rm
+        //    (
+        //        IOPosition<passivePositionParticleCloud>
+        //        (
+        //            lagrangianPositions,
+        //            cloud::geometryType::POSITIONS
+        //        ).objectPath()
+        //    );
+        //}
 
         // Restore cloud name
         lpi.rename(cloudName);
diff --git a/applications/utilities/parallelProcessing/redistributePar/parLagrangianDistributor.H b/applications/utilities/parallelProcessing/redistributePar/parLagrangianDistributor.H
index 6ce206e795..f79054a9f8 100644
--- a/applications/utilities/parallelProcessing/redistributePar/parLagrangianDistributor.H
+++ b/applications/utilities/parallelProcessing/redistributePar/parLagrangianDistributor.H
@@ -6,7 +6,7 @@
      \\/     M anipulation  |
 -------------------------------------------------------------------------------
     Copyright (C) 2015 OpenFOAM Foundation
-    Copyright (C) 2018-2022 OpenCFD Ltd.
+    Copyright (C) 2018-2023 OpenCFD Ltd.
 -------------------------------------------------------------------------------
 License
     This file is part of OpenFOAM.
@@ -114,7 +114,14 @@ public:
         static void findClouds
         (
             const fvMesh&,
+
+            //! All cloud names on any processor
             wordList& cloudNames,
+
+            //! Per cloud name, whether my processor has it
+            boolList& haveClouds,
+
+            //! Per cloud nmae, the field names
             List<wordList>& objectNames
         );
 
@@ -131,6 +138,7 @@ public:
         static label readFields
         (
             const passivePositionParticleCloud& cloud,
+            const bool haveCloud,
             const IOobjectList& objects,
             const wordRes& selectedFields = wordRes()
         );
@@ -139,18 +147,11 @@ public:
         static label readAllFields
         (
             const passivePositionParticleCloud& cloud,
+            const bool haveCloud,
             const IOobjectList& objects,
             const wordRes& selectedFields = wordRes()
         );
 
-        //- Read and store all fields for known cloud field types
-        //  Uses the current cloud instance to obtain the IOobjectList
-        static label readAllFields
-        (
-            const passivePositionParticleCloud& cloud,
-            const wordRes& selectedFields = wordRes()
-        );
-
 
     // Member Functions
 
@@ -171,6 +172,7 @@ public:
         (
             const mapDistributeBase& lagrangianMap,
             const word& cloudName,
+            const bool haveCloud,
             const IOobjectList& cloudObjs,
             const wordRes& selectedFields
         ) const;
@@ -189,6 +191,7 @@ public:
         (
             const mapDistributeBase& map,
             const word& cloudName,
+            const bool haveCloud,
             const IOobjectList& objects,
             const wordRes& selectedFields = wordRes()
         ) const;
@@ -199,11 +202,14 @@ public:
         (
             const mapDistributeBase& map,
             const word& cloudName,
+            const bool haveCloud,
             const IOobjectList& objects,
             const wordRes& selectedFields = wordRes()
         ) const;
 
-        //- Redistribute and write stored lagrangian fields
+        //- Redistribute and write stored lagrangian fields.
+        //  Note: does no reading so no need to check for existence
+        //  of lagrangian files
         template<class Container>
         label distributeStoredFields
         (
diff --git a/applications/utilities/parallelProcessing/redistributePar/parLagrangianDistributorFields.C b/applications/utilities/parallelProcessing/redistributePar/parLagrangianDistributorFields.C
index 730c3fcc1b..9c16c360b5 100644
--- a/applications/utilities/parallelProcessing/redistributePar/parLagrangianDistributorFields.C
+++ b/applications/utilities/parallelProcessing/redistributePar/parLagrangianDistributorFields.C
@@ -5,7 +5,7 @@
     \\  /    A nd           | www.openfoam.com
      \\/     M anipulation  |
 -------------------------------------------------------------------------------
-    Copyright (C) 2022 OpenCFD Ltd.
+    Copyright (C) 2022-2023 OpenCFD Ltd.
 -------------------------------------------------------------------------------
 License
     This file is part of OpenFOAM.
@@ -33,6 +33,7 @@ License
 Foam::label Foam::parLagrangianDistributor::readAllFields
 (
     const passivePositionParticleCloud& cloud,
+    const bool haveCloud,
     const IOobjectList& cloudObjs,
     const wordRes& selectedFields
 )
@@ -48,6 +49,7 @@ Foam::label Foam::parLagrangianDistributor::readAllFields
             <IOField<Type>>                                                   \
             (                                                                 \
                 cloud,                                                        \
+                haveCloud,                                                    \
                 cloudObjs,                                                    \
                 selectedFields                                                \
             );                                                                \
@@ -56,6 +58,7 @@ Foam::label Foam::parLagrangianDistributor::readAllFields
             <IOField<Field<Type>>>                                            \
             (                                                                 \
                 cloud,                                                        \
+                haveCloud,                                                    \
                 cloudObjs,                                                    \
                 selectedFields                                                \
             );                                                                \
@@ -64,6 +67,7 @@ Foam::label Foam::parLagrangianDistributor::readAllFields
             <CompactIOField<Field<Type>, Type>>                               \
             (                                                                 \
                 cloud,                                                        \
+                haveCloud,                                                    \
                 cloudObjs,                                                    \
                 selectedFields                                                \
             );                                                                \
@@ -84,21 +88,22 @@ Foam::label Foam::parLagrangianDistributor::readAllFields
 }
 
 
-Foam::label Foam::parLagrangianDistributor::readAllFields
-(
-    const passivePositionParticleCloud& cloud,
-    const wordRes& selectedFields
-)
-{
-    IOobjectList cloudObjs(cloud, cloud.time().timeName());
-    return readAllFields(cloud, cloudObjs, selectedFields);
-}
+//Foam::label Foam::parLagrangianDistributor::readAllFields
+//(
+//    const passivePositionParticleCloud& cloud,
+//    const wordRes& selectedFields
+//)
+//{
+//    IOobjectList cloudObjs(cloud, cloud.time().timeName());
+//    return readAllFields(cloud, cloudObjs, selectedFields);
+//}
 
 
 Foam::label Foam::parLagrangianDistributor::distributeAllFields
 (
     const mapDistributeBase& lagrangianMap,
     const word& cloudName,
+    const bool haveCloud,
     const IOobjectList& cloudObjs,
     const wordRes& selectedFields
 ) const
@@ -114,6 +119,7 @@ Foam::label Foam::parLagrangianDistributor::distributeAllFields
             (                                                                 \
                 lagrangianMap,                                                \
                 cloudName,                                                    \
+                haveCloud,                                                    \
                 cloudObjs,                                                    \
                 selectedFields                                                \
             );                                                                \
@@ -122,6 +128,7 @@ Foam::label Foam::parLagrangianDistributor::distributeAllFields
             (                                                                 \
                 lagrangianMap,                                                \
                 cloudName,                                                    \
+                haveCloud,                                                    \
                 cloudObjs,                                                    \
                 selectedFields                                                \
             );                                                                \
diff --git a/applications/utilities/parallelProcessing/redistributePar/parLagrangianDistributorTemplates.C b/applications/utilities/parallelProcessing/redistributePar/parLagrangianDistributorTemplates.C
index 3ea8ec4a50..e44e949c44 100644
--- a/applications/utilities/parallelProcessing/redistributePar/parLagrangianDistributorTemplates.C
+++ b/applications/utilities/parallelProcessing/redistributePar/parLagrangianDistributorTemplates.C
@@ -64,91 +64,85 @@ Foam::label Foam::parLagrangianDistributor::distributeFields
 (
     const mapDistributeBase& map,
     const word& cloudName,
+    const bool haveCloud,
     const IOobjectList& objects,
     const wordRes& selectedFields
 ) const
 {
-    typedef IOField<Type> fieldType;
+    typedef IOField<Type> Container;
 
     const wordList fieldNames
     (
-        filterObjects<fieldType>
+        filterObjects<IOField<Type>>
         (
             objects,
             selectedFields
         )
     );
 
+    // Read if present
+    IOobject srcIOobject
+    (
+        "none",
+        srcMesh_.time().timeName(),
+        cloud::prefix/cloudName,
+        srcMesh_,
+        IOobjectOption::LAZY_READ,
+        IOobjectOption::NO_WRITE,
+        IOobjectOption::NO_REGISTER
+    );
 
-    bool reconstruct = false;
+    IOobject tgtIOobject
+    (
+        "none",
+        tgtMesh_.time().timeName(),
+        cloud::prefix/cloudName,
+        tgtMesh_,
+        IOobjectOption::NO_READ,
+        IOobjectOption::NO_WRITE,
+        IOobjectOption::NO_REGISTER
+    );
 
-    label nFields = 0;
-    for (const word& objectName : fieldNames)
+    //bool reconstruct = false;
+
+    if (verbose_ && fieldNames.size())
     {
-        if (!nFields)
-        {
-            // Performing an all-to-one (reconstruct)?
-            reconstruct =
-                returnReduceAnd(!map.constructSize() || Pstream::master());
-        }
-
-        if (verbose_)
-        {
-            if (!nFields)
-            {
-                Info<< "    Distributing lagrangian "
-                    << fieldType::typeName << "s\n" << nl;
-            }
-            Info<< "        " <<  objectName << nl;
-        }
-        ++nFields;
-
-        // Read if present
-        IOField<Type> field
-        (
-            IOobject
-            (
-                objectName,
-                srcMesh_.time().timeName(),
-                cloud::prefix/cloudName,
-                srcMesh_,
-                IOobject::READ_IF_PRESENT,
-                IOobject::NO_WRITE,
-                IOobject::NO_REGISTER
-            ),
-            label(0)
-        );
-
-        map.distribute(field);
-
-
-        const IOobject fieldIO
-        (
-            objectName,
-            tgtMesh_.time().timeName(),
-            cloud::prefix/cloudName,
-            tgtMesh_,
-            IOobject::NO_READ,
-            IOobject::NO_WRITE,
-            IOobject::NO_REGISTER
-        );
-
-        if (field.size())
-        {
-            IOField<Type>(fieldIO, std::move(field)).write();
-        }
-        else if (!reconstruct)
-        {
-            // When running with -overwrite it should also delete the old
-            // files. Below works but is not optimal.
-
-            const fileName fldName(fieldIO.objectPath());
-            Foam::rm(fldName);
-        }
+        Info<< "    Distributing lagrangian "
+            << Container::typeName << "s\n" << nl;
     }
 
-    if (nFields && verbose_) Info<< endl;
-    return nFields;
+    for (const word& objectName : fieldNames)
+    {
+        if (verbose_)
+        {
+            Info<< "        " <<  objectName << nl;
+        }
+
+        srcIOobject.resetHeader(objectName);
+        tgtIOobject.resetHeader(objectName);
+
+        // Read if present (ie, haveCloud means readOnProc)
+        Container field(srcIOobject, haveCloud);
+
+        // Distribute
+        map.distribute(field);
+
+        const bool writeOnProc = field.size();
+
+        // Write
+        Container(tgtIOobject, std::move(field)).write(writeOnProc);
+
+        //if (!writeOnProc && !reconstruct)
+        //{
+        //    // When running with -overwrite it should also delete the old
+        //    // files. Below works but is not optimal.
+        //
+        //    Foam::rm(tgtIOobject.objectPath());
+        //}
+    }
+
+    if (verbose_ && fieldNames.size()) Info<< endl;
+    return fieldNames.size();
 }
 
 
@@ -157,25 +151,27 @@ Foam::label Foam::parLagrangianDistributor::distributeFieldFields
 (
     const mapDistributeBase& map,
     const word& cloudName,
+    const bool haveCloud,
     const IOobjectList& objects,
     const wordRes& selectedFields
 ) const
 {
-    typedef CompactIOField<Field<Type>, Type> fieldType;
+    typedef CompactIOField<Field<Type>, Type> Container;
 
     DynamicList<word> fieldNames;
 
-    fieldNames.append
+    // CompactIOField Field names
+    fieldNames.push_back
     (
-        filterObjects<fieldType>
+        filterObjects<CompactIOField<Field<Type>, Type>>
         (
             objects,
             selectedFields
         )
     );
 
-    // Append IOField Field names
-    fieldNames.append
+    // IOField Field names
+    fieldNames.push_back
     (
         filterObjects<IOField<Field<Type>>>
         (
@@ -184,80 +180,69 @@ Foam::label Foam::parLagrangianDistributor::distributeFieldFields
         )
     );
 
-    bool reconstruct = false;
+    // Read if present
+    IOobject srcIOobject
+    (
+        "none",
+        srcMesh_.time().timeName(),
+        cloud::prefix/cloudName,
+        srcMesh_,
+        IOobjectOption::LAZY_READ,
+        IOobjectOption::NO_WRITE,
+        IOobjectOption::NO_REGISTER
+    );
+
+    IOobject tgtIOobject
+    (
+        "none",
+        tgtMesh_.time().timeName(),
+        cloud::prefix/cloudName,
+        tgtMesh_,
+        IOobjectOption::NO_READ,
+        IOobjectOption::NO_WRITE,
+        IOobjectOption::NO_REGISTER
+    );
+
+    //bool reconstruct = false;
+
+    if (verbose_ && fieldNames.size())
+    {
+        Info<< "    Distributing lagrangian "
+            << Container::typeName << "s\n" << nl;
+    }
 
-    label nFields = 0;
     for (const word& objectName : fieldNames)
     {
-        if (!nFields)
-        {
-            // Performing an all-to-one (reconstruct)?
-            reconstruct =
-                returnReduceAnd(!map.constructSize() || Pstream::master());
-        }
-
         if (verbose_)
         {
-            if (!nFields)
-            {
-                Info<< "    Distributing lagrangian "
-                    << fieldType::typeName << "s\n" << nl;
-            }
             Info<< "        " <<  objectName << nl;
         }
-        ++nFields;
 
-        // Read if present
-        CompactIOField<Field<Type>, Type> field
-        (
-            IOobject
-            (
-                objectName,
-                srcMesh_.time().timeName(),
-                cloud::prefix/cloudName,
-                srcMesh_,
-                IOobject::READ_IF_PRESENT,
-                IOobject::NO_WRITE,
-                IOobject::NO_REGISTER
-            ),
-            label(0)
-        );
+        srcIOobject.resetHeader(objectName);
+        tgtIOobject.resetHeader(objectName);
+
+        // Read if present (ie, haveCloud means readOnProc)
+        Container field(srcIOobject, haveCloud);
 
         // Distribute
         map.distribute(field);
 
+        const bool writeOnProc = field.size();
+
         // Write
-        const IOobject fieldIO
-        (
-            objectName,
-            tgtMesh_.time().timeName(),
-            cloud::prefix/cloudName,
-            tgtMesh_,
-            IOobject::NO_READ,
-            IOobject::NO_WRITE,
-            IOobject::NO_REGISTER
-        );
+        Container(tgtIOobject, std::move(field)).write(writeOnProc);
 
-        if (field.size())
-        {
-            CompactIOField<Field<Type>, Type>
-            (
-                fieldIO,
-                std::move(field)
-            ).write();
-        }
-        else if (!reconstruct)
-        {
-            // When running with -overwrite it should also delete the old
-            // files. Below works but is not optimal.
-
-            const fileName fldName(fieldIO.objectPath());
-            Foam::rm(fldName);
-        }
+        //if (!writeOnProc && !reconstruct)
+        //{
+        //    // When running with -overwrite it should also delete the old
+        //    // files. Below works but is not optimal.
+        //
+        //    Foam::rm(tgtIOobject.objectPath());
+        //}
     }
 
-    if (nFields && verbose_) Info<< endl;
-    return nFields;
+    if (verbose_ && fieldNames.size()) Info<< endl;
+    return fieldNames.size();
 }
 
 
@@ -265,12 +250,11 @@ template<class Container>
 Foam::label Foam::parLagrangianDistributor::readFields
 (
     const passivePositionParticleCloud& cloud,
+    const bool haveCloud,
     const IOobjectList& objects,
     const wordRes& selectedFields
 )
 {
-    const word fieldClassName(Container::typeName);
-
     const wordList fieldNames
     (
         filterObjects<Container>
@@ -280,40 +264,40 @@ Foam::label Foam::parLagrangianDistributor::readFields
         )
     );
 
-    label nFields = 0;
+    // Read if present
+    IOobject readIO
+    (
+        "none",
+        cloud.time().timeName(),
+        cloud,
+        IOobjectOption::LAZY_READ,
+        IOobjectOption::NO_WRITE,
+        IOobjectOption::REGISTER
+    );
+
+    if (verbose_ && fieldNames.size())
+    {
+        Info<< "    Reading lagrangian "
+            << Container::typeName << "s\n" << nl;
+    }
+
     for (const word& objectName : fieldNames)
     {
         if (verbose_)
         {
-            if (!nFields)
-            {
-                Info<< "    Reading lagrangian "
-                    << Container::typeName << "s\n" << nl;
-            }
             Info<< "        " <<  objectName << nl;
         }
-        ++nFields;
 
-        // Read if present
-        Container* fieldPtr = new Container
-        (
-            IOobject
-            (
-                objectName,
-                cloud.time().timeName(),
-                cloud,
-                IOobject::LAZY_READ,
-                IOobject::NO_WRITE,
-                IOobject::REGISTER
-            ),
-            label(0)
-        );
+        readIO.resetHeader(objectName);
+
+        // Read if present (ie, haveCloud means readOnProc)
+        Container* fieldPtr = new Container(readIO, haveCloud);
 
         fieldPtr->store();
     }
 
-    if (nFields && verbose_) Info<< endl;
-    return nFields;
+    if (verbose_ && fieldNames.size()) Info<< endl;
+    return fieldNames.size();
 }
 
 
@@ -324,58 +308,55 @@ Foam::label Foam::parLagrangianDistributor::distributeStoredFields
     passivePositionParticleCloud& cloud
 ) const
 {
-    bool reconstruct = false;
-    label nFields = 0;
+    // Parallel-consistent
+    UPtrList<Container> fields(cloud.sorted<Container>());
 
-    for (Container& field : cloud.sorted<Container>())
+    IOobject writeIO
+    (
+        "none",
+        tgtMesh_.time().timeName(),
+        cloud::prefix/cloud.name(),
+        tgtMesh_,
+        IOobjectOption::NO_READ,
+        IOobjectOption::NO_WRITE,
+        IOobjectOption::NO_REGISTER
+    );
+
+    //bool reconstruct = false;
+
+    if (verbose_ && fields.size())
     {
-        if (!nFields)
-        {
-            // Performing an all-to-one (reconstruct)?
-            reconstruct =
-                returnReduceAnd(!map.constructSize() || Pstream::master());
-        }
+        Info<< "    Distributing lagrangian "
+            << Container::typeName << "s\n" << nl;
+    }
 
+    for (Container& field : fields)
+    {
         if (verbose_)
         {
-            if (!nFields)
-            {
-                Info<< "    Distributing lagrangian "
-                    << Container::typeName << "s\n" << nl;
-            }
             Info<< "        " <<  field.name() << nl;
         }
-        ++nFields;
 
         map.distribute(field);
 
-        const IOobject fieldIO
-        (
-            field.name(),
-            tgtMesh_.time().timeName(),
-            cloud::prefix/cloud.name(),
-            tgtMesh_,
-            IOobject::NO_READ,
-            IOobject::NO_WRITE,
-            IOobject::NO_REGISTER
-        );
+        writeIO.resetHeader(field.name());
 
-        if (field.size())
-        {
-            Container(fieldIO, std::move(field)).write();
-        }
-        else if (!reconstruct)
-        {
-            // When running with -overwrite it should also delete the old
-            // files. Below works but is not optimal.
+        const bool writeOnProc = field.size();
 
-            const fileName fldName(fieldIO.objectPath());
-            Foam::rm(fldName);
-        }
+        // Write
+        Container(writeIO, std::move(field)).write(writeOnProc);
+
+        //if (!writeOnProc && !reconstruct)
+        //{
+        //    // When running with -overwrite it should also delete the old
+        //    // files. Below works but is not optimal.
+        //
+        //    Foam::rm(writeIO.objectPath());
+        //}
     }
 
-    if (nFields && verbose_) Info<< endl;
-    return nFields;
+    if (verbose_ && fields.size()) Info<< endl;
+    return fields.size();
 }
 
 
diff --git a/applications/utilities/parallelProcessing/redistributePar/redistributeLagrangian.H b/applications/utilities/parallelProcessing/redistributePar/redistributeLagrangian.H
index 99f08b3c29..bb118558f5 100644
--- a/applications/utilities/parallelProcessing/redistributePar/redistributeLagrangian.H
+++ b/applications/utilities/parallelProcessing/redistributePar/redistributeLagrangian.H
@@ -6,7 +6,7 @@
      \\/     M anipulation  |
 -------------------------------------------------------------------------------
     Copyright (C) 2011-2017 OpenFOAM Foundation
-    Copyright (C) 2015-2022 OpenCFD Ltd.
+    Copyright (C) 2015-2023 OpenCFD Ltd.
 -------------------------------------------------------------------------------
 License
     This file is part of OpenFOAM.
@@ -48,6 +48,7 @@ readLagrangian
 (
     const fvMesh& mesh,
     const wordList& cloudNames,
+    const boolUList& haveClouds,
     const wordRes& selectedFields
 )
 {
@@ -78,11 +79,10 @@ readLagrangian
 
         IOobjectList cloudObjs(clouds[i], clouds[i].time().timeName());
 
-        //Pout<< "Found cloud objects:" << cloudObjs.names() << endl;
-
         parLagrangianDistributor::readAllFields
         (
             clouds[i],
+            haveClouds[i],
             cloudObjs,
             selectedFields
         );
@@ -101,11 +101,19 @@ readLagrangian
 )
 {
     wordList cloudNames;
+    boolList haveClouds;
     List<wordList> fieldNames;
-    // Find all cloudNames on all processors
-    parLagrangianDistributor::findClouds(mesh, cloudNames, fieldNames);
 
-    return readLagrangian(mesh, cloudNames, selectedFields);
+    // Find all cloudNames on all processors
+    parLagrangianDistributor::findClouds
+    (
+        mesh,
+        cloudNames,
+        haveClouds,
+        fieldNames
+    );
+
+    return readLagrangian(mesh, cloudNames, haveClouds, selectedFields);
 }
 
 
@@ -121,9 +129,17 @@ void reconstructLagrangian
     // Clouds (note: might not be present on all processors)
 
     wordList cloudNames;
+    boolList haveClouds;
     List<wordList> fieldNames;
+
     // Find all cloudNames on all processors
-    parLagrangianDistributor::findClouds(mesh, cloudNames, fieldNames);
+    parLagrangianDistributor::findClouds
+    (
+        mesh,
+        cloudNames,
+        haveClouds,
+        fieldNames
+    );
 
     if (cloudNames.empty())
     {
@@ -142,13 +158,16 @@ void reconstructLagrangian
                 baseMesh,
                 mesh.nCells(),      // range of cell indices in clouds
                 distMap
+                //writeHandler        // Which processors to write
             )
         );
     }
     const auto& distributor = *distributorPtr;
 
-    for (const word& cloudName : cloudNames)
+    forAll(cloudNames, cloudi)
     {
+        const word& cloudName = cloudNames[cloudi];
+
         Info<< "Reconstructing lagrangian fields for cloud "
             << cloudName << nl << endl;
 
@@ -169,6 +188,7 @@ void reconstructLagrangian
         (
             lagrangianMapPtr(),
             cloudName,
+            haveClouds[cloudi],
             cloudObjs,
             selectedFields
         );

From aec4ba30a3e8ca996401fb759db2ad6a1d4bea09 Mon Sep 17 00:00:00 2001
From: mattijs <mattijs>
Date: Thu, 24 Nov 2022 09:00:00 +0000
Subject: [PATCH 02/12] ENH: handle watching included files

---
 .../includeEntry/includeEntry.C               | 51 ++++++++++++-------
 .../includeEtcEntry/includeEtcEntry.C         | 32 +++++++++++-
 .../masterUncollatedFileOperation.C           | 50 +++++++++++-------
 3 files changed, 95 insertions(+), 38 deletions(-)

diff --git a/src/OpenFOAM/db/dictionary/functionEntries/includeEntry/includeEntry.C b/src/OpenFOAM/db/dictionary/functionEntries/includeEntry/includeEntry.C
index 78620c7da3..96db436c4a 100644
--- a/src/OpenFOAM/db/dictionary/functionEntries/includeEntry/includeEntry.C
+++ b/src/OpenFOAM/db/dictionary/functionEntries/includeEntry/includeEntry.C
@@ -6,7 +6,7 @@
      \\/     M anipulation  |
 -------------------------------------------------------------------------------
     Copyright (C) 2011-2017 OpenFOAM Foundation
-    Copyright (C) 2018-2021 OpenCFD Ltd.
+    Copyright (C) 2018-2022 OpenCFD Ltd.
 -------------------------------------------------------------------------------
 License
     This file is part of OpenFOAM.
@@ -27,12 +27,13 @@ License
 \*---------------------------------------------------------------------------*/
 
 #include "includeEntry.H"
-#include "addToMemberFunctionSelectionTable.H"
 #include "stringOps.H"
 #include "IFstream.H"
 #include "IOstreams.H"
-#include "Time.H"
+#include "UPstream.H"
 #include "fileOperation.H"
+#include "regIOobject.H"
+#include "addToMemberFunctionSelectionTable.H"
 
 // * * * * * * * * * * * * * * Static Data Members * * * * * * * * * * * * * //
 
@@ -133,6 +134,15 @@ bool Foam::functionEntries::includeEntry::execute
     Istream& is
 )
 {
+    const auto* rioPtr = isA<regIOobject>(parentDict.topDict());
+
+    const label oldComm
+    (
+        rioPtr && rioPtr->global()
+      ? fileHandler().comm(UPstream::worldComm)
+      : fileHandler().comm()
+    );
+
     const fileName rawName(is);
     const fileName fName(resolveFile(is.name().path(), rawName, parentDict));
 
@@ -148,20 +158,19 @@ bool Foam::functionEntries::includeEntry::execute
         }
 
         // Add watch on included file
-        const dictionary& top = parentDict.topDict();
-        if (isA<regIOobject>(top))
+        if (rioPtr)
         {
-            regIOobject& rio = const_cast<regIOobject&>
-            (
-                dynamic_cast<const regIOobject&>(top)
-            );
-            rio.addWatch(fName);
+            const_cast<regIOobject&>(*rioPtr).addWatch(fName);
         }
 
         parentDict.read(ifs);
+
+        fileHandler().comm(oldComm);
         return true;
     }
 
+    fileHandler().comm(oldComm);
+
     if (!mandatory)
     {
         return true; // Never fails if optional
@@ -185,6 +194,15 @@ bool Foam::functionEntries::includeEntry::execute
     Istream& is
 )
 {
+    const auto* rioPtr = isA<regIOobject>(parentDict.topDict());
+
+    const label oldComm
+    (
+        rioPtr && rioPtr->global()
+      ? fileHandler().comm(UPstream::worldComm)
+      : fileHandler().comm()
+    );
+
     const fileName rawName(is);
     const fileName fName(resolveFile(is.name().path(), rawName, parentDict));
 
@@ -200,20 +218,19 @@ bool Foam::functionEntries::includeEntry::execute
         }
 
         // Add watch on included file
-        const dictionary& top = parentDict.topDict();
-        if (isA<regIOobject>(top))
+        if (rioPtr)
         {
-            regIOobject& rio = const_cast<regIOobject&>
-            (
-                dynamic_cast<const regIOobject&>(top)
-            );
-            rio.addWatch(fName);
+            const_cast<regIOobject&>(*rioPtr).addWatch(fName);
         }
 
         entry.read(parentDict, ifs);
+
+        fileHandler().comm(oldComm);
         return true;
     }
 
+    fileHandler().comm(oldComm);
+
     if (!mandatory)
     {
         return true; // Never fails if optional
diff --git a/src/OpenFOAM/db/dictionary/functionEntries/includeEtcEntry/includeEtcEntry.C b/src/OpenFOAM/db/dictionary/functionEntries/includeEtcEntry/includeEtcEntry.C
index 9c55a6f0c9..504673640d 100644
--- a/src/OpenFOAM/db/dictionary/functionEntries/includeEtcEntry/includeEtcEntry.C
+++ b/src/OpenFOAM/db/dictionary/functionEntries/includeEtcEntry/includeEtcEntry.C
@@ -6,7 +6,7 @@
      \\/     M anipulation  |
 -------------------------------------------------------------------------------
     Copyright (C) 2015-2017 OpenFOAM Foundation
-    Copyright (C) 2019-2021 OpenCFD Ltd.
+    Copyright (C) 2019-2022 OpenCFD Ltd.
 -------------------------------------------------------------------------------
 License
     This file is part of OpenFOAM.
@@ -27,12 +27,14 @@ License
 \*---------------------------------------------------------------------------*/
 
 #include "includeEtcEntry.H"
-#include "addToMemberFunctionSelectionTable.H"
 #include "etcFiles.H"
 #include "stringOps.H"
 #include "IFstream.H"
 #include "IOstreams.H"
+#include "UPstream.H"
 #include "fileOperation.H"
+#include "regIOobject.H"
+#include "addToMemberFunctionSelectionTable.H"
 
 // * * * * * * * * * * * * * * Static Data Members * * * * * * * * * * * * * //
 
@@ -113,6 +115,15 @@ bool Foam::functionEntries::includeEtcEntry::execute
     Istream& is
 )
 {
+    const regIOobject* rioPtr = isA<regIOobject>(parentDict.topDict());
+
+    const label oldComm
+    (
+        rioPtr && rioPtr->global()
+      ? fileHandler().comm(UPstream::worldComm)
+      : fileHandler().comm()
+    );
+
     const fileName rawName(is);
     const fileName fName(resolveEtcFile(rawName, parentDict));
 
@@ -127,9 +138,13 @@ bool Foam::functionEntries::includeEtcEntry::execute
             Info<< fName << nl;
         }
         parentDict.read(ifs);
+
+        fileHandler().comm(oldComm);
         return true;
     }
 
+    fileHandler().comm(oldComm);
+
     if (!mandatory)
     {
         return true; // Never fails if optional
@@ -153,6 +168,15 @@ bool Foam::functionEntries::includeEtcEntry::execute
     Istream& is
 )
 {
+    const regIOobject* rioPtr = isA<regIOobject>(parentDict.topDict());
+
+    const label oldComm
+    (
+        rioPtr && rioPtr->global()
+      ? fileHandler().comm(UPstream::worldComm)
+      : fileHandler().comm()
+    );
+
     const fileName rawName(is);
     const fileName fName(resolveEtcFile(rawName, parentDict));
 
@@ -167,9 +191,13 @@ bool Foam::functionEntries::includeEtcEntry::execute
             Info<< fName << nl;
         }
         entry.read(parentDict, ifs);
+
+        fileHandler().comm(oldComm);
         return true;
     }
 
+    fileHandler().comm(oldComm);
+
     if (!mandatory)
     {
         return true; // Never fails if optional
diff --git a/src/OpenFOAM/global/fileOperations/masterUncollatedFileOperation/masterUncollatedFileOperation.C b/src/OpenFOAM/global/fileOperations/masterUncollatedFileOperation/masterUncollatedFileOperation.C
index f7781c32f2..f48b12c8d6 100644
--- a/src/OpenFOAM/global/fileOperations/masterUncollatedFileOperation/masterUncollatedFileOperation.C
+++ b/src/OpenFOAM/global/fileOperations/masterUncollatedFileOperation/masterUncollatedFileOperation.C
@@ -2766,30 +2766,42 @@ void Foam::fileOperations::masterUncollatedFileOperation::addWatches
 {
     const labelList& watchIndices = rio.watchIndices();
 
+    // Do on master and distribute effect to subprocs such that after
+    // all have consistent numbering & files
+
     DynamicList<label> newWatchIndices;
-    labelHashSet removedWatches(watchIndices);
-
-    for (const fileName& f : files)
+    if (UPstream::master())
     {
-        const label index = findWatch(watchIndices, f);
+        // Switch off comms inside findWatch/addWatch etc.
+        const bool oldParRun = UPstream::parRun(false);
 
-        if (index == -1)
-        {
-            newWatchIndices.append(addWatch(f));
-        }
-        else
-        {
-            // Existing watch
-            newWatchIndices.append(watchIndices[index]);
-            removedWatches.erase(index);
-        }
-    }
+        labelHashSet removedWatches(watchIndices);
 
-    // Remove any unused watches
-    for (const label index : removedWatches)
-    {
-        removeWatch(watchIndices[index]);
+        for (const fileName& f : files)
+        {
+            const label index = findWatch(watchIndices, f);
+
+            if (index == -1)
+            {
+                newWatchIndices.push_back(addWatch(f));
+            }
+            else
+            {
+                // Existing watch
+                newWatchIndices.push_back(watchIndices[index]);
+                removedWatches.erase(index);
+            }
+        }
+
+        // Remove any unused watches
+        for (const label index : removedWatches)
+        {
+            removeWatch(watchIndices[index]);
+        }
+
+        UPstream::parRun(oldParRun);
     }
+    Pstream::broadcast(newWatchIndices);
 
     rio.watchIndices() = newWatchIndices;
 }

From 65523e9d24242a7c72ec4425bd17731c39b4b60f Mon Sep 17 00:00:00 2001
From: mattijs <mattijs>
Date: Wed, 25 Jan 2023 13:21:44 +0000
Subject: [PATCH 03/12] ENH: add delayed handling of file watches to
 regIOobject

- this delay the communication of file watches, which helps avoid
  communication deadlocks with master-only reading.

Co-authored-by: Mark Olesen <>
---
 src/OpenFOAM/db/regIOobject/regIOobject.C     | 103 +++++++++++++-----
 src/OpenFOAM/db/regIOobject/regIOobject.H     |  12 +-
 src/OpenFOAM/db/regIOobject/regIOobjectRead.C |   6 +-
 .../db/regIOobject/regIOobjectWrite.C         |   2 +-
 4 files changed, 87 insertions(+), 36 deletions(-)

diff --git a/src/OpenFOAM/db/regIOobject/regIOobject.C b/src/OpenFOAM/db/regIOobject/regIOobject.C
index 25711b8729..932bab6156 100644
--- a/src/OpenFOAM/db/regIOobject/regIOobject.C
+++ b/src/OpenFOAM/db/regIOobject/regIOobject.C
@@ -52,7 +52,6 @@ Foam::regIOobject::regIOobject(const IOobject& io, const bool isTimeObject)
     IOobject(io),
     registered_(false),
     ownedByRegistry_(false),
-    watchIndices_(),
     eventNo_(isTimeObject ? 0 : db().getEvent()), // No event for top-level Time
     metaDataPtr_(nullptr),
     isPtr_(nullptr)
@@ -70,8 +69,9 @@ Foam::regIOobject::regIOobject(const regIOobject& rio)
     IOobject(rio),
     registered_(false),
     ownedByRegistry_(false),
-    watchIndices_(rio.watchIndices_),
     eventNo_(db().getEvent()),
+    watchFiles_(rio.watchFiles_),
+    watchIndices_(rio.watchIndices_),
     metaDataPtr_(rio.metaDataPtr_.clone()),
     isPtr_(nullptr)
 {
@@ -84,7 +84,6 @@ Foam::regIOobject::regIOobject(const regIOobject& rio, bool registerCopy)
     IOobject(rio),
     registered_(false),
     ownedByRegistry_(false),
-    watchIndices_(),
     eventNo_(db().getEvent()),
     metaDataPtr_(rio.metaDataPtr_.clone()),
     isPtr_(nullptr)
@@ -111,7 +110,6 @@ Foam::regIOobject::regIOobject
     IOobject(newName, rio.instance(), rio.local(), rio.db()),
     registered_(false),
     ownedByRegistry_(false),
-    watchIndices_(),
     eventNo_(db().getEvent()),
     metaDataPtr_(rio.metaDataPtr_.clone()),
     isPtr_(nullptr)
@@ -135,7 +133,6 @@ Foam::regIOobject::regIOobject
     IOobject(io),
     registered_(false),
     ownedByRegistry_(false),
-    watchIndices_(),
     eventNo_(db().getEvent()),
     metaDataPtr_(rio.metaDataPtr_.clone()),
     isPtr_(nullptr)
@@ -234,6 +231,7 @@ bool Foam::regIOobject::checkOut()
         fileHandler().removeWatch(watchIndices_[i]);
     }
     watchIndices_.clear();
+    watchFiles_.clear();
 
     if (registered_)
     {
@@ -253,17 +251,23 @@ Foam::label Foam::regIOobject::addWatch(const fileName& f)
     if
     (
         registered_
-     && readOpt() == IOobject::MUST_READ_IF_MODIFIED
+     && readOpt() == IOobjectOption::READ_MODIFIED
      && time().runTimeModifiable()
     )
     {
-        index = fileHandler().findWatch(watchIndices_, f);
+        //- 1. Directly add to fileHandler
+        //index = fileHandler().findWatch(watchIndices_, f);
+        //
+        //if (index == -1)
+        //{
+        //    index = watchIndices_.size();
+        //    watchIndices_.push_back(fileHandler().addWatch(f));
+        //}
 
-        if (index == -1)
-        {
-            index = watchIndices_.size();
-            watchIndices_.append(fileHandler().addWatch(f));
-        }
+        //- 2. Delay adding; add to list and handle in addWatch() later on
+        //     Note: what do we return?
+        index = watchFiles_.size();
+        watchFiles_.push_back(f);
     }
 
     return index;
@@ -275,7 +279,7 @@ void Foam::regIOobject::addWatch()
     if
     (
         registered_
-     && readOpt() == IOobject::MUST_READ_IF_MODIFIED
+     && readOpt() == IOobjectOption::READ_MODIFIED
      && time().runTimeModifiable()
     )
     {
@@ -307,37 +311,80 @@ void Foam::regIOobject::addWatch()
             )
         );
 
-        if (masterOnly && Pstream::parRun())
+        if (masterOnly && UPstream::parRun())
         {
-            // Get master watched files
-            fileNameList watchFiles;
-            if (Pstream::master())
+            // Get all files watched on master, and broadcast at once
+            fileNameList filesToWatch;
+            if (UPstream::master())
             {
-                watchFiles.resize(watchIndices_.size());
+                const bool oldParRun = UPstream::parRun(false);
+
+                filesToWatch.resize(watchIndices_.size());
                 forAll(watchIndices_, i)
                 {
-                    watchFiles[i] = fileHandler().getFile(watchIndices_[i]);
+                    filesToWatch[i] = fileHandler().getFile(watchIndices_[i]);
                 }
-            }
-            Pstream::broadcast(watchFiles);
 
-            if (!Pstream::master())
+                UPstream::parRun(oldParRun);
+            }
+            Pstream::broadcast(filesToWatch);
+
+
+            // Add master files in same order
+            if (!UPstream::master())
             {
-                // unregister current ones
+                const bool oldParRun = UPstream::parRun(false);
+
+                // Unregister current watched indices
                 forAllReverse(watchIndices_, i)
                 {
                     fileHandler().removeWatch(watchIndices_[i]);
                 }
 
+                // Insert the ones from master, in master order
                 watchIndices_.clear();
-                forAll(watchFiles, i)
+                for (const auto& file : filesToWatch)
                 {
-                    watchIndices_.append(fileHandler().addWatch(watchFiles[i]));
+                    watchIndices_.push_back(fileHandler().addWatch(file));
                 }
-            }
-        }
 
-        watchIndices_.append(fileHandler().addWatch(f));
+                UPstream::parRun(oldParRun);
+            }
+
+
+            // Files that were explicitly added via addWatch(const fileName&)
+            // (e.g. through #include)
+            for (const auto& file : watchFiles_)
+            {
+                watchIndices_.push_back(fileHandler().addWatch(file));
+            }
+
+            // Append the local file
+            watchIndices_.push_back(fileHandler().addWatch(f));
+        }
+        else
+        {
+            DynamicList<fileName> filesToWatch
+            (
+                watchIndices_.size()+watchFiles_.size()+1
+            );
+
+            // Get existing watched files from fileHandler
+            for (const label index : watchIndices_)
+            {
+                filesToWatch.push_back(fileHandler().getFile(index));
+            }
+
+            // The files explicitly added from addWatch(const fileName&)
+            // (e.g. through #include)
+            filesToWatch.push_back(std::move(watchFiles_));
+
+            // The local file
+            filesToWatch.push_back(f);
+
+            // Re-do all watches
+            fileHandler().addWatches(*this, filesToWatch);
+        }
     }
 }
 
diff --git a/src/OpenFOAM/db/regIOobject/regIOobject.H b/src/OpenFOAM/db/regIOobject/regIOobject.H
index 48553c2400..0d89cc9ff1 100644
--- a/src/OpenFOAM/db/regIOobject/regIOobject.H
+++ b/src/OpenFOAM/db/regIOobject/regIOobject.H
@@ -43,6 +43,7 @@ SourceFiles
 
 #include "IOobject.H"
 #include "OSspecific.H"
+#include "DynamicList.H"
 
 // * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
 
@@ -94,12 +95,15 @@ private:
         //- Is this object owned by the registry
         bool ownedByRegistry_;
 
-        //- List of modification watch indices
-        mutable labelList watchIndices_;
-
-        //- eventNo of last update
+        //- The eventNo of last update
         label eventNo_;
 
+        //- List of additional files to watch
+        mutable DynamicList<fileName> watchFiles_;
+
+        //- List of modification watch indices
+        mutable DynamicList<label> watchIndices_;
+
         //- Dictionary for any meta-data
         autoPtr<dictionary> metaDataPtr_;
 
diff --git a/src/OpenFOAM/db/regIOobject/regIOobjectRead.C b/src/OpenFOAM/db/regIOobject/regIOobjectRead.C
index 7f47d0f818..84f8b5d6ad 100644
--- a/src/OpenFOAM/db/regIOobject/regIOobjectRead.C
+++ b/src/OpenFOAM/db/regIOobject/regIOobjectRead.C
@@ -58,11 +58,11 @@ bool Foam::regIOobject::readHeaderOk
     {
         if (masterOnly)
         {
-            if (Pstream::master())
+            if (UPstream::master())
             {
-                const bool oldParRun = Pstream::parRun(false);
+                const bool oldParRun = UPstream::parRun(false);
                 isHeaderOk = headerOk();
-                Pstream::parRun(oldParRun);
+                UPstream::parRun(oldParRun);
             }
             Pstream::broadcast(isHeaderOk);
         }
diff --git a/src/OpenFOAM/db/regIOobject/regIOobjectWrite.C b/src/OpenFOAM/db/regIOobject/regIOobjectWrite.C
index 44dcdcd449..86d0642aad 100644
--- a/src/OpenFOAM/db/regIOobject/regIOobjectWrite.C
+++ b/src/OpenFOAM/db/regIOobject/regIOobjectWrite.C
@@ -105,7 +105,7 @@ bool Foam::regIOobject::writeObject
     );
 
     bool osGood = false;
-    if (!masterOnly || Pstream::master())
+    if (!masterOnly || UPstream::master())
     {
         osGood = fileHandler().writeObject(*this, streamOpt, writeOnProc);
     }

From 030a467a23f53c22fc4c36b02cffd3efdce0cbf1 Mon Sep 17 00:00:00 2001
From: mattijs <mattijs>
Date: Thu, 24 Nov 2022 09:00:00 +0000
Subject: [PATCH 04/12] ENH: broadcast dynamicCode, create missing processor
 directories

- code is compiled dynamically on the master node.
  In the normal (non-distributed) case, simply poll the NFS
  to see when it appears on the sub-procs.

  For a case with distributed roots, first broadcast it (via MPI)
  to the IO master nodes and then poll afterwards.

- on startup, detect and create missing processorXXX/ subdirectories
  on distributed filesystems
---
 .../functionEntries/codeStream/codeStream.C   |  12 +-
 .../db/dynamicLibrary/codedBase/codedBase.C   |  16 +-
 src/OpenFOAM/global/argList/argList.C         | 138 +++++++++++++++++-
 3 files changed, 158 insertions(+), 8 deletions(-)

diff --git a/src/OpenFOAM/db/dictionary/functionEntries/codeStream/codeStream.C b/src/OpenFOAM/db/dictionary/functionEntries/codeStream/codeStream.C
index 9384284164..07f501ab93 100644
--- a/src/OpenFOAM/db/dictionary/functionEntries/codeStream/codeStream.C
+++ b/src/OpenFOAM/db/dictionary/functionEntries/codeStream/codeStream.C
@@ -196,7 +196,17 @@ Foam::functionEntries::codeStream::getFunction
     //  (flag set by regIOobject::read, baseIOdictionary constructor)
     if (!masterOnly && returnReduceOr(lib == nullptr))
     {
-        // Broadcast distributed...
+        // Broadcast to distributed masters
+        if (fileHandler().distributed())
+        {
+            fileHandler().broadcastCopy
+            (
+                UPstream::worldComm,
+                UPstream::master(fileHandler().comm()),
+                libPath,
+                libPath
+            );
+        }
 
         dynamicCode::waitForFile(libPath, context.dict());
     }
diff --git a/src/OpenFOAM/db/dynamicLibrary/codedBase/codedBase.C b/src/OpenFOAM/db/dynamicLibrary/codedBase/codedBase.C
index 7d57638c0d..b2fc8c5704 100644
--- a/src/OpenFOAM/db/dynamicLibrary/codedBase/codedBase.C
+++ b/src/OpenFOAM/db/dynamicLibrary/codedBase/codedBase.C
@@ -254,9 +254,21 @@ void Foam::codedBase::createLibrary
         UPstream::barrier(UPstream::worldComm);
     }
 
-    // Broadcast distributed...
+    const fileName libPath = dynCode.libPath();
 
-    dynamicCode::waitForFile(dynCode.libPath(), context.dict());
+    // Broadcast to distributed masters
+    if (fileHandler().distributed())
+    {
+        fileHandler().broadcastCopy
+        (
+            UPstream::worldComm,
+            UPstream::master(fileHandler().comm()),
+            libPath,
+            libPath
+        );
+    }
+
+    dynamicCode::waitForFile(libPath, context.dict());
 }
 
 
diff --git a/src/OpenFOAM/global/argList/argList.C b/src/OpenFOAM/global/argList/argList.C
index 6ded4cec85..91bdfd6d96 100644
--- a/src/OpenFOAM/global/argList/argList.C
+++ b/src/OpenFOAM/global/argList/argList.C
@@ -1763,16 +1763,144 @@ void Foam::argList::parse
     }
 
     // If needed, adjust fileHandler for distributed roots
-    if (runControl_.distributed())
+    if (runControl_.distributed() && fileOperation::fileHandlerPtr_)
     {
-        if (fileOperation::fileHandlerPtr_)
+        fileOperation::fileHandlerPtr_->distributed(true);
+
+        const labelList& ranks = fileHandler().ioRanks();
+
+        if (runControl_.parRun() && ranks.size())
         {
-            fileOperation::fileHandlerPtr_->distributed(true);
+            // Detect processor directories both on local proc and on
+            // (world) master proc. If the local proc doesn't have them
+            // but the master has it will attempt to copy them.
+
+            // Expected local directory name
+            const fileName procDir
+            (
+                rootPath_
+              / globalCase_
+              / ("processor" + Foam::name(UPstream::myProcNo()))
+            );
+
+            // Try and find my local directory using the fileHandler. This
+            // will check the local disk on the IO rank
+            // (since running distributed)
+            fileNameList pathDirs(UPstream::nProcs());
+            auto& pathDir = pathDirs[UPstream::myProcNo()];
+            pathDir = fileHandler().filePath(procDir, false);
+
+            if (returnReduceOr(pathDir.empty()))
+            {
+                // There is at least one processor that cannot find
+                // the processor directory. Look for it on the master.
+                // E.g. decomposed into 4 processors, two roots:
+                //      processors4_0-1/
+                //      processors4_2-3/
+                // So:
+                // - processor0 reads the same disk as processor0
+                // - processor2 needs the whole directory sent over
+                // - processor3 reads the same disk as processor2
+                if (UPstream::master() && bannerEnabled())
+                {
+                    Info<< "I/O    :"
+                        << " distributed - copying missing directories"
+                        << nl;
+                }
+
+                // Collect all wanted directories (or empty). Note: could
+                // just collect missing ones ...
+
+                Pstream::gatherList(pathDirs);
+                fileName masterRootPath(rootPath_);
+                Pstream::broadcast(masterRootPath);
+
+                List<fileNameList> rankToDirs(UPstream::nProcs());
+                if (UPstream::master())
+                {
+                    const bool oldParRun = Pstream::parRun(false);
+
+                    label rank = 0;
+                    for (label proci = 1; proci < pathDirs.size(); ++proci)
+                    {
+                        if (ranks.contains(proci))
+                        {
+                            rank = proci;
+                        }
+
+                        if (pathDirs[proci].empty())
+                        {
+                            // Synthesise corresponding name on the master
+                            // processor
+                            const fileName procDir
+                            (
+                                rootPath_
+                              / globalCase_
+                              / ("processor" + Foam::name(proci))
+                            );
+                            const fileName foundDir
+                            (
+                                fileHandler().filePath(procDir, false)
+                            );
+
+                            if
+                            (
+                                !foundDir.empty()
+                             && !rankToDirs[rank].contains(foundDir)
+                            )
+                            {
+                                rankToDirs[rank].push_back(foundDir);
+                            }
+                        }
+                    }
+
+                    UPstream::parRun(oldParRun);
+                }
+                Pstream::broadcast(rankToDirs);
+
+                // Copy missing directories on all the IOranks.
+                // Note: instead of passing 'writeOnProc' flag we could create
+                // communicator just between master and IOrank, but that is
+                // also expensive.
+
+                forAll(rankToDirs, proci)
+                {
+                    // Am I the reponsible IOrank for this processor
+                    const bool amIO = (UPstream::myProcNo() == proci);
+
+                    // Construct equivalent directory on proci
+                    for (const auto& srcDir : rankToDirs[proci])
+                    {
+                        const fileName tgtDir
+                        (
+                            rootPath_
+                          / srcDir.relative(masterRootPath)
+                        );
+
+                        if (amIO)
+                        {
+                            // I am the IO rank
+                            Pout<< "On rank " << proci << nl
+                                << "    copying : " << srcDir << nl
+                                << "    to      : " << tgtDir << endl;
+                        }
+
+                        fileHandler().broadcastCopy
+                        (
+                            UPstream::worldComm,
+                            amIO,
+                            tgtDir,
+                            tgtDir
+                        );
+                    }
+                }
+            }
         }
     }
 
+
     // Keep/discard sub-process host/root information for reporting:
-    if (Pstream::master() && runControl_.parRun())
+    if (UPstream::master() && runControl_.parRun())
     {
         if (!writeHostsSwitch)
         {
@@ -1785,7 +1913,7 @@ void Foam::argList::parse
         }
     }
 
-    if (Pstream::master() && bannerEnabled())
+    if (UPstream::master() && bannerEnabled())
     {
         Info<< "Case   : " << (rootPath_/globalCase_).c_str() << nl
             << "nProcs : " << nProcs << nl;

From c880efb4a6b5bfea891ee38c5bcab6c6ee82fcae Mon Sep 17 00:00:00 2001
From: mattijs <mattijs>
Date: Thu, 24 Nov 2022 09:00:00 +0000
Subject: [PATCH 05/12] ENH: improve fileHandler support for redistributePar

- use file handlers to manage where meshes/fields are to be read and
  written.
---
 .../redistributePar/loadOrCreateMesh.C        |  172 ++-
 .../redistributePar/loadOrCreateMesh.H        |   17 +-
 .../parFaFieldDistributorCache.C              |   29 +-
 .../parFaFieldDistributorCache.H              |    1 +
 .../redistributePar/redistributePar.C         | 1197 +++++++++++------
 5 files changed, 963 insertions(+), 453 deletions(-)

diff --git a/applications/utilities/parallelProcessing/redistributePar/loadOrCreateMesh.C b/applications/utilities/parallelProcessing/redistributePar/loadOrCreateMesh.C
index 3e3e8a1ad1..cf0ef53c03 100644
--- a/applications/utilities/parallelProcessing/redistributePar/loadOrCreateMesh.C
+++ b/applications/utilities/parallelProcessing/redistributePar/loadOrCreateMesh.C
@@ -6,7 +6,7 @@
      \\/     M anipulation  |
 -------------------------------------------------------------------------------
     Copyright (C) 2012-2017 OpenFOAM Foundation
-    Copyright (C) 2015-2022 OpenCFD Ltd.
+    Copyright (C) 2015-2023 OpenCFD Ltd.
 -------------------------------------------------------------------------------
 License
     This file is part of OpenFOAM.
@@ -33,6 +33,101 @@ License
 
 // * * * * * * * * * * * * * * * Global Functions  * * * * * * * * * * * * * //
 
+bool Foam::checkFileExistence(const fileName& fName)
+{
+    // Trimmed-down version of lookupAndCacheProcessorsPath
+    // with Foam::exists() check. No caching.
+
+    // Check for two conditions:
+    // - file has to exist
+    // - if collated the entry has to exist inside the file
+
+    // Note: bypass fileOperation::filePath(IOobject&) since has problems
+    //       with going to a different number of processors
+    //       (in collated format). Use file-based searching instead
+
+    const auto& handler = Foam::fileHandler();
+
+    typedef fileOperation::procRangeType procRangeType;
+
+    fileName path, pDir, local;
+    procRangeType group;
+    label numProcs;
+    const label proci =
+        fileOperation::splitProcessorPath
+        (fName, path, pDir, local, group, numProcs);
+
+    bool found = false;
+
+    if (proci != -1)
+    {
+        // Read all directories to see any beginning with processor
+        // ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+        const fileNameList dirEntries
+        (
+            handler.readDir(path, fileName::Type::DIRECTORY)
+        );
+
+        // Extract info from processorN or processorsNN
+        // - highest processor number
+        // - directory+offset containing data for proci
+
+        // label nProcs = 0;
+        for (const fileName& dirN : dirEntries)
+        {
+            // Analyse directory name
+            fileName rp, rd, rl;
+            label rNum;
+            const label readProci =
+                fileOperation::splitProcessorPath
+                (dirN, rp, rd, rl, group, rNum);
+
+            if (proci == readProci)
+            {
+                // Found "processorN"
+                if (Foam::exists(path/dirN/local))
+                {
+                    found = true;
+                    break;
+                }
+            }
+            else if (rNum != -1)
+            {
+                // "processorsNN" or "processorsNN_start-end"
+                if (group.empty())
+                {
+                    // "processorsNN"
+                    if (proci < rNum && Foam::exists(path/dirN/local))
+                    {
+                        found = true;
+                        break;
+                    }
+                }
+                else if (group.contains(proci))
+                {
+                    // "processorsNN_start-end"
+                    // - save the local proc offset
+
+                    if (Foam::exists(path/dirN/local))
+                    {
+                        found = true;
+                        break;
+                    }
+                }
+            }
+        }
+    }
+
+    if (!found)
+    {
+        found = Foam::exists(fName);
+    }
+
+    return found;
+}
+
+
 Foam::boolList Foam::haveMeshFile
 (
     const Time& runTime,
@@ -41,18 +136,12 @@ Foam::boolList Foam::haveMeshFile
     const bool verbose
 )
 {
+    bool found = checkFileExistence(runTime.path()/meshPath/meshFile);
+
+    // Globally consistent information about who has a mesh
     boolList haveFileOnProc
     (
-        UPstream::listGatherValues<bool>
-        (
-            fileHandler().isFile
-            (
-                fileHandler().filePath
-                (
-                    runTime.path()/meshPath/meshFile
-                )
-            )
-        )
+        UPstream::allGatherValues<bool>(found, UPstream::worldComm)
     );
 
     if (verbose)
@@ -62,7 +151,6 @@ Foam::boolList Foam::haveMeshFile
             << "    " << flatOutput(haveFileOnProc) << nl << endl;
     }
 
-    Pstream::broadcast(haveFileOnProc);
     return haveFileOnProc;
 }
 
@@ -107,38 +195,52 @@ void Foam::removeProcAddressing(const polyMesh& mesh)
 }
 
 
-void Foam::removeEmptyDir(const fileName& path)
+void Foam::masterMeshInstance
+(
+    const IOobject& io,
+    fileName& facesInstance,
+    fileName& pointsInstance
+)
 {
-    // Remove directory: silent, emptyOnly
-    Foam::rmDir(path, true, true);
-}
+    const fileName meshSubDir
+    (
+        polyMesh::regionName(io.name()) / polyMesh::meshSubDir
+    );
 
-
-void Foam::removeEmptyDirs(const fileName& meshPath)
-{
-    // Delete resulting directory if empty
-    fileName path(meshPath);
-    path.clean();
-
-    // Do subdirectories
+    if (UPstream::master())
     {
-        const fileNameList dirs
+        const bool oldParRun = UPstream::parRun(false);
+        const label oldNumProcs = fileHandler().nProcs();
+        const int oldCache = fileOperation::cacheLevel(0);
+
+        facesInstance = io.time().findInstance
         (
-            Foam::readDir
-            (
-                path,
-                fileName::DIRECTORY,
-                false,                  // filterGz
-                false                   // followLink
-            )
+            meshSubDir,
+            "faces",
+            IOobjectOption::MUST_READ
         );
-        for (const auto& dir : dirs)
+        pointsInstance = io.time().findInstance
+        (
+            meshSubDir,
+            "points",
+            IOobjectOption::MUST_READ
+        );
+
+        fileOperation::cacheLevel(oldCache);
+        if (oldParRun)
         {
-            removeEmptyDirs(path/dir);
+            const_cast<fileOperation&>(fileHandler()).nProcs(oldNumProcs);
         }
+        UPstream::parRun(oldParRun);
     }
 
-    removeEmptyDir(path);
+    // Broadcast information to all
+    Pstream::broadcasts
+    (
+        UPstream::worldComm,
+        facesInstance,
+        pointsInstance
+    );
 }
 
 
diff --git a/applications/utilities/parallelProcessing/redistributePar/loadOrCreateMesh.H b/applications/utilities/parallelProcessing/redistributePar/loadOrCreateMesh.H
index 7f467b6f78..25383a810f 100644
--- a/applications/utilities/parallelProcessing/redistributePar/loadOrCreateMesh.H
+++ b/applications/utilities/parallelProcessing/redistributePar/loadOrCreateMesh.H
@@ -6,7 +6,7 @@
      \\/     M anipulation  |
 -------------------------------------------------------------------------------
     Copyright (C) 2012 OpenFOAM Foundation
-    Copyright (C) 2022 OpenCFD Ltd.
+    Copyright (C) 2022-2023 OpenCFD Ltd.
 -------------------------------------------------------------------------------
 License
     This file is part of OpenFOAM.
@@ -48,6 +48,9 @@ namespace Foam
 // Forward Declarations
 class faMesh;
 
+//- Check for availability of given file
+bool checkFileExistence(const fileName& fName);
+
 //- Check for availability of specified mesh file (default: "faces")
 boolList haveMeshFile
 (
@@ -64,11 +67,13 @@ void removeProcAddressing(const faMesh& mesh);
 //- Remove procAddressing
 void removeProcAddressing(const polyMesh& mesh);
 
-//- Remove empty directory
-void removeEmptyDir(const fileName& path);
-
-//- Remove empty directories from bottom up
-void removeEmptyDirs(const fileName& path);
+//- Determine master faces instance
+void masterMeshInstance
+(
+    const IOobject& io,
+    fileName& facesInstance,
+    fileName& pointsInstance
+);
 
 }
 
diff --git a/applications/utilities/parallelProcessing/redistributePar/parFaFieldDistributorCache.C b/applications/utilities/parallelProcessing/redistributePar/parFaFieldDistributorCache.C
index 04d3ba8857..6aa040c4c9 100644
--- a/applications/utilities/parallelProcessing/redistributePar/parFaFieldDistributorCache.C
+++ b/applications/utilities/parallelProcessing/redistributePar/parFaFieldDistributorCache.C
@@ -64,6 +64,7 @@ void Foam::parFaFieldDistributorCache::read
     const bool decompose,  // i.e. read from undecomposed case
 
     const boolList& areaMeshOnProc,
+    refPtr<fileOperation>& readHandler,
     const fileName& areaMeshInstance,
     faMesh& mesh
 )
@@ -76,10 +77,16 @@ void Foam::parFaFieldDistributorCache::read
     // Missing an area mesh somewhere?
     if (areaMeshOnProc.found(false))
     {
+        const bool oldParRun = UPstream::parRun(false);
+        const int oldCache = fileOperation::cacheLevel(0);
+
         // A zero-sized mesh with boundaries.
         // This is used to create zero-sized fields.
         subsetterPtr.reset(new faMeshSubset(mesh, zero{}));
 
+        fileOperation::cacheLevel(oldCache);
+        UPstream::parRun(oldParRun);  // Restore parallel state
+
         // Deregister from polyMesh ...
         auto& obr = const_cast<objectRegistry&>
         (
@@ -92,14 +99,26 @@ void Foam::parFaFieldDistributorCache::read
         obr.checkOut("faSolution");
     }
 
-
     // Get original objects (before incrementing time!)
-    if (Pstream::master() && decompose)
+    if (UPstream::master() && decompose)
     {
         runTime.caseName() = baseRunTime.caseName();
         runTime.processorCase(false);
     }
-    IOobjectList objects(mesh.mesh(), runTime.timeName());
+
+    IOobjectList objects;   //(mesh.mesh(), runTime.timeName());
+
+    if (readHandler)
+    {
+        auto oldHandler = fileOperation::fileHandler(readHandler);
+        const auto oldComm = UPstream::commWorld(fileHandler().comm());
+
+        objects = IOobjectList(mesh.mesh(), runTime.timeName());
+        readHandler = fileOperation::fileHandler(oldHandler);
+        UPstream::commWorld(oldComm);
+    }
+
+
     if (Pstream::master() && decompose)
     {
         runTime.caseName() = proc0CaseName;
@@ -116,11 +135,13 @@ void Foam::parFaFieldDistributorCache::read
         runTime.processorCase(false);
     }
 
+
     #undef  doFieldReading
     #define doFieldReading(Storage)                                   \
     fieldsDistributor::readFields                                     \
     (                                                                 \
-        areaMeshOnProc, mesh, subsetterPtr, objects, Storage,         \
+        areaMeshOnProc, readHandler, mesh, subsetterPtr, objects,     \
+        Storage,                                                      \
         true  /* (deregister field) */                                \
     );
 
diff --git a/applications/utilities/parallelProcessing/redistributePar/parFaFieldDistributorCache.H b/applications/utilities/parallelProcessing/redistributePar/parFaFieldDistributorCache.H
index 936bf9eee5..5942aba3a1 100644
--- a/applications/utilities/parallelProcessing/redistributePar/parFaFieldDistributorCache.H
+++ b/applications/utilities/parallelProcessing/redistributePar/parFaFieldDistributorCache.H
@@ -105,6 +105,7 @@ public:
             const bool decompose,  // i.e. read from undecomposed case
 
             const boolList& areaMeshOnProc,
+            refPtr<fileOperation>& readHandler,
             const fileName& areaMeshInstance,
             faMesh& mesh
         );
diff --git a/applications/utilities/parallelProcessing/redistributePar/redistributePar.C b/applications/utilities/parallelProcessing/redistributePar/redistributePar.C
index 7aeee6df86..a566b9d89e 100644
--- a/applications/utilities/parallelProcessing/redistributePar/redistributePar.C
+++ b/applications/utilities/parallelProcessing/redistributePar/redistributePar.C
@@ -106,22 +106,50 @@ Usage
 
 #include "redistributeLagrangian.H"
 
-#include "cyclicACMIFvPatch.H"
-#include "masterUncollatedFileOperation.H"
-#include "uncollatedFileOperation.H"
-#include "collatedFileOperation.H"
-
 using namespace Foam;
 
 // * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
 
 const int debug(::Foam::debug::debugSwitch("redistributePar", 0));
+#define InfoOrPout (::debug ? Pout : Info())
+
+
+// Allocate a new file handler on valid processors only
+// retaining the original IO ranks if possible
+autoPtr<fileOperation>
+getNewHandler(const boolUList& useProc, bool verbose = true)
+{
+    autoPtr<fileOperation> handler
+    (
+        fileOperation::New(fileHandler(), useProc, verbose)
+    );
+
+    if (::debug && handler)
+    {
+        Pout<< "Allocated " << handler().info()
+            << " ptr:" << Foam::name(handler.get()) << endl;
+    }
+
+    return handler;
+}
+
+
+// Allocate a new file handler on valid processors only
+// retaining the original IO ranks if possible
+void newHandler(const boolUList& useProc, refPtr<fileOperation>& handler)
+{
+    if (!handler)
+    {
+        handler = getNewHandler(useProc);
+    }
+}
+
 
 void createTimeDirs(const fileName& path)
 {
     // Get current set of local processor's time directories. Uses
     // fileHandler
-    const instantList localTimeDirs(Time::findTimes(path, "constant"));
+    instantList localTimeDirs(Time::findTimes(path, "constant"));
 
     instantList masterTimeDirs;
     if (Pstream::master())
@@ -132,20 +160,19 @@ void createTimeDirs(const fileName& path)
         masterTimeDirs = localTimeDirs;
     }
     Pstream::broadcast(masterTimeDirs);
-    //DebugVar(masterTimeDirs);
-    //DebugVar(localTimeDirs);
 
     // Sync any cached times (e.g. masterUncollatedFileOperation::times_)
     // since only master would have done the findTimes
     for (const instant& t : masterTimeDirs)
     {
-        if (!localTimeDirs.found(t))
+        if (!localTimeDirs.contains(t))
         {
             const fileName timePath(path/t.name());
 
             //Pout<< "Time:" << t << nl
             //    << "    raw       :" << timePath << nl
             //    << endl;
+            // Bypass fileHandler
             Foam::mkDir(timePath);
         }
     }
@@ -158,29 +185,91 @@ void createTimeDirs(const fileName& path)
 
 void copyUniform
 (
-    const fileOperation& fh,
-    const bool decompose,
+    refPtr<fileOperation>& readHandler,
+    refPtr<fileOperation>& writeHandler,
+
     const bool reconstruct,
+    const bool decompose,
+
     const word& readTimeName,
+    const fileName& readCaseName,
+
     const objectRegistry& readDb,
     const objectRegistry& writeDb
 )
 {
-    // Detect uniform/ at original database + time
-    const IOobject readIO("uniform", readTimeName, readDb);
-    const fileName readPath
-    (
-        fh.dirPath
+    // 3 modes: reconstruct, decompose, redistribute
+
+    // In reconstruct mode (separate reconstructed mesh):
+    // - read using readDb + readHandler
+    // - write using writeDb + writeHandler
+
+    // In decompose mode (since re-using processor0 mesh):
+    // - read using readDb + readCaseName + readHandler
+    // - write using writeDb + writeHandler
+
+    // In redistribute mode:
+    // - read using readDb + readHandler
+    // - write using writeDb + writeHandler
+
+    fileName readPath;
+    if (readHandler)
+    {
+        auto oldHandler = fileOperation::fileHandler(readHandler);
+        const label oldComm = UPstream::commWorld(fileHandler().comm());
+
+        //Pout<< "** copyUniform: switching to handler:" << fileHandler().type()
+        //    << " with comm:" << fileHandler().comm()
+        //    << " with procs:" << UPstream::procID(fileHandler().comm())
+        //    << endl;
+
+
+        Time& readTime = const_cast<Time&>(readDb.time());
+        bool oldProcCase = readTime.processorCase();
+        string oldCaseName;
+        if (decompose)
+        {
+            //Pout<< "***Setting caseName to " << readCaseName
+            //    << " to read undecomposed uniform" << endl;
+            oldCaseName = readTime.caseName();
+            readTime.caseName() = readCaseName;
+            oldProcCase = readTime.processorCase(false);
+        }
+
+        // Detect uniform/ at original database + time
+        readPath = fileHandler().dirPath
         (
             false,          // local directory
-            readIO,
+            IOobject("uniform", readTimeName, readDb),
             false           // do not search in time
-        )
-    );
-    //if (Pstream::master() && !readPath.empty())
+        );
+
+
+        readHandler = fileOperation::fileHandler(oldHandler);
+        UPstream::commWorld(oldComm);
+
+        //Pout<< "** copyUniform:"
+        //    << " switched back to handler:" << fileHandler().type()
+        //    << " with comm:" << fileHandler().comm()
+        //    << " with procs:" << UPstream::procID(fileHandler().comm())
+        //    << endl;
+
+
+        if (decompose)
+        {
+            // Reset caseName on master
+            //Pout<< "***Restoring caseName to " << oldCaseName << endl;
+            readTime.caseName() = oldCaseName;
+            readTime.processorCase(oldProcCase);
+        }
+    }
+
+    Pstream::broadcast(readPath, UPstream::worldComm);
+
     if (!readPath.empty())
     {
-        Info<< "Detected additional non-decomposed files in "
+        InfoOrPout
+            << "Detected additional non-decomposed files in "
             << readPath << endl;
 
         // readPath: searching is the same for all file handlers. Typical:
@@ -194,69 +283,64 @@ void copyUniform
         //  collated   : <case>/processors2/0.1/uniform. Should be done by
         //               local master only.
 
-        // See what local directory
-        const IOobject writeIO("uniform", writeDb.time().timeName(), writeDb);
-        const fileName writePath
+        const IOobject writeIO
         (
-            fh.objectPath
-            (
-                writeIO,
-                word::null
-            )
+            "uniform",
+            writeDb.time().timeName(),
+            writeDb
         );
-        // Do we already have this directory?
-        const fileName currentPath(fh.dirPath(false, writeIO, false));
 
-        if (::debug)
+        // Switch to writeHandler
+        if (writeHandler)
         {
-            Pout<< "    readPath   :" << readPath << endl;
-            Pout<< "    writePath  :" << writePath << endl;
-            Pout<< "    currentPath:" << currentPath << endl;
-        }
+            //const label oldWorldComm = UPstream::worldComm;
+            auto oldHandler = fileOperation::fileHandler(writeHandler);
 
-        if (readPath == writePath)
-        {
-            return;
-        }
+            // Check: fileHandler.comm() is size 1 for uncollated
+            const label writeComm = fileHandler().comm();
+            //UPstream::worldComm = writeComm;
 
-        if (currentPath.empty())
-        {
-            if (decompose)
+            if (reconstruct)
             {
-                // All processors copy to destination
-                fh.cp(readPath, writePath);
-            }
-            else if (reconstruct)
-            {
-                // Only master
-                if (Pstream::master())
-                {
-                    const bool oldParRun = Pstream::parRun(false);
-                    fh.cp(readPath, writePath);
-                    Pstream::parRun(oldParRun);
-                }
+                const bool oldParRun = UPstream::parRun(false);
+                const fileName writePath
+                (
+                    fileHandler().objectPath
+                    (
+                        writeIO,
+                        word::null
+                    )
+                );
+                fileHandler().cp(readPath, writePath);
+                UPstream::parRun(oldParRun);
             }
             else
             {
-                // Redistribute. If same destination path do only on master,
-                // if different path do on all processors. For now check
-                // if collated file handler only. tbd.
-                if (isA<fileOperations::collatedFileOperation>(fh))
+                const fileName writePath
+                (
+                    fileHandler().objectPath
+                    (
+                        writeIO,
+                        word::null
+                    )
+                );
+
+                if (::debug)
                 {
-                    // Collated
-                    if (Pstream::master())
-                    {
-                        const bool oldParRun = Pstream::parRun(false);
-                        fh.cp(readPath, writePath);
-                        Pstream::parRun(oldParRun);
-                    }
-                }
-                else
-                {
-                    // Assume uncollated
-                    fh.cp(readPath, writePath);
+                    Pout<< "    readPath   :" << readPath << endl;
+                    Pout<< "    writePath  :" << writePath << endl;
                 }
+
+                fileHandler().broadcastCopy
+                (
+                    writeComm,                   // send to all in writeComm
+                    UPstream::master(writeComm), // to use ioranks. Check!
+                    readPath,
+                    writePath
+                );
             }
+            writeHandler = fileOperation::fileHandler(oldHandler);
+            //UPstream::worldComm = oldWorldComm;
         }
     }
 }
@@ -300,17 +384,17 @@ void printMeshData(const polyMesh& mesh)
         const label nLocalCells = globalCells.localSize(proci);
         const label nBndFaces = globalBoundaryFaces.localSize(proci);
 
-        Info<< nl
+        InfoOrPout<< nl
             << "Processor " << proci;
 
         if (!nLocalCells)
         {
-            Info<< " (empty)" << endl;
+            InfoOrPout<< " (empty)" << endl;
             continue;
         }
         else
         {
-            Info<< nl
+            InfoOrPout<< nl
                 << "    Number of cells = " << nLocalCells << endl;
         }
 
@@ -319,7 +403,8 @@ void printMeshData(const polyMesh& mesh)
 
         forAll(patchNeiProcNo[proci], i)
         {
-            Info<< "    Number of faces shared with processor "
+            InfoOrPout
+                << "    Number of faces shared with processor "
                 << patchNeiProcNo[proci][i] << " = "
                 << patchSize[proci][i] << nl;
 
@@ -327,7 +412,8 @@ void printMeshData(const polyMesh& mesh)
         }
 
         {
-            Info<< "    Number of processor patches = " << nei.size() << nl
+            InfoOrPout
+                << "    Number of processor patches = " << nei.size() << nl
                 << "    Number of processor faces = " << nProcFaces << nl
                 << "    Number of boundary faces = "
                 << nBndFaces-nProcFaces << endl;
@@ -342,7 +428,8 @@ void printMeshData(const polyMesh& mesh)
 
     // Summary stats
 
-    Info<< nl
+    InfoOrPout
+        << nl
         << "Number of processor faces = " << (totProcFaces/2) << nl
         << "Max number of cells = " << maxProcCells;
 
@@ -350,30 +437,33 @@ void printMeshData(const polyMesh& mesh)
     {
         scalar avgValue = scalar(globalCells.totalSize())/Pstream::nProcs();
 
-        Info<< " (" << 100.0*(maxProcCells-avgValue)/avgValue
+        InfoOrPout
+            << " (" << 100.0*(maxProcCells-avgValue)/avgValue
             << "% above average " << avgValue << ')';
     }
-    Info<< nl;
+    InfoOrPout<< nl;
 
-    Info<< "Max number of processor patches = " << maxProcPatches;
+    InfoOrPout<< "Max number of processor patches = " << maxProcPatches;
     if (totProcPatches)
     {
         scalar avgValue = scalar(totProcPatches)/Pstream::nProcs();
 
-        Info<< " (" << 100.0*(maxProcPatches-avgValue)/avgValue
+        InfoOrPout
+            << " (" << 100.0*(maxProcPatches-avgValue)/avgValue
             << "% above average " << avgValue << ')';
     }
-    Info<< nl;
+    InfoOrPout<< nl;
 
-    Info<< "Max number of faces between processors = " << maxProcFaces;
+    InfoOrPout<< "Max number of faces between processors = " << maxProcFaces;
     if (totProcFaces)
     {
         scalar avgValue = scalar(totProcFaces)/Pstream::nProcs();
 
-        Info<< " (" << 100.0*(maxProcFaces-avgValue)/avgValue
+        InfoOrPout
+            << " (" << 100.0*(maxProcFaces-avgValue)/avgValue
             << "% above average " << avgValue << ')';
     }
-    Info<< nl << endl;
+    InfoOrPout<< nl << endl;
 }
 
 
@@ -387,22 +477,22 @@ void writeDecomposition
 {
     // Write the decomposition as labelList for use with 'manual'
     // decomposition method.
-    labelIOList cellDecomposition
+    IOListRef<label>
     (
         IOobject
         (
             "cellDecomposition",
             mesh.facesInstance(),  // mesh read from facesInstance
             mesh,
-            IOobject::NO_READ,
-            IOobject::NO_WRITE,
-            IOobject::NO_REGISTER
+            IOobjectOption::NO_READ,
+            IOobjectOption::NO_WRITE,
+            IOobjectOption::NO_REGISTER
         ),
         decomp
-    );
-    cellDecomposition.write();
+    ).write();
 
-    Info<< "Writing wanted cell distribution to volScalarField " << name
+    InfoOrPout
+        << "Writing wanted cell distribution to volScalarField " << name
         << " for postprocessing purposes." << nl << endl;
 
     volScalarField procCells
@@ -412,12 +502,12 @@ void writeDecomposition
             name,
             mesh.time().timeName(),
             mesh,
-            IOobject::NO_READ,
-            IOobject::NO_WRITE,
-            IOobject::NO_REGISTER
+            IOobjectOption::NO_READ,
+            IOobjectOption::NO_WRITE,
+            IOobjectOption::NO_REGISTER
         ),
         mesh,
-        dimensionedScalar(word::null, dimless, -1),
+        dimensionedScalar("", dimless, -1),
         fvPatchFieldBase::zeroGradientType()
     );
 
@@ -433,6 +523,7 @@ void writeDecomposition
 
 void determineDecomposition
 (
+    refPtr<fileOperation>& readHandler,
     const Time& baseRunTime,
     const fileName& decompDictFile, // optional location for decomposeParDict
     const bool decompose,       // decompose, i.e. read from undecomposed case
@@ -444,6 +535,10 @@ void determineDecomposition
     labelList& decomp
 )
 {
+    // Switch to readHandler since decomposition method might do IO
+    // (e.g. read decomposeParDict)
+    auto oldHandler = fileOperation::fileHandler(readHandler);
+
     // Read decomposeParDict (on all processors)
     const decompositionModel& method = decompositionModel::New
     (
@@ -467,9 +562,10 @@ void determineDecomposition
     Time& tm = const_cast<Time&>(mesh.time());
 
     const bool oldProcCase = tm.processorCase();
-    if (Pstream::master() && decompose)
+    if (decompose)
     {
-        Info<< "Setting caseName to " << baseRunTime.caseName()
+        InfoOrPout
+            << "Setting caseName to " << baseRunTime.caseName()
             << " to read decomposeParDict" << endl;
         tm.caseName() = baseRunTime.caseName();
         tm.processorCase(false);
@@ -487,8 +583,9 @@ void determineDecomposition
                 weightName,
                 tm.timeName(),
                 mesh,
-                IOobject::MUST_READ,
-                IOobject::NO_WRITE
+                IOobjectOption::MUST_READ,
+                IOobjectOption::NO_WRITE,
+                IOobjectOption::NO_REGISTER
             ),
             mesh
         );
@@ -498,9 +595,13 @@ void determineDecomposition
     nDestProcs = decomposer.nDomains();
     decomp = decomposer.decompose(mesh, cellWeights);
 
-    if (Pstream::master() && decompose)
+    readHandler = fileOperation::fileHandler(oldHandler);
+
+
+    if (decompose)
     {
-        Info<< "Restoring caseName" << endl;
+        InfoOrPout
+            << "Restoring caseName" << endl;
         tm.caseName() = proc0CaseName;
         tm.processorCase(oldProcCase);
     }
@@ -508,30 +609,36 @@ void determineDecomposition
     // Dump decomposition to volScalarField
     if (writeCellDist)
     {
+        const label oldNumProcs =
+            const_cast<fileOperation&>(fileHandler()).nProcs(nDestProcs);
+
         // Note: on master make sure to write to processor0
         if (decompose)
         {
-            if (Pstream::master())
+            if (UPstream::master())
             {
-                const bool oldParRun = Pstream::parRun(false);
+                const bool oldParRun = UPstream::parRun(false);
 
-                Info<< "Setting caseName to " << baseRunTime.caseName()
+                InfoOrPout
+                    << "Setting caseName to " << baseRunTime.caseName()
                     << " to write undecomposed cellDist" << endl;
 
                 tm.caseName() = baseRunTime.caseName();
                 tm.processorCase(false);
                 writeDecomposition("cellDist", mesh, decomp);
-                Info<< "Restoring caseName" << endl;
+                InfoOrPout<< "Restoring caseName" << endl;
                 tm.caseName() = proc0CaseName;
                 tm.processorCase(oldProcCase);
 
-                Pstream::parRun(oldParRun);
+                UPstream::parRun(oldParRun);
             }
         }
         else
         {
             writeDecomposition("cellDist", mesh, decomp);
         }
+
+        const_cast<fileOperation&>(fileHandler()).nProcs(oldNumProcs);
     }
 }
 
@@ -551,6 +658,7 @@ void correctCoupledBoundaryConditions(fvMesh& mesh)
 // Inplace redistribute mesh and any fields
 autoPtr<mapDistributePolyMesh> redistributeAndWrite
 (
+    refPtr<fileOperation>& readHandler,
     refPtr<fileOperation>& writeHandler,
     const Time& baseRunTime,
     const fileName& proc0CaseName,
@@ -575,7 +683,7 @@ autoPtr<mapDistributePolyMesh> redistributeAndWrite
     const bool oldProcCase = runTime.processorCase();
 
     //// Print some statistics
-    //Info<< "Before distribution:" << endl;
+    //Pout<< "Before distribution:" << endl;
     //printMeshData(mesh);
 
     // Storage of fields
@@ -610,11 +718,14 @@ autoPtr<mapDistributePolyMesh> redistributeAndWrite
     // Track how many (if any) pointFields are read/mapped
     label nPointFields = 0;
 
+    refPtr<fileOperation> noWriteHandler;
+
     parPointFieldDistributor pointDistributor
     (
         oldPointMesh,   // source mesh
         false,          // savePoints=false (ie, delay until later)
-        false           // Do not write
+        //false           // Do not write
+        noWriteHandler    // Do not write
     );
 
 
@@ -633,36 +744,67 @@ autoPtr<mapDistributePolyMesh> redistributeAndWrite
             // A zero-sized mesh with boundaries.
             // This is used to create zero-sized fields.
             subsetterPtr.reset(new fvMeshSubset(mesh, zero{}));
+            subsetterPtr().subMesh().init(true);
+            subsetterPtr().subMesh().globalData();
+            subsetterPtr().subMesh().tetBasePtIs();
+            subsetterPtr().subMesh().geometricD();
         }
 
 
         // Get original objects (before incrementing time!)
-        if (Pstream::master() && decompose)
+        if (decompose)
         {
+            InfoOrPout
+                << "Setting caseName to " << baseRunTime.caseName()
+                << " to read IOobjects" << endl;
             runTime.caseName() = baseRunTime.caseName();
             runTime.processorCase(false);
         }
-        IOobjectList objects(mesh, runTime.timeName());
-        if (Pstream::master() && decompose)
+
+        //IOobjectList objects(mesh, runTime.timeName());
+        // Swap to reading fileHandler and read IOobjects
+        IOobjectList objects;
+        if (readHandler)
         {
+            auto oldHandler = fileOperation::fileHandler(readHandler);
+            const label oldComm = UPstream::commWorld(fileHandler().comm());
+
+            //Pout<< "** redistributeAndWrite:"
+            //    << " switching to handler:" << fileHandler().type()
+            //    << " with comm:" << fileHandler().comm()
+            //    << " with procs:" << UPstream::procID(fileHandler().comm())
+            //    << endl;
+
+            objects = IOobjectList(mesh, runTime.timeName());
+            readHandler = fileOperation::fileHandler(oldHandler);
+            UPstream::commWorld(oldComm);
+
+            //Pout<< "** redistributeAndWrite:"
+            //    << " switched back to handler:" << fileHandler().type()
+            //    << " with comm:" << fileHandler().comm()
+            //    << " with procs:" << UPstream::procID(fileHandler().comm())
+            //    << endl;
+        }
+
+        if (decompose)
+        {
+            InfoOrPout
+                << "Restoring caseName" << endl;
             runTime.caseName() = proc0CaseName;
             runTime.processorCase(oldProcCase);
         }
 
-        Info<< "From time " << runTime.timeName()
+        InfoOrPout
+            << "From time " << runTime.timeName()
             << " mesh:" << mesh.objectRegistry::objectRelPath()
             << " have objects:" << objects.names() << endl;
 
         // We don't want to map the decomposition (mapping already tested when
         // mapping the cell centre field)
-        auto iter = objects.find("cellDist");
-        if (iter.good())
-        {
-            objects.erase(iter);
-        }
+        (void)objects.erase("cellDist");
 
 
-        if (Pstream::master() && decompose)
+        if (decompose)
         {
             runTime.caseName() = baseRunTime.caseName();
             runTime.processorCase(false);
@@ -675,7 +817,8 @@ autoPtr<mapDistributePolyMesh> redistributeAndWrite
         {                                                                     \
             fieldsDistributor::readFields                                     \
             (                                                                 \
-                volMeshOnProc, mesh, subsetterPtr, objects, Storage           \
+                volMeshOnProc, readHandler, mesh, subsetterPtr,               \
+                objects, Storage                                              \
             );                                                                \
         }
 
@@ -708,7 +851,8 @@ autoPtr<mapDistributePolyMesh> redistributeAndWrite
         {                                                                     \
             fieldsDistributor::readFields                                     \
             (                                                                 \
-                volMeshOnProc, oldPointMesh, subsetterPtr, objects, Storage,  \
+                volMeshOnProc, readHandler, oldPointMesh,                     \
+                subsetterPtr, objects, Storage,                               \
                 true  /* (deregister field) */                                \
             );                                                                \
             nPointFields += Storage.size();                                   \
@@ -724,7 +868,7 @@ autoPtr<mapDistributePolyMesh> redistributeAndWrite
 
         // Done reading
 
-        if (Pstream::master() && decompose)
+        if (decompose)
         {
             runTime.caseName() = proc0CaseName;
             runTime.processorCase(oldProcCase);
@@ -738,6 +882,58 @@ autoPtr<mapDistributePolyMesh> redistributeAndWrite
     }
 
 
+    // Read refinement data
+    autoPtr<hexRef8Data> refDataPtr;
+    {
+        // Read refinement data
+        if (decompose)
+        {
+            runTime.caseName() = baseRunTime.caseName();
+            runTime.processorCase(false);
+        }
+        IOobject io
+        (
+            "dummy",
+            volMeshInstance,    //mesh.facesInstance(),
+            polyMesh::meshSubDir,
+            mesh,
+            IOobjectOption::LAZY_READ,
+            IOobjectOption::NO_WRITE,
+            IOobjectOption::NO_REGISTER
+        );
+
+        if (readHandler)
+        {
+            auto oldHandler = fileOperation::fileHandler(readHandler);
+            const label oldComm = UPstream::commWorld(fileHandler().comm());
+
+            // Read
+            refDataPtr.reset(new hexRef8Data(io));
+
+            UPstream::commWorld(oldComm);
+            readHandler = fileOperation::fileHandler(oldHandler);
+        }
+        else
+        {
+            io.readOpt(IOobjectOption::NO_READ);
+            refDataPtr.reset(new hexRef8Data(io));
+        }
+
+        if (decompose)
+        {
+            runTime.caseName() = proc0CaseName;
+            runTime.processorCase(oldProcCase);
+        }
+
+        // Make sure all processors have valid data (since only some will
+        // read)
+        refDataPtr().sync(io);
+
+        // Now we've read refinement data we can remove it
+        meshRefinement::removeFiles(mesh);
+    }
+
+
     // Mesh distribution engine
     fvMeshDistribute distributor(mesh);
 
@@ -745,7 +941,7 @@ autoPtr<mapDistributePolyMesh> redistributeAndWrite
     autoPtr<mapDistributePolyMesh> distMap = distributor.distribute(decomp);
 
     // Print some statistics
-    Info<< "After distribution:" << endl;
+    InfoOrPout<< "After distribution:" << endl;
     printMeshData(mesh);
 
     // Get other side of processor boundaries
@@ -807,8 +1003,8 @@ autoPtr<mapDistributePolyMesh> redistributeAndWrite
             mesh.facesInstance(),
             polyMesh::meshSubDir,
             mesh.thisDb(),
-            IOobject::NO_READ,
-            IOobject::AUTO_WRITE
+            IOobjectOption::NO_READ,
+            IOobjectOption::AUTO_WRITE
         ),
         distMap()
     );
@@ -816,39 +1012,57 @@ autoPtr<mapDistributePolyMesh> redistributeAndWrite
 
     if (reconstruct)
     {
-        if (Pstream::master())
+        auto oldHandler = fileOperation::fileHandler(writeHandler);
+
+        if (UPstream::master())
         {
-            Info<< "Setting caseName to " << baseRunTime.caseName()
+            InfoOrPout
+                << "Setting caseName to " << baseRunTime.caseName()
                 << " to write reconstructed mesh (and fields)." << endl;
             runTime.caseName() = baseRunTime.caseName();
             const bool oldProcCase(runTime.processorCase(false));
+            const label oldNumProcs
+            (
+                const_cast<fileOperation&>(fileHandler()).nProcs(nDestProcs)
+            );
+            const bool oldParRun = UPstream::parRun(false);
 
             mesh.write();
             topoSet::removeFiles(mesh);
 
             // Now we've written all. Reset caseName on master
-            Info<< "Restoring caseName" << endl;
+            InfoOrPout<< "Restoring caseName" << endl;
+            UPstream::parRun(oldParRun);
+            const_cast<fileOperation&>(fileHandler()).nProcs(oldNumProcs);
             runTime.caseName() = proc0CaseName;
             runTime.processorCase(oldProcCase);
         }
+
+        writeHandler = fileOperation::fileHandler(oldHandler);
     }
     else
     {
         auto oldHandler = fileOperation::fileHandler(writeHandler);
 
+        const label oldNumProcs
+        (
+            const_cast<fileOperation&>(fileHandler()).nProcs(nDestProcs)
+        );
         mesh.write();
+        const_cast<fileOperation&>(fileHandler()).nProcs(oldNumProcs);
 
         writeHandler = fileOperation::fileHandler(oldHandler);
 
         topoSet::removeFiles(mesh);
     }
-    Info<< "Written redistributed mesh to "
+    InfoOrPout
+        << "Written redistributed mesh to "
         << mesh.facesInstance() << nl << endl;
 
 
-    if (decompose || reconstruct)
+    if (decompose)
     {
-        // Decompose (1 -> N) or reconstruct (N -> 1)
+        // Decompose (1 -> N)
         // so {boundary,cell,face,point}ProcAddressing have meaning
         fvMeshTools::writeProcAddressing
         (
@@ -858,6 +1072,20 @@ autoPtr<mapDistributePolyMesh> redistributeAndWrite
             writeHandler
         );
     }
+    else if (reconstruct)
+    {
+        // Reconstruct (N -> 1)
+        // so {boundary,cell,face,point}ProcAddressing have meaning. Make sure
+        // to write these to meshes containing the source meshes (i.e. using
+        // the read handler)
+        fvMeshTools::writeProcAddressing
+        (
+            mesh,
+            distMap(),
+            decompose,
+            readHandler //writeHandler
+        );
+    }
     else
     {
         // Redistribute (N -> M)
@@ -868,50 +1096,42 @@ autoPtr<mapDistributePolyMesh> redistributeAndWrite
 
 
     // Refinement data
+    if (refDataPtr)
     {
+        auto& refData = refDataPtr();
 
-        // Read refinement data
-        if (Pstream::master() && decompose)
-        {
-            runTime.caseName() = baseRunTime.caseName();
-            runTime.processorCase(false);
-        }
+        // Set instance
         IOobject io
         (
             "dummy",
             mesh.facesInstance(),
             polyMesh::meshSubDir,
             mesh,
-            IOobject::READ_IF_PRESENT,
-            IOobject::NO_WRITE,
-            IOobject::NO_REGISTER
+            IOobjectOption::NO_READ,
+            IOobjectOption::NO_WRITE,
+            IOobjectOption::NO_REGISTER
         );
-
-        hexRef8Data refData(io);
-        if (Pstream::master() && decompose)
-        {
-            runTime.caseName() = proc0CaseName;
-            runTime.processorCase(oldProcCase);
-        }
-
-        // Make sure all processors have valid data (since only some will
-        // read)
         refData.sync(io);
 
+
         // Distribute
         refData.distribute(distMap());
 
 
-        // Now we've read refinement data we can remove it
-        meshRefinement::removeFiles(mesh);
+        auto oldHandler = fileOperation::fileHandler(writeHandler);
 
         if (reconstruct)
         {
-            if (Pstream::master())
+            if (UPstream::master())
             {
-                const bool oldParRun = Pstream::parRun(false);
+                const bool oldParRun = UPstream::parRun(false);
+                const label oldNumProcs
+                (
+                    const_cast<fileOperation&>(fileHandler()).nProcs(nDestProcs)
+                );
 
-                Info<< "Setting caseName to " << baseRunTime.caseName()
+                InfoOrPout
+                    << "Setting caseName to " << baseRunTime.caseName()
                     << " to write reconstructed refinement data." << endl;
                 runTime.caseName() = baseRunTime.caseName();
                 const bool oldProcCase(runTime.processorCase(false));
@@ -919,23 +1139,31 @@ autoPtr<mapDistributePolyMesh> redistributeAndWrite
                 refData.write();
 
                 // Now we've written all. Reset caseName on master
-                Info<< "Restoring caseName" << endl;
+                InfoOrPout<< "Restoring caseName" << endl;
                 runTime.caseName() = proc0CaseName;
                 runTime.processorCase(oldProcCase);
 
-                Pstream::parRun(oldParRun);
+                const_cast<fileOperation&>(fileHandler()).nProcs(oldNumProcs);
+                UPstream::parRun(oldParRun);
             }
         }
         else
         {
+            const label oldNumProcs
+            (
+                const_cast<fileOperation&>(fileHandler()).nProcs(nDestProcs)
+            );
             refData.write();
+            const_cast<fileOperation&>(fileHandler()).nProcs(oldNumProcs);
         }
+
+        writeHandler = fileOperation::fileHandler(oldHandler);
     }
 
     //// Sets. Disabled for now.
     //{
     //    // Read sets
-    //    if (Pstream::master() && decompose)
+    //    if (decompose)
     //    {
     //        runTime.caseName() = baseRunTime.caseName();
     //        runTime.processorCase(false);
@@ -945,7 +1173,7 @@ autoPtr<mapDistributePolyMesh> redistributeAndWrite
     //    PtrList<cellSet> cellSets;
     //    ReadFields(objects, cellSets);
     //
-    //    if (Pstream::master() && decompose)
+    //    if (decompose)
     //    {
     //        runTime.caseName() = proc0CaseName;
     //        runTime.processorCase(oldProcCase);
@@ -960,7 +1188,7 @@ autoPtr<mapDistributePolyMesh> redistributeAndWrite
     //    {
     //        if (Pstream::master())
     //        {
-    //            Info<< "Setting caseName to " << baseRunTime.caseName()
+    //            Pout<< "Setting caseName to " << baseRunTime.caseName()
     //                << " to write reconstructed refinement data." << endl;
     //            runTime.caseName() = baseRunTime.caseName();
     //            const bool oldProcCase(runTime.processorCase(false));
@@ -971,7 +1199,7 @@ autoPtr<mapDistributePolyMesh> redistributeAndWrite
     //            }
     //
     //            // Now we've written all. Reset caseName on master
-    //            Info<< "Restoring caseName" << endl;
+    //            Pout<< "Restoring caseName" << endl;
     //            runTime.caseName() = proc0CaseName;
     //            runTime.processorCase(oldProcCase);
     //        }
@@ -1014,7 +1242,7 @@ int main(int argc, char *argv[])
     #include "addOverwriteOption.H"
     argList::addBoolOption("decompose", "Decompose case");
     argList::addBoolOption("reconstruct", "Reconstruct case");
-    argList::addVerboseOption();
+    argList::addVerboseOption("Additional verbosity");
     argList::addDryRunOption
     (
         "Test without writing the decomposition. "
@@ -1058,32 +1286,30 @@ int main(int argc, char *argv[])
     //                than it writes to
     // - reconstruct - reads parallel, write on master only and to parent
     //                 directory
-    refPtr<fileOperation> writeHandler;
-    if
-    (
-        fileHandler().type()
-     != fileOperations::uncollatedFileOperation::typeName
-    )
-    {
-        // Install 'uncollated' as fileHandler. Save old one in writeHandler.
-        writeHandler =
-            fileOperation::fileHandler(fileOperation::NewUncollated());
-    }
+
+    //- Disable caching of times etc. Cause massive parallel problems when e.g
+    //  decomposing.
+    fileOperation::cacheLevel(0);
+
+    // Detect if running data-distributed (processors cannot see all other
+    // processors' files)
+    const bool nfs = !fileHandler().distributed();
 
     // Switch off parallel synchronisation of cached time directories
-    fileHandler().distributed(true);
+    //fileHandler().distributed(true);
 
-    // File handler to be used for writing
-    const fileOperation& fh
+    // Read handler on processors with a volMesh
+    refPtr<fileOperation> volMeshReadHandler;
+
+    // Read handler on processors with an areaMesh
+    refPtr<fileOperation> areaMeshReadHandler;
+
+    // Handler for master-only operation
+    refPtr<fileOperation> masterOnlyHandler
     (
-        writeHandler
-      ? writeHandler()
-      : fileHandler()
+        getNewHandler(boolList(one{}, true))
     );
 
-    // Make sure to call findTimes on all processors to force caching of
-    // time directories
-    (void)fileHandler().findTimes(args.path(), "constant");
 
     // Need this line since we don't include "setRootCase.H"
     #include "foamDlOpenLibs.H"
@@ -1122,7 +1348,7 @@ int main(int argc, char *argv[])
 
     if (decompose)
     {
-        Info<< "Decomposing case (like decomposePar)" << nl << endl;
+        InfoOrPout<< "Decomposing case (like decomposePar)" << nl << endl;
         if (reconstruct)
         {
             FatalErrorInFunction
@@ -1132,7 +1358,8 @@ int main(int argc, char *argv[])
     }
     else if (reconstruct)
     {
-        Info<< "Reconstructing case (like reconstructParMesh)" << nl << endl;
+        InfoOrPout<< "Reconstructing case (like reconstructParMesh)"
+            << nl << endl;
     }
 
     if ((decompose || reconstruct) && !overwrite)
@@ -1151,25 +1378,26 @@ int main(int argc, char *argv[])
     }
 
 
-    if (!Foam::isDir(args.rootPath()))
+    if
+    (
+        fileHandler().ioRanks().contains(UPstream::myProcNo())
+    && !Foam::isDir(args.rootPath())
+    )
     {
-        FatalErrorInFunction
+        //FatalErrorInFunction
+        WarningInFunction
             << ": cannot open root directory " << args.rootPath()
-            << exit(FatalError);
+            << endl;
+            //<< exit(FatalError);
     }
 
-    // Detect if running data-distributed (multiple roots)
-    bool nfs = true;
-    {
-        List<fileName> roots(1, args.rootPath());
-        Pstream::combineReduce(roots, ListOps::uniqueEqOp<fileName>());
-        nfs = (roots.size() == 1);
-    }
 
     if (!nfs)
     {
-        Info<< "Detected multiple roots i.e. non-nfs running"
+        InfoOrPout<< "Detected multiple roots i.e. non-nfs running"
             << nl << endl;
+
+        fileHandler().mkDir(args.globalPath());
     }
 
     // Check if we have processor directories. Ideally would like to
@@ -1181,7 +1409,7 @@ int main(int argc, char *argv[])
     {
         if (decompose)
         {
-            Info<< "Removing existing processor directory:"
+            InfoOrPout<< "Removing existing processor directory:"
                 << args.relativePath(procDir) << endl;
             fileHandler().rmDir(procDir);
         }
@@ -1189,17 +1417,17 @@ int main(int argc, char *argv[])
     else
     {
         // Directory does not exist. If this happens on master -> decompose mode
-        if (Pstream::master())
+        if (UPstream::master() && !reconstruct)
         {
            decompose = true;
-           Info<< "No processor directories; switching on decompose mode"
+           InfoOrPout
+                << "No processor directories; switching on decompose mode"
                << nl << endl;
         }
     }
     // If master changed to decompose mode make sure all nodes know about it
     Pstream::broadcast(decompose);
 
-
     // If running distributed we have problem of new processors not finding
     // a system/controlDict. However if we switch on the master-only reading
     // the problem becomes that the time directories are differing sizes and
@@ -1208,16 +1436,49 @@ int main(int argc, char *argv[])
     // processors
     if (!reconstruct)
     {
-        Info<< "Creating time directories on all processors" << nl << endl;
+        InfoOrPout<< "Creating time directories on all processors"
+            << nl << endl;
         createTimeDirs(args.path());
     }
 
     // Construct time
     // ~~~~~~~~~~~~~~
 
-    #include "createTime.H"
-    runTime.functionObjects().off();  // Extra safety?
+    // Disable master-only file checking since
+    // - uncollated : interferes with running read-handler on less processors
+    //                than nProcs (when redistributing from less to more)
+    // - (masterUn)collated : does not use timeStampMaster anyway
+    Info<< "Disabling uncollated master reading" << nl << endl;
+    IOobject::fileModificationChecking = IOobject::timeStamp;
 
+    // Set up uncollated file handler (still over all processors)
+    refPtr<fileOperation> basicHandler(fileOperation::NewUncollated());
+    auto oldHandler = fileOperation::fileHandler(basicHandler);
+
+    // Replace #include "createTime.H" with our own version
+    // that has MUST_READ instead of READ_MODIFIED
+
+    Info<< "Create time\n" << endl;
+    Time runTime
+    (
+        Time::controlDictName,
+        args,
+        false,  // no enableFunctionObjects
+        true,   // permit enableLibs
+        IOobjectOption::MUST_READ  // Without watching
+    );
+
+    // Restore file handler
+    (void) fileOperation::fileHandler(oldHandler);
+
+    refPtr<fileOperation> writeHandler;
+
+
+    runTime.functionObjects().off();  // Extra safety?
+    // Make sure that no runTime checking is done since fileOperation::addWatch
+    // etc. does broadcast over world, even if constructed only on a subset
+    // of procs
+    runTime.runTimeModifiable(false);
 
     // Save local processor0 casename
     const fileName proc0CaseName = runTime.caseName();
@@ -1231,13 +1492,13 @@ int main(int argc, char *argv[])
 
     if (!nfs)
     {
-        Info<< "Creating time directories for undecomposed Time"
+        InfoOrPout<< "Creating time directories for undecomposed Time"
             << " on all processors" << nl << endl;
         createTimeDirs(args.globalPath());
     }
 
 
-    Info<< "Create undecomposed database" << nl << endl;
+    InfoOrPout<< "Create undecomposed database" << nl << endl;
     Time baseRunTime
     (
         runTime.controlDict(),
@@ -1246,8 +1507,13 @@ int main(int argc, char *argv[])
         runTime.system(),
         runTime.constant(),
         false,  // No function objects
-        false   // No extra controlDict libs
+        false,  // No extra controlDict libs
+        IOobjectOption::MUST_READ  // Without watching
     );
+    // Make sure that no runTime checking is done since fileOperation::addWatch
+    // etc. does broadcast over world, even if constructed only on a subset
+    // of procs
+    baseRunTime.runTimeModifiable(false);
 
 
     wordHashSet masterTimeDirSet;
@@ -1270,14 +1536,13 @@ int main(int argc, char *argv[])
 
     if (regionNames.size() == 1 && regionNames[0] != polyMesh::defaultRegion)
     {
-        Info<< "Using region: " << regionNames[0] << nl << endl;
+        InfoOrPout<< "Using region: " << regionNames[0] << nl << endl;
     }
 
 
     // Demand driven lagrangian mapper
     autoPtr<parLagrangianDistributor> lagrangianDistributorPtr;
 
-
     if (reconstruct)
     {
         // use the times list from the master processor
@@ -1297,7 +1562,8 @@ int main(int argc, char *argv[])
         // ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 
-        Info<< nl
+        InfoOrPout
+            << nl
             << "Reconstructing mesh and addressing" << nl << endl;
 
         forAll(regionNames, regioni)
@@ -1308,7 +1574,8 @@ int main(int argc, char *argv[])
             const fileName volMeshSubDir(regionDir/polyMesh::meshSubDir);
             const fileName areaMeshSubDir(regionDir/faMesh::meshSubDir);
 
-            Info<< nl
+            InfoOrPout
+                << nl
                 << "Reconstructing mesh " << regionDir << nl << endl;
 
             bool areaMeshDetected = false;
@@ -1320,7 +1587,7 @@ int main(int argc, char *argv[])
                 runTime.setTime(timeDirs[timeI], timeI);
                 baseRunTime.setTime(timeDirs[timeI], timeI);
 
-                Info<< "Time = " << runTime.timeName() << endl << endl;
+                InfoOrPout<< "Time = " << runTime.timeName() << endl << endl;
 
                 // Where meshes are
                 fileName volMeshInstance;
@@ -1330,7 +1597,7 @@ int main(int argc, char *argv[])
                 (
                     volMeshSubDir,
                     "faces",
-                    IOobject::READ_IF_PRESENT
+                    IOobjectOption::READ_IF_PRESENT
                 );
 
                 if (doFiniteArea)
@@ -1339,7 +1606,7 @@ int main(int argc, char *argv[])
                     (
                         areaMeshSubDir,
                         "faceLabels",
-                        IOobject::READ_IF_PRESENT
+                        IOobjectOption::READ_IF_PRESENT
                     );
                 }
 
@@ -1364,6 +1631,9 @@ int main(int argc, char *argv[])
                     "faces"
                 );
 
+                // Create handler for reading
+                newHandler(volMeshOnProc, volMeshReadHandler);
+
                 if (doFiniteArea)
                 {
                     areaMeshOnProc = haveMeshFile
@@ -1372,8 +1642,20 @@ int main(int argc, char *argv[])
                         areaMeshInstance/areaMeshSubDir,
                         "faceLabels"
                     );
-
                     areaMeshDetected = areaMeshOnProc.found(true);
+
+                    if (areaMeshOnProc == volMeshOnProc)
+                    {
+                        if (volMeshReadHandler)
+                        {
+                            // Use same reader for faMesh as for fvMesh
+                            areaMeshReadHandler.ref(volMeshReadHandler.ref());
+                        }
+                    }
+                    else
+                    {
+                        newHandler(areaMeshOnProc, areaMeshReadHandler);
+                    }
                 }
 
 
@@ -1397,7 +1679,7 @@ int main(int argc, char *argv[])
                             volMeshInstance,
                             volMeshSubDir,
                             runTime,
-                            IOobject::MUST_READ
+                            IOobjectOption::MUST_READ
                         )
                     );
 
@@ -1410,7 +1692,7 @@ int main(int argc, char *argv[])
                             volMeshInstance,
                             volMeshSubDir,
                             runTime,
-                            IOobject::READ_IF_PRESENT
+                            IOobjectOption::READ_IF_PRESENT
                         )
                     );
 
@@ -1438,7 +1720,7 @@ int main(int argc, char *argv[])
                             areaMeshInstance,
                             areaMeshSubDir,
                             runTime,
-                            IOobject::MUST_READ
+                            IOobjectOption::MUST_READ
                         )
                     );
 
@@ -1451,7 +1733,7 @@ int main(int argc, char *argv[])
                             areaMeshInstance,
                             areaMeshSubDir,
                             runTime,
-                            IOobject::READ_IF_PRESENT
+                            IOobjectOption::READ_IF_PRESENT
                         )
                     );
 
@@ -1477,7 +1759,8 @@ int main(int argc, char *argv[])
 
                 if (Pstream::master())
                 {
-                    Info<< "Checking " << baseRunTime.caseName()
+                    InfoOrPout
+                        << "Checking " << baseRunTime.caseName()
                         << " for undecomposed volume and area meshes..."
                         << endl;
 
@@ -1493,7 +1776,7 @@ int main(int argc, char *argv[])
                                 volMeshInstance,
                                 volMeshSubDir,
                                 baseRunTime,
-                                IOobject::NO_READ
+                                IOobjectOption::NO_READ
                             ),
                             label(0)
                         );
@@ -1511,7 +1794,7 @@ int main(int argc, char *argv[])
                                 areaMeshInstance,
                                 areaMeshSubDir,
                                 baseRunTime,
-                                IOobject::NO_READ
+                                IOobjectOption::NO_READ
                             )
                         );
                         areaMeshHaveUndecomposed = labelsIO.headerOk();
@@ -1529,7 +1812,8 @@ int main(int argc, char *argv[])
 
                 // Report
                 {
-                    Info<< "    volume mesh ["
+                    InfoOrPout
+                        << "    volume mesh ["
                         << volMeshHaveUndecomposed << "] : "
                         << volMeshInstance << nl
                         << "    area   mesh ["
@@ -1545,13 +1829,15 @@ int main(int argc, char *argv[])
                  || !returnReduceAnd(haveVolAddressing)
                 )
                 {
-                    Info<< "No undecomposed mesh. Creating from: "
+                    InfoOrPout
+                        << "No undecomposed mesh. Creating from: "
                         << volMeshInstance << endl;
 
                     if (areaMeshHaveUndecomposed)
                     {
                         areaMeshHaveUndecomposed = false;
-                        Info<< "Also ignore any undecomposed area mesh"
+                        InfoOrPout
+                            << "Also ignore any undecomposed area mesh"
                             << endl;
                     }
 
@@ -1562,15 +1848,15 @@ int main(int argc, char *argv[])
                             regionName,
                             volMeshInstance,
                             runTime,
-                            Foam::IOobject::MUST_READ
+                            IOobjectOption::MUST_READ
                         ),
-                        decompose
+                        volMeshReadHandler
                     );
                     fvMeshTools::setBasicGeometry(volMeshPtr());
                     fvMesh& mesh = volMeshPtr();
 
 
-                    Info<< nl << "Reconstructing mesh" << nl << endl;
+                    InfoOrPout<< nl << "Reconstructing mesh" << nl << endl;
 
                     // Reconstruct (1 processor)
                     const label nDestProcs(1);
@@ -1578,7 +1864,8 @@ int main(int argc, char *argv[])
 
                     redistributeAndWrite
                     (
-                        writeHandler,
+                        volMeshReadHandler,
+                        masterOnlyHandler,  //writeHandler,
                         baseRunTime,
                         proc0CaseName,
 
@@ -1614,10 +1901,11 @@ int main(int argc, char *argv[])
                     )
                 )
                 {
-                    Info<< "Loading area mesh from "
+                    InfoOrPout
+                        << "Loading area mesh from "
                         << areaMeshInstance << endl;
 
-                    Info<< "    getting volume mesh support" << endl;
+                    InfoOrPout<< "    getting volume mesh support" << endl;
 
                     autoPtr<fvMesh> baseMeshPtr = fvMeshTools::newMesh
                     (
@@ -1626,7 +1914,7 @@ int main(int argc, char *argv[])
                             regionName,
                             baseRunTime.timeName(),
                             baseRunTime,
-                            IOobject::MUST_READ
+                            IOobjectOption::MUST_READ
                         ),
                         true            // read on master only
                     );
@@ -1639,11 +1927,10 @@ int main(int argc, char *argv[])
                             regionName,
                             baseMeshPtr().facesInstance(),
                             runTime,
-                            Foam::IOobject::MUST_READ
+                            IOobjectOption::MUST_READ
                         ),
-                        decompose
+                        volMeshReadHandler
                     );
-                    fvMeshTools::setBasicGeometry(volMeshPtr());
                     fvMesh& mesh = volMeshPtr();
 
                     // Read volume proc addressing back to base mesh
@@ -1653,6 +1940,19 @@ int main(int argc, char *argv[])
                     );
 
 
+                    //autoPtr<faMesh> areaMeshPtr =
+                    // faMeshTools::loadOrCreateMesh
+                    //(
+                    //    IOobject
+                    //    (
+                    //        regionName,
+                    //        areaMeshInstance,
+                    //        runTime,
+                    //        IOobjectOption::MUST_READ
+                    //    ),
+                    //    mesh,  // <- The referenced polyMesh (from above)
+                    //    decompose
+                    //);
                     autoPtr<faMesh> areaMeshPtr = faMeshTools::loadOrCreateMesh
                     (
                         IOobject
@@ -1660,10 +1960,10 @@ int main(int argc, char *argv[])
                             regionName,
                             areaMeshInstance,
                             runTime,
-                            Foam::IOobject::MUST_READ
+                            IOobjectOption::MUST_READ
                         ),
                         mesh,  // <- The referenced polyMesh (from above)
-                        decompose
+                        areaMeshReadHandler
                     );
                     faMesh& areaMesh = areaMeshPtr();
 
@@ -1690,15 +1990,18 @@ int main(int argc, char *argv[])
 
                     if (Pstream::master())
                     {
-                        Info<< "Setting caseName to " << baseRunTime.caseName()
+                        InfoOrPout
+                            << "Setting caseName to " << baseRunTime.caseName()
                             << " to write reconstructed area mesh." << endl;
                         runTime.caseName() = baseRunTime.caseName();
                         const bool oldProcCase(runTime.processorCase(false));
+                        const bool oldParRun(Pstream::parRun(false));
 
                         areaBaseMeshPtr().write();
 
                         // Now we've written all. Reset caseName on master
-                        Info<< "Restoring caseName" << endl;
+                        InfoOrPout<< "Restoring caseName" << endl;
+                        Pstream::parRun(oldParRun);
                         runTime.caseName() = proc0CaseName;
                         runTime.processorCase(oldProcCase);
                     }
@@ -1709,7 +2012,7 @@ int main(int argc, char *argv[])
                         areaBaseMeshPtr(),  // Reconstruct location
                         faDistMap,
                         false,              // decompose=false
-                        writeHandler,
+                        writeHandler,       //writeHandler,
                         areaMeshPtr.get()   // procMesh
                     );
                 }
@@ -1723,35 +2026,69 @@ int main(int argc, char *argv[])
             // Pass2 : read mesh and addressing and reconstruct fields
             // ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-            Info<< nl
+            InfoOrPout
+                << nl
                 << "Reconstructing fields" << nl << endl;
 
             runTime.setTime(timeDirs[0], 0);
             baseRunTime.setTime(timeDirs[0], 0);
-            Info<< "Time = " << runTime.timeName() << endl << endl;
+            InfoOrPout<< "Time = " << runTime.timeName() << endl << endl;
 
 
             // Read undecomposed mesh on master and 'empty' mesh
             // (zero faces, point, cells but valid patches and zones) on slaves.
             // This is a bit of tricky code and hidden inside fvMeshTools for
             // now.
-            Info<< "Reading undecomposed mesh (on master)" << endl;
-
-            autoPtr<fvMesh> baseMeshPtr = fvMeshTools::newMesh
+            InfoOrPout<< "Reading undecomposed mesh (on master)" << endl;
+            //autoPtr<fvMesh> baseMeshPtr = fvMeshTools::newMesh
+            //(
+            //    IOobject
+            //    (
+            //        regionName,
+            //        baseRunTime.timeName(),
+            //        baseRunTime,
+            //        IOobjectOption::MUST_READ
+            //    ),
+            //    true            // read on master only
+            //);
+            fileName facesInstance;
+            fileName pointsInstance;
+            masterMeshInstance
             (
                 IOobject
                 (
                     regionName,
                     baseRunTime.timeName(),
                     baseRunTime,
-                    IOobject::MUST_READ
+                    IOobjectOption::MUST_READ
                 ),
-                true            // read on master only
+                facesInstance,
+                pointsInstance
             );
 
-            fvMeshTools::setBasicGeometry(baseMeshPtr());
+            autoPtr<fvMesh> baseMeshPtr = fvMeshTools::loadOrCreateMesh
+            (
+                IOobject
+                (
+                    regionName,
+                    facesInstance,              //baseRunTime.timeName(),
+                    baseRunTime,
+                    IOobjectOption::MUST_READ
+                ),
+                masterOnlyHandler               // read on master only
+            );
 
-            Info<< "Reading local, decomposed mesh" << endl;
+            if (::debug)
+            {
+                Pout<< "Undecomposed mesh :"
+                    << " instance:" << baseMeshPtr().facesInstance()
+                    << " nCells:" << baseMeshPtr().nCells()
+                    << " nFaces:" << baseMeshPtr().nFaces()
+                    << " nPoints:" << baseMeshPtr().nPoints()
+                    << endl;
+            }
+
+            InfoOrPout<< "Reading local, decomposed mesh" << endl;
             autoPtr<fvMesh> volMeshPtr = fvMeshTools::loadOrCreateMesh
             (
                 IOobject
@@ -1759,9 +2096,9 @@ int main(int argc, char *argv[])
                     regionName,
                     baseMeshPtr().facesInstance(),
                     runTime,
-                    Foam::IOobject::MUST_READ
+                    IOobjectOption::MUST_READ
                 ),
-                decompose
+                volMeshReadHandler              // read on fvMesh processors
             );
             fvMesh& mesh = volMeshPtr();
 
@@ -1774,19 +2111,43 @@ int main(int argc, char *argv[])
 
             if (areaMeshDetected)
             {
-                areaBaseMeshPtr = faMeshTools::newMesh
+                //areaBaseMeshPtr = faMeshTools::newMesh
+                //(
+                //    IOobject
+                //    (
+                //        regionName,
+                //        baseRunTime.timeName(),
+                //        baseRunTime,
+                //        IOobjectOption::MUST_READ
+                //    ),
+                //    baseMeshPtr(),
+                //    true            // read on master only
+                //);
+                areaBaseMeshPtr = faMeshTools::loadOrCreateMesh
                 (
                     IOobject
                     (
                         regionName,
-                        baseRunTime.timeName(),
+                        baseMeshPtr().facesInstance(),
                         baseRunTime,
-                        IOobject::MUST_READ
+                        IOobjectOption::MUST_READ
                     ),
                     baseMeshPtr(),
-                    true            // read on master only
+                    masterOnlyHandler
                 );
 
+                //areaMeshPtr = faMeshTools::loadOrCreateMesh
+                //(
+                //    IOobject
+                //    (
+                //        regionName,
+                //        areaBaseMeshPtr().facesInstance(),
+                //        runTime,
+                //        IOobjectOption::MUST_READ
+                //    ),
+                //    mesh,
+                //    decompose
+                //);
                 areaMeshPtr = faMeshTools::loadOrCreateMesh
                 (
                     IOobject
@@ -1794,10 +2155,10 @@ int main(int argc, char *argv[])
                         regionName,
                         areaBaseMeshPtr().facesInstance(),
                         runTime,
-                        IOobject::MUST_READ
+                        IOobjectOption::MUST_READ
                     ),
                     mesh,
-                    decompose
+                    areaMeshReadHandler
                 );
 
                 areaDistMap =
@@ -1817,7 +2178,7 @@ int main(int argc, char *argv[])
                         areaMeshPtr(),      // source
                         areaBaseMeshPtr(),  // target
                         areaDistMap,
-                        Pstream::master()   // only write on master
+                        masterOnlyHandler   // only write on master
                     )
                 );
                 // Report some messages. Tbd.
@@ -1825,17 +2186,6 @@ int main(int argc, char *argv[])
             }
 
 
-            if (writeHandler && Pstream::master())
-            {
-                // Remove any left-over empty processor directories created
-                // by loadOrCreateMesh to get around e.g. collated start-up
-                // problems. Should not happen in reconstruct mode ...
-                const bool oldParRun = Pstream::parRun(false);
-                removeEmptyDirs(mesh.time().path());
-                Pstream::parRun(oldParRun);
-            }
-
-
             // Read addressing back to base mesh
             autoPtr<mapDistributePolyMesh> distMap;
             distMap = fvMeshTools::readProcAddressing(mesh, baseMeshPtr);
@@ -1844,10 +2194,10 @@ int main(int argc, char *argv[])
             auto fvDistributorPtr =
                 autoPtr<parFvFieldDistributor>::New
                 (
-                    mesh,            // source
-                    baseMeshPtr(),   // target
+                    mesh,               // source
+                    baseMeshPtr(),      // target
                     distMap(),
-                    UPstream::master()  // Write reconstructed on master
+                    masterOnlyHandler   // Write on master only
                 );
 
             // Construct point field mapper
@@ -1861,7 +2211,8 @@ int main(int argc, char *argv[])
                     basePointMesh,   // target
                     distMap(),
                     false,           // delay
-                    UPstream::master()  // Write reconstructed on master
+                    //UPstream::master()  // Write reconstructed on master
+                    masterOnlyHandler   // Write on master only
                 );
 
 
@@ -1871,7 +2222,8 @@ int main(int argc, char *argv[])
             // detect points by hand
             if (mesh.pointsInstance() != mesh.facesInstance())
             {
-                Info<< "    Detected initial mesh motion;"
+                InfoOrPout
+                    << "    Detected initial mesh motion;"
                     << " reconstructing points" << nl
                     << endl;
                 fvDistributorPtr().reconstructPoints();
@@ -1883,7 +2235,8 @@ int main(int argc, char *argv[])
             {
                 if (newTimes && masterTimeDirSet.found(timeDirs[timeI].name()))
                 {
-                    Info<< "Skipping time " << timeDirs[timeI].name()
+                    InfoOrPout
+                        << "Skipping time " << timeDirs[timeI].name()
                         << nl << endl;
                     continue;
                 }
@@ -1892,7 +2245,7 @@ int main(int argc, char *argv[])
                 runTime.setTime(timeDirs[timeI], timeI);
                 baseRunTime.setTime(timeDirs[timeI], timeI);
 
-                Info<< "Time = " << runTime.timeName() << endl << endl;
+                InfoOrPout<< "Time = " << runTime.timeName() << endl << endl;
 
 
                 // Check if any new meshes need to be read.
@@ -1900,7 +2253,8 @@ int main(int argc, char *argv[])
 
                 if (procStat == polyMesh::POINTS_MOVED)
                 {
-                    Info<< "    Detected mesh motion; reconstructing points"
+                    InfoOrPout
+                        << "    Detected mesh motion; reconstructing points"
                         << nl << endl;
                     fvDistributorPtr().reconstructPoints();
                 }
@@ -1910,7 +2264,8 @@ int main(int argc, char *argv[])
                  || procStat == polyMesh::TOPO_PATCH_CHANGE
                 )
                 {
-                    Info<< "    Detected topology change;"
+                    InfoOrPout
+                        << "    Detected topology change;"
                         << " reconstructing addressing" << nl << endl;
 
                     if (baseMeshPtr)
@@ -1919,17 +2274,36 @@ int main(int argc, char *argv[])
                         // processors will have mesh files. So instead just
                         // recreate baseMesh
                         baseMeshPtr.clear();
-                        baseMeshPtr = fvMeshTools::newMesh
+                        //baseMeshPtr = fvMeshTools::newMesh
+                        //(
+                        //    IOobject
+                        //    (
+                        //        regionName,
+                        //        baseRunTime.timeName(),
+                        //        baseRunTime,
+                        //        IOobjectOption::MUST_READ
+                        //    ),
+                        //    true            // read on master only
+                        //);
+                        baseMeshPtr = fvMeshTools::loadOrCreateMesh
                         (
                             IOobject
                             (
                                 regionName,
                                 baseRunTime.timeName(),
                                 baseRunTime,
-                                IOobject::MUST_READ
+                                IOobjectOption::MUST_READ
                             ),
-                            true            // read on master only
+                            masterOnlyHandler   // read on master only
                         );
+                        if (::debug)
+                        {
+                            Pout<< "Undecomposed mesh :"
+                                << " nCells:" << baseMeshPtr().nCells()
+                                << " nFaces:" << baseMeshPtr().nFaces()
+                                << " nPoints:" << baseMeshPtr().nPoints()
+                                << endl;
+                        }
                     }
 
                     // Re-read procAddressing
@@ -1945,7 +2319,7 @@ int main(int argc, char *argv[])
                             mesh,           // source
                             baseMeshPtr(),  // target
                             distMap(),
-                            UPstream::master()  // Write reconstruct on master
+                            masterOnlyHandler   // Write on master only
                         )
                     );
 
@@ -1961,7 +2335,7 @@ int main(int argc, char *argv[])
                             basePointMesh,  // target
                             distMap(),
                             false,          // delay until later
-                            UPstream::master()  // Write reconstruct on master
+                            masterOnlyHandler   // Write on master only
                         )
                     );
 
@@ -1969,7 +2343,8 @@ int main(int argc, char *argv[])
 
                     if (areaMeshPtr)
                     {
-                        Info<< "    Discarding finite-area addressing"
+                        InfoOrPout
+                            << "    Discarding finite-area addressing"
                             << " (TODO)" << nl << endl;
 
                         areaBaseMeshPtr.reset();
@@ -1983,7 +2358,6 @@ int main(int argc, char *argv[])
                 // Get list of objects
                 IOobjectList objects(mesh, runTime.timeName());
 
-
                 // Mesh fields (vol, surface, volInternal)
                 fvDistributorPtr()
                     .distributeAllFields(objects, selectedFields);
@@ -2002,6 +2376,7 @@ int main(int argc, char *argv[])
                     mesh,
                     distMap(),
                     selectedLagrangianFields
+                    //masterOnlyHandler
                 );
 
                 if (faDistributor)
@@ -2010,14 +2385,20 @@ int main(int argc, char *argv[])
                         .distributeAllFields(objects, selectedFields);
                 }
 
+
                 // If there are any "uniform" directories copy them from
                 // the master processor
+
                 copyUniform
                 (
-                    fh,
-                    decompose,
-                    reconstruct,
+                    volMeshReadHandler, //masterOnlyHandler,  // readHandler
+                    masterOnlyHandler,  // writeHandler
+
+                    true,               // reconstruct
+                    false,              // decompose
+
                     mesh.time().timeName(),
+                    word::null,         // optional caseName for reading
                     mesh,
                     baseMeshPtr()
                 );
@@ -2025,12 +2406,16 @@ int main(int argc, char *argv[])
                 // but would then have to replicate the whole time loop ...
                 copyUniform
                 (
-                    fh,
-                    decompose,
-                    reconstruct,
+                    volMeshReadHandler, //masterOnlyHandler,  // readHandler,
+                    masterOnlyHandler,  // writeHandler
+
+                    true,               // reconstruct
+                    false,              // decompose
+
                     mesh.time().timeName(),
-                    mesh.time(),                // runTime
-                    baseMeshPtr().time()        // baseRunTime
+                    word::null,         // optional caseName for reading
+                    mesh.time(),            // runTime
+                    baseMeshPtr().time()    // baseRunTime
                 );
             }
         }
@@ -2063,7 +2448,8 @@ int main(int argc, char *argv[])
             )[0].value();
         }
         Pstream::broadcast(masterTime);
-        Info<< "Setting time to that of master or undecomposed case : "
+        InfoOrPout
+            << "Setting time to that of master or undecomposed case : "
             << masterTime << endl;
         runTime.setTime(masterTime, 0);
         baseRunTime.setTime(masterTime, 0);
@@ -2079,7 +2465,8 @@ int main(int argc, char *argv[])
             const fileName volMeshSubDir(regionDir/polyMesh::meshSubDir);
             const fileName areaMeshSubDir(regionDir/faMesh::meshSubDir);
 
-            Info<< nl << nl
+            InfoOrPout
+                << nl << nl
                 << (decompose ? "Decomposing" : "Redistributing")
                 << " mesh " << regionDir << nl << endl;
 
@@ -2101,7 +2488,8 @@ int main(int argc, char *argv[])
             {
                 if (decompose)
                 {
-                    Info<< "Checking undecomposed mesh in case: "
+                    InfoOrPout
+                        << "Checking undecomposed mesh in case: "
                         << baseRunTime.caseName() << endl;
                     runTime.caseName() = baseRunTime.caseName();
                     runTime.processorCase(false);
@@ -2112,7 +2500,7 @@ int main(int argc, char *argv[])
                 (
                     volMeshSubDir,
                     "faces",
-                    IOobject::READ_IF_PRESENT
+                    IOobjectOption::READ_IF_PRESENT
                 );
 
                 if (doFiniteArea)
@@ -2121,7 +2509,7 @@ int main(int argc, char *argv[])
                     (
                         areaMeshSubDir,
                         "faceLabels",
-                        IOobject::READ_IF_PRESENT
+                        IOobjectOption::READ_IF_PRESENT
                     );
 
                     // Note: findInstance returns "constant" even if not found,
@@ -2151,7 +2539,8 @@ int main(int argc, char *argv[])
 
                 if (decompose)
                 {
-                    Info<< "    volume mesh ["
+                    InfoOrPout
+                        << "    volume mesh ["
                         << volMeshHaveUndecomposed << "] : "
                         << volMeshMasterInstance << nl
                         << "    area   mesh ["
@@ -2160,6 +2549,7 @@ int main(int argc, char *argv[])
                         << nl << nl;
 
                     // Restoring caseName
+                    InfoOrPout<< "Restoring caseName" << endl;
                     runTime.caseName() = proc0CaseName;
                     runTime.processorCase(oldProcCase);
                 }
@@ -2187,6 +2577,10 @@ int main(int argc, char *argv[])
                 "faces"
             );
 
+            // Create handler for reading
+            newHandler(volMeshOnProc, volMeshReadHandler);
+
+
             if (doFiniteArea)
             {
                 areaMeshOnProc = haveMeshFile
@@ -2195,6 +2589,20 @@ int main(int argc, char *argv[])
                     areaMeshMasterInstance/areaMeshSubDir,
                     "faceLabels"
                 );
+
+                // Create handler for reading
+                if (areaMeshOnProc == volMeshOnProc)
+                {
+                    if (volMeshReadHandler)
+                    {
+                        // Use same reader for faMesh as for fvMesh
+                        areaMeshReadHandler.ref(volMeshReadHandler.ref());
+                    }
+                }
+                else
+                {
+                    newHandler(areaMeshOnProc, areaMeshReadHandler);
+                }
             }
 
             // Prior to loadOrCreateMesh, note which meshes already exist
@@ -2281,15 +2689,15 @@ int main(int argc, char *argv[])
             // Load mesh (or create dummy one)
             // ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-            if (Pstream::master() && decompose)
+            if (decompose)
             {
-                Info<< "Setting caseName to " << baseRunTime.caseName()
+                InfoOrPout
+                    << "Setting caseName to " << baseRunTime.caseName()
                     << " to read undecomposed mesh" << endl;
                 runTime.caseName() = baseRunTime.caseName();
                 runTime.processorCase(false);
             }
 
-            // Volume mesh
             autoPtr<fvMesh> volMeshPtr = fvMeshTools::loadOrCreateMesh
             (
                 IOobject
@@ -2297,9 +2705,9 @@ int main(int argc, char *argv[])
                     regionName,
                     volMeshMasterInstance,
                     runTime,
-                    Foam::IOobject::MUST_READ
+                    IOobjectOption::MUST_READ
                 ),
-                decompose
+                volMeshReadHandler
             );
             fvMesh& mesh = volMeshPtr();
 
@@ -2321,6 +2729,18 @@ int main(int argc, char *argv[])
                 )
             )
             {
+                //areaMeshPtr = faMeshTools::loadOrCreateMesh
+                //(
+                //    IOobject
+                //    (
+                //        regionName,
+                //        areaMeshMasterInstance,
+                //        runTime,
+                //        IOobjectOption::MUST_READ
+                //    ),
+                //    mesh,  // <- The referenced polyMesh (from above)
+                //    decompose
+                //);
                 areaMeshPtr = faMeshTools::loadOrCreateMesh
                 (
                     IOobject
@@ -2328,10 +2748,10 @@ int main(int argc, char *argv[])
                         regionName,
                         areaMeshMasterInstance,
                         runTime,
-                        Foam::IOobject::MUST_READ
+                        IOobjectOption::MUST_READ
                     ),
                     mesh,  // <- The referenced polyMesh (from above)
-                    decompose
+                    areaMeshReadHandler
                 );
 
                 faMeshTools::forceDemandDriven(*areaMeshPtr);
@@ -2339,64 +2759,9 @@ int main(int argc, char *argv[])
             }
 
 
-            if (writeHandler)
+            if (decompose)
             {
-                // Remove any left-over empty processor directories created
-                // by loadOrCreateMesh to get around the collated start-up
-                // problems
-                Info<< "Removing left-over empty processor directories" << nl;
-
-                if (Pstream::master())  //fileHandler().comm()))
-                {
-                    const auto myProci = UPstream::myProcNo();  //comm()
-                    const auto& procs = UPstream::procID
-                    (
-                        UPstream::worldComm
-                    );
-                    const bool oldParRun = Pstream::parRun(false);
-                    for (const auto proci : procs)
-                    {
-                        if
-                        (
-                           !volMeshOnProc[proci]
-                         && volMeshDir[proci] != volMeshDir[myProci]
-                        )
-                        {
-                            Foam::rmDir(volMeshDir[proci], true); // silent
-                        }
-
-                        if
-                        (
-                            !areaMeshOnProc[proci]
-                         && areaMeshDir[proci] != areaMeshDir[myProci]
-                        )
-                        {
-                            Foam::rmDir(areaMeshDir[proci], true); // silent
-                        }
-
-                        // Remove empty processor directories
-                        // Eg, <path-name>/processorN/constant/polyMesh
-                        // to  <path-name>/processorN
-                        if (proci != myProci)
-                        {
-                            removeEmptyDir
-                            (
-                                volMeshDir[proci].path().path()
-                            );
-                        }
-                    }
-
-                    // Remove empty directory
-                    removeEmptyDirs(mesh.time().path());
-
-                    Pstream::parRun(oldParRun);
-                }
-            }
-
-
-            if (Pstream::master() && decompose)
-            {
-                Info<< "Restoring caseName" << endl;
+                InfoOrPout<< "Restoring caseName" << endl;
                 runTime.caseName() = proc0CaseName;
                 runTime.processorCase(oldProcCase);
             }
@@ -2417,6 +2782,7 @@ int main(int argc, char *argv[])
             labelList finalDecomp;
             determineDecomposition
             (
+                volMeshReadHandler,         // how to read decomposeParDict
                 baseRunTime,
                 decompDictFile,
                 decompose,
@@ -2431,29 +2797,20 @@ int main(int argc, char *argv[])
 
             if (dryrun)
             {
-                if (!Pstream::master())
-                {
-                    if (areaMeshPtr && !areaMeshOnProc[Pstream::myProcNo()])
-                    {
-                        // Remove dummy mesh created by loadOrCreateMesh
-                        const bool oldParRun = Pstream::parRun(false);
-                        areaMeshPtr->removeFiles();
-                        Pstream::parRun(oldParRun);  // Restore parallel state
-                    }
-
-                    if (!volMeshOnProc[Pstream::myProcNo()])
-                    {
-                        // Remove dummy mesh created by loadOrCreateMesh
-                        const bool oldParRun = Pstream::parRun(false);
-                        mesh.removeFiles();
-                        // Silent rmdir
-                        Foam::rmDir(mesh.objectRegistry::objectPath(), true);
-                        Pstream::parRun(oldParRun);  // Restore parallel state
-                    }
-                }
                 continue;
             }
 
+            if (!writeHandler && nDestProcs < fileHandler().nProcs())
+            {
+                boolList isWriteProc(UPstream::nProcs(), false);
+                isWriteProc.slice(0, nDestProcs) = true;
+                InfoOrPout
+                    << "    dest procs ["
+                    << isWriteProc << "]" << nl
+                    << endl;
+                newHandler(isWriteProc, writeHandler);
+            }
+
             // Area fields first. Read and deregister
             parFaFieldDistributorCache areaFields;
             if (areaMeshPtr)
@@ -2465,6 +2822,7 @@ int main(int argc, char *argv[])
                     decompose,
 
                     areaMeshOnProc,
+                    areaMeshReadHandler,
                     areaMeshMasterInstance,
                     (*areaMeshPtr)
                 );
@@ -2472,8 +2830,11 @@ int main(int argc, char *argv[])
 
 
             // Detect lagrangian fields
-            if (Pstream::master() && decompose)
+            if (decompose)
             {
+                InfoOrPout
+                    << "Setting caseName to " << baseRunTime.caseName()
+                    << " to read lagrangian" << endl;
                 runTime.caseName() = baseRunTime.caseName();
                 runTime.processorCase(false);
             }
@@ -2488,8 +2849,9 @@ int main(int argc, char *argv[])
                 )
             );
 
-            if (Pstream::master() && decompose)
+            if (decompose)
             {
+                InfoOrPout<< "Restoring caseName" << endl;
                 runTime.caseName() = proc0CaseName;
                 runTime.processorCase(oldProcCase);
             }
@@ -2498,7 +2860,8 @@ int main(int argc, char *argv[])
             // - but not lagrangian fields; these are done later
             autoPtr<mapDistributePolyMesh> distMap = redistributeAndWrite
             (
-                writeHandler,
+                volMeshReadHandler,         // readHandler
+                writeHandler,               // writeHandler,
                 baseRunTime,
                 proc0CaseName,
 
@@ -2554,17 +2917,19 @@ int main(int argc, char *argv[])
                 {
                     if (Pstream::master())
                     {
-                        Info<< "Setting caseName to " << baseRunTime.caseName()
+                        InfoOrPout
+                            << "Setting caseName to " << baseRunTime.caseName()
                             << " to write reconstructed mesh (and fields)."
                             << endl;
                         runTime.caseName() = baseRunTime.caseName();
                         const bool oldProcCase(runTime.processorCase(false));
-                        //const bool oldParRun = Pstream::parRun(false);
+                        const bool oldParRun = UPstream::parRun(false);
 
                         areaProcMeshPtr->write();
 
                         // Now we've written all. Reset caseName on master
-                        Info<< "Restoring caseName" << endl;
+                        InfoOrPout<< "Restoring caseName" << endl;
+                        UPstream::parRun(oldParRun);
                         runTime.caseName() = proc0CaseName;
                         runTime.processorCase(oldProcCase);
                     }
@@ -2581,9 +2946,9 @@ int main(int argc, char *argv[])
                             areaProcMeshPtr->facesInstance(),
                             faMesh::meshSubDir,
                             areaProcMeshPtr->thisDb(),
-                            IOobject::NO_READ,
-                            IOobject::NO_WRITE,
-                            IOobject::NO_REGISTER
+                            IOobjectOption::NO_READ,
+                            IOobjectOption::NO_WRITE,
+                            IOobjectOption::NO_REGISTER
                         ),
                         faDistMap
                     ).write();
@@ -2604,7 +2969,8 @@ int main(int argc, char *argv[])
                     }
                 }
 
-                Info<< "Written redistributed mesh to "
+                InfoOrPout
+                    << "Written redistributed mesh to "
                     << areaProcMeshPtr->facesInstance() << nl << endl;
 
                 faMeshDistributor distributor
@@ -2612,43 +2978,58 @@ int main(int argc, char *argv[])
                     areaMeshPtr(),      // source
                     areaProcMeshPtr(),  // target
                     faDistMap,
-                    true                // isWriteProc (unused)
+                    writeHandler
                 );
 
                 areaFields.redistributeAndWrite(distributor, true);
             }
 
-            // Copy region-specific uniform
-            // (e.g. solid/uniform/cumulativeContErr)
+
+            // Get reference to standard write handler
+            refPtr<fileOperation> defaultHandler;
+            defaultHandler.ref(const_cast<fileOperation&>(fileHandler()));
+
             copyUniform
             (
-                fh,
-                decompose,
-                reconstruct,
-                oldTimeName,    // provided read time
-                mesh,           // read location is mesh (but oldTimeName)
-                mesh            // write location is mesh
+                masterOnlyHandler,  // read handler
+                defaultHandler,     //TBD: should be all IOranks
+
+                reconstruct,        // reconstruct
+                decompose,          // decompose
+
+                oldTimeName,
+                (decompose ? baseRunTime.caseName() : proc0CaseName),
+                mesh,               // read location is mesh (but oldTimeName)
+                mesh                // write location is mesh
             );
         }
 
-        // Copy non-region specific uniform (e.g. uniform/time)
+
+        // Get reference to standard write handler
+        refPtr<fileOperation> defaultHandler;
+        defaultHandler.ref(const_cast<fileOperation&>(fileHandler()));
+
         copyUniform
         (
-            fh,
-            decompose,
-            reconstruct,
-            oldTimeName,    // provided read time
-            (               // read location
+            masterOnlyHandler,  // read handler
+            defaultHandler,     //TBD: should be all IOranks
+
+            reconstruct,        // reconstruct (=false)
+            decompose,          // decompose
+
+            oldTimeName,        // provided read time
+            (decompose ? baseRunTime.caseName() : proc0CaseName),
+            (                   // read location
                 decompose
               ? baseRunTime
               : runTime
             ),
-            runTime         // writing location
+            runTime             // writing location
         );
     }
 
 
-    Info<< "End\n" << endl;
+    InfoOrPout<< "End\n" << endl;
 
     return 0;
 }

From 259e8c22a54f0e3739e12428375651744203c056 Mon Sep 17 00:00:00 2001
From: mattijs <mattijs>
Date: Wed, 14 Jun 2023 13:17:01 +0100
Subject: [PATCH 06/12] ENH: improve check for decomposedBlock and simpler
 startup

---
 .../redistributePar/loadOrCreateMesh.C        | 74 ++++++++++++++++++-
 .../redistributePar/redistributePar.C         |  6 --
 2 files changed, 73 insertions(+), 7 deletions(-)

diff --git a/applications/utilities/parallelProcessing/redistributePar/loadOrCreateMesh.C b/applications/utilities/parallelProcessing/redistributePar/loadOrCreateMesh.C
index cf0ef53c03..95522b56c6 100644
--- a/applications/utilities/parallelProcessing/redistributePar/loadOrCreateMesh.C
+++ b/applications/utilities/parallelProcessing/redistributePar/loadOrCreateMesh.C
@@ -30,6 +30,7 @@ License
 #include "faMesh.H"
 #include "Pstream.H"
 #include "OSspecific.H"
+#include "decomposedBlockData.H"
 
 // * * * * * * * * * * * * * * * Global Functions  * * * * * * * * * * * * * //
 
@@ -47,7 +48,6 @@ bool Foam::checkFileExistence(const fileName& fName)
     //       (in collated format). Use file-based searching instead
 
     const auto& handler = Foam::fileHandler();
-
     typedef fileOperation::procRangeType procRangeType;
 
     fileName path, pDir, local;
@@ -136,8 +136,80 @@ Foam::boolList Foam::haveMeshFile
     const bool verbose
 )
 {
+    #if 0
+
+    // Simple directory scanning - too fragile
     bool found = checkFileExistence(runTime.path()/meshPath/meshFile);
 
+    #else
+
+    // Trimmed-down version of lookupAndCacheProcessorsPath
+    // with Foam::exists() check. No caching.
+
+    // Check for two conditions:
+    // - file has to exist
+    // - if collated the entry has to exist inside the file
+
+    // Note: bypass fileOperation::filePath(IOobject&) since has problems
+    //       with going to a different number of processors
+    //       (in collated format). Use file-based searching instead
+
+    const auto& handler = Foam::fileHandler();
+    typedef fileOperation::procRangeType procRangeType;
+
+    const fileName fName
+    (
+        handler.filePath(runTime.path()/meshPath/meshFile)
+    );
+
+    bool found = (!fName.empty() && handler.isFile(fName));
+
+    if (found)
+    {
+        autoPtr<ISstream> isPtr(fileHandler().NewIFstream(fName));
+        if (isPtr && isPtr->good())
+        {
+            auto& is = *isPtr;
+
+            IOobject io(meshFile, meshPath, runTime);
+            io.readHeader(is);
+
+            if (decomposedBlockData::isCollatedType(io))
+            {
+                fileName path, pDir, local;
+                procRangeType group;
+                label numProcs;
+                label proci = fileOperation::splitProcessorPath
+                (
+                    fName,
+                    path,
+                    pDir,
+                    local,
+                    group,
+                    numProcs
+                );
+
+                label myBlockNumber = 0;
+                if (proci == -1 && group.empty())
+                {
+                    // 'processorsXXX' format so contains all ranks
+                    // according to worldComm
+                    myBlockNumber = UPstream::myProcNo(UPstream::worldComm);
+                }
+                else
+                {
+                    // 'processorsXXX_n-m' format so check for the
+                    // relative rank
+                    myBlockNumber = UPstream::myProcNo(fileHandler().comm());
+                }
+
+                // Check if block for the local rank is inside file
+                found = decomposedBlockData::hasBlock(is, myBlockNumber);
+            }
+        }
+    }
+    #endif
+
     // Globally consistent information about who has a mesh
     boolList haveFileOnProc
     (
diff --git a/applications/utilities/parallelProcessing/redistributePar/redistributePar.C b/applications/utilities/parallelProcessing/redistributePar/redistributePar.C
index a566b9d89e..3faf8138b4 100644
--- a/applications/utilities/parallelProcessing/redistributePar/redistributePar.C
+++ b/applications/utilities/parallelProcessing/redistributePar/redistributePar.C
@@ -1451,10 +1451,6 @@ int main(int argc, char *argv[])
     Info<< "Disabling uncollated master reading" << nl << endl;
     IOobject::fileModificationChecking = IOobject::timeStamp;
 
-    // Set up uncollated file handler (still over all processors)
-    refPtr<fileOperation> basicHandler(fileOperation::NewUncollated());
-    auto oldHandler = fileOperation::fileHandler(basicHandler);
-
     // Replace #include "createTime.H" with our own version
     // that has MUST_READ instead of READ_MODIFIED
 
@@ -1468,8 +1464,6 @@ int main(int argc, char *argv[])
         IOobjectOption::MUST_READ  // Without watching
     );
 
-    // Restore file handler
-    (void) fileOperation::fileHandler(oldHandler);
 
     refPtr<fileOperation> writeHandler;
 

From 4183419be0b17bd0d1d95caae93855ba67d5877d Mon Sep 17 00:00:00 2001
From: mattijs <mattijs>
Date: Tue, 23 May 2023 15:19:15 +0200
Subject: [PATCH 07/12] TUT: redistributePar

---
 tutorials/IO/fileHandler/Allrun               |   3 +
 tutorials/IO/fileHandler/Test-distributed     |   5 +-
 tutorials/IO/fileHandler_dynamicCode/0/U      |  41 +++++++
 tutorials/IO/fileHandler_dynamicCode/0/p      |  40 +++++++
 .../constant/transportProperties              |  20 ++++
 .../system/blockMeshDict                      |  72 ++++++++++++
 .../system/controlDict                        | 111 ++++++++++++++++++
 .../system/decomposeParDict                   |  27 +++++
 .../fileHandler_dynamicCode/system/fvSchemes  |  50 ++++++++
 .../fileHandler_dynamicCode/system/fvSolution |  51 ++++++++
 .../system/solverControls                     |  11 ++
 .../system/solverControls.0                   |  11 ++
 .../system/solverControls.5                   |  11 ++
 .../iglooWithFridges/system/decomposeParDict  |   4 +-
 14 files changed, 453 insertions(+), 4 deletions(-)
 create mode 100644 tutorials/IO/fileHandler_dynamicCode/0/U
 create mode 100644 tutorials/IO/fileHandler_dynamicCode/0/p
 create mode 100644 tutorials/IO/fileHandler_dynamicCode/constant/transportProperties
 create mode 100644 tutorials/IO/fileHandler_dynamicCode/system/blockMeshDict
 create mode 100644 tutorials/IO/fileHandler_dynamicCode/system/controlDict
 create mode 100644 tutorials/IO/fileHandler_dynamicCode/system/decomposeParDict
 create mode 100644 tutorials/IO/fileHandler_dynamicCode/system/fvSchemes
 create mode 100644 tutorials/IO/fileHandler_dynamicCode/system/fvSolution
 create mode 100644 tutorials/IO/fileHandler_dynamicCode/system/solverControls
 create mode 100644 tutorials/IO/fileHandler_dynamicCode/system/solverControls.0
 create mode 100644 tutorials/IO/fileHandler_dynamicCode/system/solverControls.5

diff --git a/tutorials/IO/fileHandler/Allrun b/tutorials/IO/fileHandler/Allrun
index 02a76ea205..16be0ee264 100755
--- a/tutorials/IO/fileHandler/Allrun
+++ b/tutorials/IO/fileHandler/Allrun
@@ -21,11 +21,14 @@ application=$(getApplication)
 
 #- Test writing collated format
 runApplication decomposePar -fileHandler collated
+#runParallel redistributePar -decompose -fileHandler collated
 runParallel $application -fileHandler collated
 runApplication reconstructPar -latestTime -fileHandler collated
+#runParallel -s reconstruct redistributePar -reconstruct -latestTime -fileHandler collated
 
 #- Test writing uncollated format
 runApplication -s uncollated decomposePar -fileHandler uncollated -force
+#runParallel -s uncollated redistributePar -decompose -fileHandler uncollated
 runParallel -s uncollated $application -fileHandler uncollated
 
 
diff --git a/tutorials/IO/fileHandler/Test-distributed b/tutorials/IO/fileHandler/Test-distributed
index 462f6bbd9a..95e9bcdf2e 100755
--- a/tutorials/IO/fileHandler/Test-distributed
+++ b/tutorials/IO/fileHandler/Test-distributed
@@ -46,7 +46,10 @@ CASE_ROOTS
 
 #export FOAM_ABORT=true
 
-runParallel -s decompose redistributePar -decompose -case test-distribute/machineA/testcase
+runParallel -s decompose redistributePar \
+    -decompose \
+    -case test-distribute/machineA/testcase \
+    -fileHandler masterUncollated
 
 runParallel checkMesh -case test-distribute/machineA/testcase
 
diff --git a/tutorials/IO/fileHandler_dynamicCode/0/U b/tutorials/IO/fileHandler_dynamicCode/0/U
new file mode 100644
index 0000000000..f525dbc89f
--- /dev/null
+++ b/tutorials/IO/fileHandler_dynamicCode/0/U
@@ -0,0 +1,41 @@
+/*--------------------------------*- C++ -*----------------------------------*\
+| =========                 |                                                 |
+| \\      /  F ield         | OpenFOAM: The Open Source CFD Toolbox           |
+|  \\    /   O peration     | Version:  v2312                                 |
+|   \\  /    A nd           | Website:  www.openfoam.com                      |
+|    \\/     M anipulation  |                                                 |
+\*---------------------------------------------------------------------------*/
+FoamFile
+{
+    version     2.0;
+    format      ascii;
+    class       volVectorField;
+    object      U;
+}
+// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
+
+dimensions      [0 1 -1 0 0 0 0];
+
+internalField   uniform (0 0 0);
+
+boundaryField
+{
+    movingWall
+    {
+        type            fixedValue;
+        value           uniform (1 0 0);
+    }
+
+    fixedWalls
+    {
+        type            noSlip;
+    }
+
+    frontAndBack
+    {
+        type            empty;
+    }
+}
+
+
+// ************************************************************************* //
diff --git a/tutorials/IO/fileHandler_dynamicCode/0/p b/tutorials/IO/fileHandler_dynamicCode/0/p
new file mode 100644
index 0000000000..b2996b1aea
--- /dev/null
+++ b/tutorials/IO/fileHandler_dynamicCode/0/p
@@ -0,0 +1,40 @@
+/*--------------------------------*- C++ -*----------------------------------*\
+| =========                 |                                                 |
+| \\      /  F ield         | OpenFOAM: The Open Source CFD Toolbox           |
+|  \\    /   O peration     | Version:  v2312                                 |
+|   \\  /    A nd           | Website:  www.openfoam.com                      |
+|    \\/     M anipulation  |                                                 |
+\*---------------------------------------------------------------------------*/
+FoamFile
+{
+    version     2.0;
+    format      ascii;
+    class       volScalarField;
+    object      p;
+}
+// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
+
+dimensions      [0 2 -2 0 0 0 0];
+
+internalField   uniform 0;
+
+boundaryField
+{
+    movingWall
+    {
+        type            zeroGradient;
+    }
+
+    fixedWalls
+    {
+        type            zeroGradient;
+    }
+
+    frontAndBack
+    {
+        type            empty;
+    }
+}
+
+
+// ************************************************************************* //
diff --git a/tutorials/IO/fileHandler_dynamicCode/constant/transportProperties b/tutorials/IO/fileHandler_dynamicCode/constant/transportProperties
new file mode 100644
index 0000000000..a3caf60520
--- /dev/null
+++ b/tutorials/IO/fileHandler_dynamicCode/constant/transportProperties
@@ -0,0 +1,20 @@
+/*--------------------------------*- C++ -*----------------------------------*\
+| =========                 |                                                 |
+| \\      /  F ield         | OpenFOAM: The Open Source CFD Toolbox           |
+|  \\    /   O peration     | Version:  v2312                                 |
+|   \\  /    A nd           | Website:  www.openfoam.com                      |
+|    \\/     M anipulation  |                                                 |
+\*---------------------------------------------------------------------------*/
+FoamFile
+{
+    version     2.0;
+    format      ascii;
+    class       dictionary;
+    object      transportProperties;
+}
+// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
+
+nu              0.01;
+
+
+// ************************************************************************* //
diff --git a/tutorials/IO/fileHandler_dynamicCode/system/blockMeshDict b/tutorials/IO/fileHandler_dynamicCode/system/blockMeshDict
new file mode 100644
index 0000000000..c2da56f474
--- /dev/null
+++ b/tutorials/IO/fileHandler_dynamicCode/system/blockMeshDict
@@ -0,0 +1,72 @@
+/*--------------------------------*- C++ -*----------------------------------*\
+| =========                 |                                                 |
+| \\      /  F ield         | OpenFOAM: The Open Source CFD Toolbox           |
+|  \\    /   O peration     | Version:  v2312                                 |
+|   \\  /    A nd           | Website:  www.openfoam.com                      |
+|    \\/     M anipulation  |                                                 |
+\*---------------------------------------------------------------------------*/
+FoamFile
+{
+    version     2.0;
+    format      ascii;
+    class       dictionary;
+    object      blockMeshDict;
+}
+// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
+
+scale   0.1;
+
+vertices
+(
+    (0 0 0)
+    (1 0 0)
+    (1 1 0)
+    (0 1 0)
+    (0 0 0.1)
+    (1 0 0.1)
+    (1 1 0.1)
+    (0 1 0.1)
+);
+
+blocks
+(
+    hex (0 1 2 3 4 5 6 7) (5 5 1) simpleGrading (1 1 1)
+);
+
+edges
+(
+);
+
+boundary
+(
+    movingWall
+    {
+        type wall;
+        faces
+        (
+            (3 7 6 2)
+        );
+    }
+    fixedWalls
+    {
+        type wall;
+        faces
+        (
+            (0 4 7 3)
+            (2 6 5 1)
+            (1 5 4 0)
+        );
+    }
+    frontAndBack
+    {
+        type empty;
+        faces
+        (
+            (0 3 2 1)
+            (4 5 6 7)
+        );
+    }
+);
+
+
+// ************************************************************************* //
diff --git a/tutorials/IO/fileHandler_dynamicCode/system/controlDict b/tutorials/IO/fileHandler_dynamicCode/system/controlDict
new file mode 100644
index 0000000000..4d2efdabad
--- /dev/null
+++ b/tutorials/IO/fileHandler_dynamicCode/system/controlDict
@@ -0,0 +1,111 @@
+/*--------------------------------*- C++ -*----------------------------------*\
+| =========                 |                                                 |
+| \\      /  F ield         | OpenFOAM: The Open Source CFD Toolbox           |
+|  \\    /   O peration     | Version:  v2312                                 |
+|   \\  /    A nd           | Website:  www.openfoam.com                      |
+|    \\/     M anipulation  |                                                 |
+\*---------------------------------------------------------------------------*/
+FoamFile
+{
+    version     2.0;
+    format      ascii;
+    class       dictionary;
+    object      controlDict;
+}
+// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
+
+DebugSwitches
+{
+//    OSspecific  1;
+//    POSIX       2;
+//    codeStream  1;
+//    codedBase   1;
+//    masterUncollated    1;
+//    hostCollated      1;
+//    collated        1;
+//    fileMonitor 1;
+}
+
+OptimisationSwitches
+{
+    fileModificationSkew    1;
+}
+
+//- Set deltaT
+#include "solverControls"
+
+
+application     icoFoam;
+
+startFrom       startTime;
+
+startTime       0;
+
+stopAt          endTime;
+//stopAt          #codeStream
+//{
+//    code
+//    #{
+//        os  << "nextWrite;" << nl;
+//    #};
+//};
+
+
+endTime         0.04;
+
+//deltaT          0.005;
+
+writeControl    timeStep;
+
+writeInterval   20;
+
+purgeWrite      0;
+
+writeFormat     ascii;
+
+writePrecision  6;
+
+writeCompression off;
+
+timeFormat      general;
+
+timePrecision   6;
+
+runTimeModifiable true;
+
+functions
+{
+    fileUpdate
+    {
+        type                timeActivatedFileUpdate;
+        libs                (utilityFunctionObjects);
+        writeControl        timeStep;
+        writeInterval       1;
+        fileToUpdate        "<system>/solverControls";
+
+        timeVsFile
+        (
+            (-1     "<system>/solverControls.0" )
+            ( 0.02  "<system>/solverControls.5" )
+        );
+    }
+
+    // Wait a bit to make runTimeModifiable work.
+    //  - see also fileModificationSkew
+    //  - could also directly use system FO
+    sleep
+    {
+        type                coded;
+        libs                (utilityFunctionObjects);
+        name                sleep;
+        writeControl        timeStep;
+
+        codeExecute
+        #{
+            sleep(1);
+        #};
+    }
+}
+
+
+// ************************************************************************* //
diff --git a/tutorials/IO/fileHandler_dynamicCode/system/decomposeParDict b/tutorials/IO/fileHandler_dynamicCode/system/decomposeParDict
new file mode 100644
index 0000000000..deb7d98a7f
--- /dev/null
+++ b/tutorials/IO/fileHandler_dynamicCode/system/decomposeParDict
@@ -0,0 +1,27 @@
+/*--------------------------------*- C++ -*----------------------------------*\
+| =========                 |                                                 |
+| \\      /  F ield         | OpenFOAM: The Open Source CFD Toolbox           |
+|  \\    /   O peration     | Version:  v2312                                 |
+|   \\  /    A nd           | Website:  www.openfoam.com                      |
+|    \\/     M anipulation  |                                                 |
+\*---------------------------------------------------------------------------*/
+FoamFile
+{
+    version     2.0;
+    format      ascii;
+    class       dictionary;
+    object      decomposeParDict;
+}
+// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
+
+numberOfSubdomains  5;
+
+method  hierarchical;
+
+coeffs
+{
+    n   (5 1 1);
+}
+
+
+// ************************************************************************* //
diff --git a/tutorials/IO/fileHandler_dynamicCode/system/fvSchemes b/tutorials/IO/fileHandler_dynamicCode/system/fvSchemes
new file mode 100644
index 0000000000..2a4d2c9557
--- /dev/null
+++ b/tutorials/IO/fileHandler_dynamicCode/system/fvSchemes
@@ -0,0 +1,50 @@
+/*--------------------------------*- C++ -*----------------------------------*\
+| =========                 |                                                 |
+| \\      /  F ield         | OpenFOAM: The Open Source CFD Toolbox           |
+|  \\    /   O peration     | Version:  v2312                                 |
+|   \\  /    A nd           | Website:  www.openfoam.com                      |
+|    \\/     M anipulation  |                                                 |
+\*---------------------------------------------------------------------------*/
+FoamFile
+{
+    version     2.0;
+    format      ascii;
+    class       dictionary;
+    object      fvSchemes;
+}
+// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
+
+ddtSchemes
+{
+    default         Euler;
+}
+
+gradSchemes
+{
+    default         Gauss linear;
+    grad(p)         Gauss linear;
+}
+
+divSchemes
+{
+    default         none;
+    div(phi,U)      Gauss linear;
+}
+
+laplacianSchemes
+{
+    default         Gauss linear orthogonal;
+}
+
+interpolationSchemes
+{
+    default         linear;
+}
+
+snGradSchemes
+{
+    default         orthogonal;
+}
+
+
+// ************************************************************************* //
diff --git a/tutorials/IO/fileHandler_dynamicCode/system/fvSolution b/tutorials/IO/fileHandler_dynamicCode/system/fvSolution
new file mode 100644
index 0000000000..059a9f375c
--- /dev/null
+++ b/tutorials/IO/fileHandler_dynamicCode/system/fvSolution
@@ -0,0 +1,51 @@
+/*--------------------------------*- C++ -*----------------------------------*\
+| =========                 |                                                 |
+| \\      /  F ield         | OpenFOAM: The Open Source CFD Toolbox           |
+|  \\    /   O peration     | Version:  v2312                                 |
+|   \\  /    A nd           | Website:  www.openfoam.com                      |
+|    \\/     M anipulation  |                                                 |
+\*---------------------------------------------------------------------------*/
+FoamFile
+{
+    version     2.0;
+    format      ascii;
+    class       dictionary;
+    object      fvSolution;
+}
+// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
+
+solvers
+{
+    p
+    {
+        solver          PCG;
+        preconditioner  DIC;
+        tolerance       1e-06;
+        relTol          0.05;
+    }
+
+    pFinal
+    {
+        $p;
+        relTol          0;
+    }
+
+    U
+    {
+        solver          smoothSolver;
+        smoother        symGaussSeidel;
+        tolerance       1e-05;
+        relTol          0;
+    }
+}
+
+PISO
+{
+    nCorrectors     2;
+    nNonOrthogonalCorrectors 0;
+    pRefCell        0;
+    pRefValue       0;
+}
+
+
+// ************************************************************************* //
diff --git a/tutorials/IO/fileHandler_dynamicCode/system/solverControls b/tutorials/IO/fileHandler_dynamicCode/system/solverControls
new file mode 100644
index 0000000000..cd39033a9a
--- /dev/null
+++ b/tutorials/IO/fileHandler_dynamicCode/system/solverControls
@@ -0,0 +1,11 @@
+/*--------------------------------*- C++ -*----------------------------------*\
+| =========                 |                                                 |
+| \\      /  F ield         | OpenFOAM: The Open Source CFD Toolbox           |
+|  \\    /   O peration     | Version:  v2312                                 |
+|   \\  /    A nd           | Website:  www.openfoam.com                      |
+|    \\/     M anipulation  |                                                 |
+\*---------------------------------------------------------------------------*/
+
+deltaT          0.001;
+
+// ************************************************************************* //
diff --git a/tutorials/IO/fileHandler_dynamicCode/system/solverControls.0 b/tutorials/IO/fileHandler_dynamicCode/system/solverControls.0
new file mode 100644
index 0000000000..418c6b27d6
--- /dev/null
+++ b/tutorials/IO/fileHandler_dynamicCode/system/solverControls.0
@@ -0,0 +1,11 @@
+/*--------------------------------*- C++ -*----------------------------------*\
+| =========                 |                                                 |
+| \\      /  F ield         | OpenFOAM: The Open Source CFD Toolbox           |
+|  \\    /   O peration     | Version:  v2312                                 |
+|   \\  /    A nd           | Website:  www.openfoam.com                      |
+|    \\/     M anipulation  |                                                 |
+\*---------------------------------------------------------------------------*/
+
+deltaT          0.005;
+
+// ************************************************************************* //
diff --git a/tutorials/IO/fileHandler_dynamicCode/system/solverControls.5 b/tutorials/IO/fileHandler_dynamicCode/system/solverControls.5
new file mode 100644
index 0000000000..cd39033a9a
--- /dev/null
+++ b/tutorials/IO/fileHandler_dynamicCode/system/solverControls.5
@@ -0,0 +1,11 @@
+/*--------------------------------*- C++ -*----------------------------------*\
+| =========                 |                                                 |
+| \\      /  F ield         | OpenFOAM: The Open Source CFD Toolbox           |
+|  \\    /   O peration     | Version:  v2312                                 |
+|   \\  /    A nd           | Website:  www.openfoam.com                      |
+|    \\/     M anipulation  |                                                 |
+\*---------------------------------------------------------------------------*/
+
+deltaT          0.001;
+
+// ************************************************************************* //
diff --git a/tutorials/heatTransfer/buoyantBoussinesqSimpleFoam/iglooWithFridges/system/decomposeParDict b/tutorials/heatTransfer/buoyantBoussinesqSimpleFoam/iglooWithFridges/system/decomposeParDict
index cac126a2fb..5a7e0bb079 100644
--- a/tutorials/heatTransfer/buoyantBoussinesqSimpleFoam/iglooWithFridges/system/decomposeParDict
+++ b/tutorials/heatTransfer/buoyantBoussinesqSimpleFoam/iglooWithFridges/system/decomposeParDict
@@ -1,7 +1,7 @@
 /*--------------------------------*- C++ -*----------------------------------*\
 | =========                 |                                                 |
 | \\      /  F ield         | OpenFOAM: The Open Source CFD Toolbox           |
-|  \\    /   O peration     | Version:  v2306                                 |
+|  \\    /   O peration     | Version:  v2312                                 |
 |   \\  /    A nd           | Website:  www.openfoam.com                      |
 |    \\/     M anipulation  |                                                 |
 \*---------------------------------------------------------------------------*/
@@ -14,8 +14,6 @@ FoamFile
 }
 // * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
 
-numberOfSubdomains 2;
-
 method             scotch;
 
 

From af267e0c1b118e50d873d625a450fea3fe7ab2db Mon Sep 17 00:00:00 2001
From: mattijs <mattijs>
Date: Thu, 6 Jul 2023 16:18:12 +0100
Subject: [PATCH 08/12] BUG: redistributePar: excessive directories for
 collated. Fixes #2936.

BUG: collated: cooperate with non-parallel
---
 .../collatedFileOperation/collatedFileOperation.C | 15 ++++++++++-----
 1 file changed, 10 insertions(+), 5 deletions(-)

diff --git a/src/OpenFOAM/global/fileOperations/collatedFileOperation/collatedFileOperation.C b/src/OpenFOAM/global/fileOperations/collatedFileOperation/collatedFileOperation.C
index 2318c474d7..22a2a5616b 100644
--- a/src/OpenFOAM/global/fileOperations/collatedFileOperation/collatedFileOperation.C
+++ b/src/OpenFOAM/global/fileOperations/collatedFileOperation/collatedFileOperation.C
@@ -386,8 +386,10 @@ bool Foam::fileOperations::collatedFileOperation::writeObject
 
     if (inst.isAbsolute() || !tm.processorCase())
     {
-        mkDir(io.path());
-        fileName pathName(io.objectPath());
+        // Note: delay mkdir to masterOFstream so it does not get created
+        //       if not needed (e.g. when running distributed)
+
+        const fileName pathName(io.objectPath());
 
         if (debug)
         {
@@ -427,10 +429,12 @@ bool Foam::fileOperations::collatedFileOperation::writeObject
     else
     {
         // Construct the equivalent processors/ directory
-        fileName path(processorsPath(io, inst, processorsDir(io)));
+        const fileName path(processorsPath(io, inst, processorsDir(io)));
 
-        mkDir(path);
-        fileName pathName(path/io.name());
+        // Note: delay mkdir to masterOFstream so it does not get created
+        //       if not needed (e.g. when running distributed)
+
+        const fileName pathName(path/io.name());
 
         if (io.global() || io.globalObject())
         {
@@ -480,6 +484,7 @@ bool Foam::fileOperations::collatedFileOperation::writeObject
                     << " appending to " << pathName << endl;
             }
 
+            mkDir(path);
             return appendObject(io, pathName, streamOpt);
         }
         else

From 9ed0bb1e4f3de956f14e1ae6c81f139ac9c6caec Mon Sep 17 00:00:00 2001
From: mattijs <mattijs>
Date: Wed, 4 Oct 2023 17:15:12 +0100
Subject: [PATCH 09/12] ENH: redistributePar: update for distribute mode

- uses read&write handlers
- fixes initialisation of partial handlers
- 'uniform' copied to wrong place
- -overwrite not deleting old processors dirs
---
 .../redistributePar/loadOrCreateMesh.C        |  5 +-
 src/OpenFOAM/db/Time/Time.C                   |  6 +-
 .../collatedFileOperation.C                   | 38 +----------
 .../fileOperation/fileOperation.C             | 65 ++++++++++++++-----
 .../fileOperation/fileOperation.H             | 22 +++++++
 .../fileOperation/fileOperationNew.C          | 55 +++++++++++++++-
 .../fileOperation/fileOperationRanks.C        |  2 -
 .../uncollatedFileOperation.C                 |  7 +-
 .../uncollatedFileOperation.H                 |  2 +-
 .../fvMesh/fvMeshTools/fvMeshTools.C          | 25 ++-----
 10 files changed, 141 insertions(+), 86 deletions(-)

diff --git a/applications/utilities/parallelProcessing/redistributePar/loadOrCreateMesh.C b/applications/utilities/parallelProcessing/redistributePar/loadOrCreateMesh.C
index 95522b56c6..aae5cedb13 100644
--- a/applications/utilities/parallelProcessing/redistributePar/loadOrCreateMesh.C
+++ b/applications/utilities/parallelProcessing/redistributePar/loadOrCreateMesh.C
@@ -162,9 +162,8 @@ Foam::boolList Foam::haveMeshFile
         handler.filePath(runTime.path()/meshPath/meshFile)
     );
 
-    bool found = (!fName.empty() && handler.isFile(fName));
-
-    if (found)
+    bool found = handler.isFile(fName);
+    if (returnReduceAnd(found)) // worldComm
     {
         autoPtr<ISstream> isPtr(fileHandler().NewIFstream(fName));
         if (isPtr && isPtr->good())
diff --git a/src/OpenFOAM/db/Time/Time.C b/src/OpenFOAM/db/Time/Time.C
index e4912421da..2b1673a1f5 100644
--- a/src/OpenFOAM/db/Time/Time.C
+++ b/src/OpenFOAM/db/Time/Time.C
@@ -202,7 +202,8 @@ void Foam::Time::setControls()
     //       alternative processorsDDD directories naming
     //       - do not look for compressed (is directory)
 
-    if (fileHandler().filePath(timePath(), false).empty())
+    const fileName timeDir(fileHandler().filePath(timePath(), false));
+    if (returnReduceAnd(timeDir.empty()))
     {
         const int oldPrecision = precision_;
         int requiredPrecision = -1;
@@ -225,7 +226,8 @@ void Foam::Time::setControls()
             oldTime = std::move(newTime);
 
             // Does a time directory exist with the new format?
-            if (!fileHandler().filePath(timePath(), false).empty())
+            const fileName timeDir(fileHandler().filePath(timePath(), false));
+            if (returnReduceOr(!timeDir.empty()))
             {
                 requiredPrecision = precision_;
             }
diff --git a/src/OpenFOAM/global/fileOperations/collatedFileOperation/collatedFileOperation.C b/src/OpenFOAM/global/fileOperations/collatedFileOperation/collatedFileOperation.C
index 22a2a5616b..be88c0c135 100644
--- a/src/OpenFOAM/global/fileOperations/collatedFileOperation/collatedFileOperation.C
+++ b/src/OpenFOAM/global/fileOperations/collatedFileOperation/collatedFileOperation.C
@@ -113,43 +113,7 @@ void Foam::fileOperations::collatedFileOperation::printBanner
 
     if (withRanks)
     {
-        // Information about the ranks
-        stringList hosts(Pstream::nProcs());
-        if (Pstream::master(comm_))
-        {
-            hosts[Pstream::myProcNo()] = hostName();
-        }
-        Pstream::gatherList(hosts);
-
-        DynamicList<label> offsetMaster(Pstream::nProcs());
-
-        forAll(hosts, ranki)
-        {
-            if (!hosts[ranki].empty())
-            {
-                offsetMaster.append(ranki);
-            }
-        }
-
-        if (offsetMaster.size() > 1)
-        {
-            DetailInfo
-                << "IO nodes:" << nl << '(' << nl;
-
-            offsetMaster.append(Pstream::nProcs());
-
-            for (label group = 1; group < offsetMaster.size(); ++group)
-            {
-                const label beg = offsetMaster[group-1];
-                const label end = offsetMaster[group];
-
-                DetailInfo
-                    << "    (" << hosts[beg].c_str() << ' '
-                    << (end-beg) << ')' << nl;
-            }
-            DetailInfo
-                << ')' << nl;
-        }
+        fileOperation::printRanks();
     }
 
     //- fileModificationChecking already set by base class (masterUncollated)
diff --git a/src/OpenFOAM/global/fileOperations/fileOperation/fileOperation.C b/src/OpenFOAM/global/fileOperations/fileOperation/fileOperation.C
index 059aa20391..bbca362e0d 100644
--- a/src/OpenFOAM/global/fileOperations/fileOperation/fileOperation.C
+++ b/src/OpenFOAM/global/fileOperations/fileOperation/fileOperation.C
@@ -80,6 +80,7 @@ Foam::word Foam::fileOperation::processorsBaseDir = "processors";
 //- Caching (e.g. of time directories) - enabled by default
 int Foam::fileOperation::cacheLevel_(1);
 
+int Foam::fileOperation::nProcsFilter_(0);
 
 // * * * * * * * * * * * * * * * Local Functions * * * * * * * * * * * * * * //
 
@@ -179,8 +180,6 @@ static bool parseProcsNumRange
 } // End anonymous namespace
 
 
-#if 0
-
 // Sorting of processor directories
 #include "stringOpsSort.H"
 namespace
@@ -214,7 +213,6 @@ void sortProcessorDirs(Foam::UList<Foam::fileOperation::dirIndex>& dirs)
 }
 
 } // End anonymous namespace
-#endif
 
 
 // * * * * * * * * * * * * * Static Member Functions * * * * * * * * * * * * //
@@ -432,6 +430,16 @@ Foam::fileOperation::lookupAndCacheProcessorsPath
     // - distributed() : different processors have different roots
     // - fileModificationChecking : (uncollated only) do IO on master only
 
+
+    // Collated : check whether/how to filter processorsXXX directory names
+    const label targetNProcs
+    (
+        (UPstream::parRun() && nProcsFilter_ < 0)
+      ? UPstream::nProcs(UPstream::worldComm)
+      : nProcsFilter_
+    );
+
+
     fileName path, pDir, local;
     procRangeType group;
     label numProcs;
@@ -528,6 +536,19 @@ Foam::fileOperation::lookupAndCacheProcessorsPath
             }
             else if (rNum != -1)
             {
+                if (targetNProcs > 1 && (targetNProcs != rNum))
+                {
+                    // Current directory can never contain wanted proc
+                    //Pout<< "For fName:" << fName
+                    //    << "Ignoring directory " << dirN
+                    //    << " since parsed rNum:" << rNum
+                    //    << " targetNProcs:" << targetNProcs
+                    //   << endl;
+                    //error::printStack(Pout);
+
+                    continue;
+                }
+
                 // "processorsNN" or "processorsNN_start-end"
                 nProcs = max(nProcs, rNum);
 
@@ -571,7 +592,7 @@ Foam::fileOperation::lookupAndCacheProcessorsPath
         if (debug)
         {
             Pout<< "fileOperation::lookupProcessorsPath " << procPath
-                << " detected:" << procDirs << endl;
+                << " detected:" << flatOutput(procDirs) << endl;
         }
 
         if (UPstream::parRun() && (!distributed() || syncPar))
@@ -655,7 +676,7 @@ Foam::fileOperation::lookupAndCacheProcessorsPath
         }
 
         // Sort processor directory names (natural order)
-        /// sortProcessorDirs(procDirs);
+        sortProcessorDirs(procDirs);
 
         if (procDirsStatus & 2u)
         {
@@ -680,7 +701,6 @@ Foam::fileOperation::lookupAndCacheProcessorsPath
 Foam::refPtr<Foam::fileOperation::dirIndexList>
 Foam::fileOperation::lookupProcessorsPath(const fileName& fName) const
 {
-    // Use parallel synchronisation
     return lookupAndCacheProcessorsPath(fName, true);
 }
 
@@ -844,6 +864,7 @@ Foam::fileName Foam::fileOperation::filePath
     }
 
     // Give preference to processors variant
+    fileName foundName;
     if (proci != -1)
     {
         // Get all processor directories
@@ -859,26 +880,40 @@ Foam::fileName Foam::fileOperation::filePath
                 {
                     Pout<< "fileOperation::filePath : " << collatedName << endl;
                 }
-                return collatedName;
+                foundName = collatedName;
             }
         }
     }
 
-    if (exists(fName, checkGzip, followLink))
+    //if (returnReduceOr(foundName.empty()))    // worldComm
+    if (foundName.empty())
+    {
+        // There is at least one processor that cannot find the processors
+        // directory. Re-do with straight supplied filename
+        if (exists(fName, checkGzip, followLink))
+        {
+            if (foundName.empty())
+            {
+                foundName = fName;
+            }
+        }
+    }
+
+    if (!foundName.empty())
     {
         if (debug)
         {
-            Pout<< "fileOperation::filePath : " << fName << endl;
+            Pout<< "fileOperation::filePath : " << foundName << endl;
         }
-        return fName;
     }
-
-    if (debug)
+    else
     {
-        Pout<< "fileOperation::filePath : Not found" << endl;
+        if (debug)
+        {
+            Pout<< "fileOperation::filePath : Not found" << endl;
+        }
     }
-
-    return fileName();
+    return foundName;
 }
 
 
diff --git a/src/OpenFOAM/global/fileOperations/fileOperation/fileOperation.H b/src/OpenFOAM/global/fileOperations/fileOperation/fileOperation.H
index 0bf282312d..5de0a2a149 100644
--- a/src/OpenFOAM/global/fileOperations/fileOperation/fileOperation.H
+++ b/src/OpenFOAM/global/fileOperations/fileOperation/fileOperation.H
@@ -150,6 +150,13 @@ private:
         //- Storage of the dummy file handler (demand-driven)
         static refPtr<fileOperation> dummyHandlerPtr_;
 
+        //- Filtering of processorsDDD directories (in parallel):
+        //      - >0 : only accept processorsDDD (default)
+        //      -  0 : accept anything (e.g. when detecting nprocs from
+        //             directory naming)
+        //      - -1 : use nProcs(UPstream::worldComm)
+        static int nProcsFilter_;
+
 
 protected:
 
@@ -452,6 +459,21 @@ public:
             return old;
         }
 
+        //- Return collated 'processorsDDD' filtering
+        static int nProcsFilter() noexcept
+        {
+            return nProcsFilter_;
+        }
+
+        //- Set collated 'processorsDDD' filtering (0 = off).
+        //  \return the previous value
+        static int nProcsFilter(int level) noexcept
+        {
+            int old(nProcsFilter_);
+            nProcsFilter_ = level;
+            return old;
+        }
+
         //- Sort directory entries according to time value,
         //  with "constant" appearing first (if it exists)
         static instantList sortTimes
diff --git a/src/OpenFOAM/global/fileOperations/fileOperation/fileOperationNew.C b/src/OpenFOAM/global/fileOperations/fileOperation/fileOperationNew.C
index b12739ed76..7b0af4fdfd 100644
--- a/src/OpenFOAM/global/fileOperations/fileOperation/fileOperationNew.C
+++ b/src/OpenFOAM/global/fileOperations/fileOperation/fileOperationNew.C
@@ -383,7 +383,9 @@ Foam::fileOperation::New_impl
 
         if (newHandler)
         {
-            newHandler->nProcs(origHandler.nProcs());
+            // Make sure that the output format uses the correct number of
+            // 'active' ranks (instead of that of the origHandler)
+            newHandler->nProcs(subProcs.size());
             newHandler->storeComm();
         }
     }
@@ -402,7 +404,30 @@ Foam::fileOperation::New
 {
     labelList subProcs = getSelectedProcs(useProc);
 
-    return fileOperation::New_impl(origHandler, subProcs, verbose);
+    // useProc subsets worldComm ranks. So we cannot use worldComm
+    // anywhere in file handler. Workaround: use temporary communicator
+    // so we can at least construct. TBD.
+
+    label oldWorldComm = -1;
+    autoPtr<UPstream::communicator> subWorldComm;
+    if (subProcs.size() != UPstream::nProcs(UPstream::worldComm))
+    {
+        subWorldComm.reset
+        (
+            new UPstream::communicator
+            (
+                UPstream::worldComm,
+                subProcs
+            )
+        );
+        oldWorldComm = UPstream::commWorld(subWorldComm());
+    }
+
+    autoPtr<fileOperation> fh(New_impl(origHandler, subProcs, verbose));
+
+    UPstream::commWorld(oldWorldComm);
+
+    return fh;
 }
 
 
@@ -416,7 +441,31 @@ Foam::fileOperation::New
 {
     labelList subProcs = getSelectedProcs(useProc);
 
-    return fileOperation::New_impl(origHandler, subProcs, verbose);
+    label oldWorldComm = -1;
+    autoPtr<UPstream::communicator> subWorldComm;
+
+    // useProc subsets worldComm ranks. So we cannot use worldComm
+    // anywhere in file handler. Workaround: use temporary communicator
+    // so we can at least construct. TBD.
+
+    if (subProcs.size() != UPstream::nProcs(UPstream::worldComm))
+    {
+        subWorldComm.reset
+        (
+            new UPstream::communicator
+            (
+                UPstream::worldComm,
+                subProcs
+            )
+        );
+        oldWorldComm = UPstream::commWorld(subWorldComm());
+    }
+
+    autoPtr<fileOperation> fh(New_impl(origHandler, subProcs, verbose));
+
+    UPstream::commWorld(oldWorldComm);
+
+    return fh;
 }
 
 
diff --git a/src/OpenFOAM/global/fileOperations/fileOperation/fileOperationRanks.C b/src/OpenFOAM/global/fileOperations/fileOperation/fileOperationRanks.C
index 318afc65e5..c500571412 100644
--- a/src/OpenFOAM/global/fileOperations/fileOperation/fileOperationRanks.C
+++ b/src/OpenFOAM/global/fileOperations/fileOperation/fileOperationRanks.C
@@ -100,8 +100,6 @@ Foam::labelRange Foam::fileOperation::subRanks(const labelUList& mainIOranks)
         }
     }
 
-    Pout<< "subRanks: " << labelRange(begProc, (endProc-begProc)) << endl;
-
     return labelRange(begProc, (endProc-begProc));
 }
 
diff --git a/src/OpenFOAM/global/fileOperations/uncollatedFileOperation/uncollatedFileOperation.C b/src/OpenFOAM/global/fileOperations/uncollatedFileOperation/uncollatedFileOperation.C
index 01103eecee..35fb161016 100644
--- a/src/OpenFOAM/global/fileOperations/uncollatedFileOperation/uncollatedFileOperation.C
+++ b/src/OpenFOAM/global/fileOperations/uncollatedFileOperation/uncollatedFileOperation.C
@@ -121,12 +121,11 @@ Foam::fileName Foam::fileOperations::uncollatedFileOperation::filePathInfo
             // Check if parallel "procesors" directory
             if (io.time().processorCase())
             {
-                refPtr<dirIndexList> pDirs
+                const refPtr<dirIndexList> pDirs
                 (
-                    fileOperation::lookupAndCacheProcessorsPath
+                    lookupProcessorsPath
                     (
-                        io.objectPath(),
-                        false // No additional parallel synchronisation
+                        io.objectPath()
                     )
                 );
 
diff --git a/src/OpenFOAM/global/fileOperations/uncollatedFileOperation/uncollatedFileOperation.H b/src/OpenFOAM/global/fileOperations/uncollatedFileOperation/uncollatedFileOperation.H
index 9b8bd51083..ae9fc3d10e 100644
--- a/src/OpenFOAM/global/fileOperations/uncollatedFileOperation/uncollatedFileOperation.H
+++ b/src/OpenFOAM/global/fileOperations/uncollatedFileOperation/uncollatedFileOperation.H
@@ -85,7 +85,7 @@ protected:
         //  \return empty fileName if not found.
         virtual refPtr<dirIndexList> lookupProcessorsPath
         (
-            const fileName&
+            const fileName& objectPath
         ) const;
 
 
diff --git a/src/finiteVolume/fvMesh/fvMeshTools/fvMeshTools.C b/src/finiteVolume/fvMesh/fvMeshTools/fvMeshTools.C
index 14f154017d..b9280f819b 100644
--- a/src/finiteVolume/fvMesh/fvMeshTools/fvMeshTools.C
+++ b/src/finiteVolume/fvMesh/fvMeshTools/fvMeshTools.C
@@ -794,25 +794,6 @@ Foam::fvMeshTools::loadOrCreateMeshImpl
         // Non-null reference when a mesh exists on given processor
         haveLocalMesh = (*readHandlerPtr).good();
     }
-    else
-    {
-        // No file handler.
-        // Check for presence of the "faces" file,
-        // but for 'decompose', only need mesh on master.
-
-        haveLocalMesh =
-        (
-            decompose
-          ? UPstream::master()
-          : fileHandler().isFile
-            (
-                fileHandler().filePath
-                (
-                    io.time().path()/io.instance()/meshSubDir/"faces"
-                )
-            )
-        );
-    }
 
     // Globally consistent information about who has a mesh
     boolList haveMesh
@@ -820,6 +801,11 @@ Foam::fvMeshTools::loadOrCreateMeshImpl
         UPstream::allGatherValues<bool>(haveLocalMesh)
     );
 
+    // Make sure all have the same number of processors
+    label masterNProcs = fileHandler().nProcs();
+    Pstream::broadcast(masterNProcs);
+    const_cast<fileOperation&>(fileHandler()).nProcs(masterNProcs);
+
 
     autoPtr<fvMesh> meshPtr;
 
@@ -960,6 +946,7 @@ Foam::fvMeshTools::loadOrCreateMeshImpl
             )
         )
         {
+            const_cast<fileOperation&>(fileHandler()).nProcs(UPstream::nProcs(oldWorldComm));
             // Can use the handler communicator as is.
             UPstream::commWorld(fileHandler().comm());
         }

From 910713398e0df3602f6f3fdf6eb71c5f89d99904 Mon Sep 17 00:00:00 2001
From: mattijs <mattijs>
Date: Thu, 19 Oct 2023 11:43:10 +0100
Subject: [PATCH 10/12] ENH: redistributePar: loose matching of processors
 directories

---
 .../redistributePar/redistributePar.C         | 281 ++++++++++--------
 src/OpenFOAM/global/argList/argList.C         |   8 +
 .../fileOperation/fileOperation.C             |  18 +-
 .../fileOperation/fileOperationNew.C          |  62 +---
 .../masterUncollatedFileOperation.C           |  52 ++++
 5 files changed, 245 insertions(+), 176 deletions(-)

diff --git a/applications/utilities/parallelProcessing/redistributePar/redistributePar.C b/applications/utilities/parallelProcessing/redistributePar/redistributePar.C
index 3faf8138b4..556057d439 100644
--- a/applications/utilities/parallelProcessing/redistributePar/redistributePar.C
+++ b/applications/utilities/parallelProcessing/redistributePar/redistributePar.C
@@ -117,7 +117,7 @@ const int debug(::Foam::debug::debugSwitch("redistributePar", 0));
 // Allocate a new file handler on valid processors only
 // retaining the original IO ranks if possible
 autoPtr<fileOperation>
-getNewHandler(const boolUList& useProc, bool verbose = true)
+getNewHandler(const boolUList& useProc, const bool verbose = true)
 {
     autoPtr<fileOperation> handler
     (
@@ -127,6 +127,7 @@ getNewHandler(const boolUList& useProc, bool verbose = true)
     if (::debug && handler)
     {
         Pout<< "Allocated " << handler().info()
+            << " myProcNo:" << UPstream::myProcNo(handler().comm())
             << " ptr:" << Foam::name(handler.get()) << endl;
     }
 
@@ -154,9 +155,6 @@ void createTimeDirs(const fileName& path)
     instantList masterTimeDirs;
     if (Pstream::master())
     {
-        //const bool oldParRun = Pstream::parRun(false);
-        //timeDirs = Time::findTimes(path, "constant");
-        //Pstream::parRun(oldParRun);  // Restore parallel state
         masterTimeDirs = localTimeDirs;
     }
     Pstream::broadcast(masterTimeDirs);
@@ -213,16 +211,10 @@ void copyUniform
     // - write using writeDb + writeHandler
 
     fileName readPath;
+
     if (readHandler)
     {
-        auto oldHandler = fileOperation::fileHandler(readHandler);
-        const label oldComm = UPstream::commWorld(fileHandler().comm());
-
-        //Pout<< "** copyUniform: switching to handler:" << fileHandler().type()
-        //    << " with comm:" << fileHandler().comm()
-        //    << " with procs:" << UPstream::procID(fileHandler().comm())
-        //    << endl;
-
+        const label oldComm = UPstream::commWorld(readHandler().comm());
 
         Time& readTime = const_cast<Time&>(readDb.time());
         bool oldProcCase = readTime.processorCase();
@@ -237,7 +229,7 @@ void copyUniform
         }
 
         // Detect uniform/ at original database + time
-        readPath = fileHandler().dirPath
+        readPath = readHandler().dirPath
         (
             false,          // local directory
             IOobject("uniform", readTimeName, readDb),
@@ -245,16 +237,8 @@ void copyUniform
         );
 
 
-        readHandler = fileOperation::fileHandler(oldHandler);
         UPstream::commWorld(oldComm);
 
-        //Pout<< "** copyUniform:"
-        //    << " switched back to handler:" << fileHandler().type()
-        //    << " with comm:" << fileHandler().comm()
-        //    << " with procs:" << UPstream::procID(fileHandler().comm())
-        //    << endl;
-
-
         if (decompose)
         {
             // Reset caseName on master
@@ -263,7 +247,6 @@ void copyUniform
             readTime.processorCase(oldProcCase);
         }
     }
-
     Pstream::broadcast(readPath, UPstream::worldComm);
 
     if (!readPath.empty())
@@ -293,16 +276,15 @@ void copyUniform
         // Switch to writeHandler
         if (writeHandler)
         {
-            //const label oldWorldComm = UPstream::worldComm;
             auto oldHandler = fileOperation::fileHandler(writeHandler);
 
             // Check: fileHandler.comm() is size 1 for uncollated
             const label writeComm = fileHandler().comm();
-            //UPstream::worldComm = writeComm;
 
             if (reconstruct)
             {
                 const bool oldParRun = UPstream::parRun(false);
+                const label oldNumProcs(fileHandler().nProcs());
                 const fileName writePath
                 (
                     fileHandler().objectPath
@@ -312,6 +294,7 @@ void copyUniform
                     )
                 );
                 fileHandler().cp(readPath, writePath);
+                const_cast<fileOperation&>(fileHandler()).nProcs(oldNumProcs);
                 UPstream::parRun(oldParRun);
             }
             else
@@ -340,7 +323,6 @@ void copyUniform
                 );
             }
             writeHandler = fileOperation::fileHandler(oldHandler);
-            //UPstream::worldComm = oldWorldComm;
         }
     }
 }
@@ -686,6 +668,7 @@ autoPtr<mapDistributePolyMesh> redistributeAndWrite
     //Pout<< "Before distribution:" << endl;
     //printMeshData(mesh);
 
+
     // Storage of fields
 
     PtrList<volScalarField> volScalarFields;
@@ -769,21 +752,9 @@ autoPtr<mapDistributePolyMesh> redistributeAndWrite
             auto oldHandler = fileOperation::fileHandler(readHandler);
             const label oldComm = UPstream::commWorld(fileHandler().comm());
 
-            //Pout<< "** redistributeAndWrite:"
-            //    << " switching to handler:" << fileHandler().type()
-            //    << " with comm:" << fileHandler().comm()
-            //    << " with procs:" << UPstream::procID(fileHandler().comm())
-            //    << endl;
-
             objects = IOobjectList(mesh, runTime.timeName());
             readHandler = fileOperation::fileHandler(oldHandler);
             UPstream::commWorld(oldComm);
-
-            //Pout<< "** redistributeAndWrite:"
-            //    << " switched back to handler:" << fileHandler().type()
-            //    << " with comm:" << fileHandler().comm()
-            //    << " with procs:" << UPstream::procID(fileHandler().comm())
-            //    << endl;
         }
 
         if (decompose)
@@ -1271,6 +1242,12 @@ int main(int argc, char *argv[])
         true  // Advanced option
     );
 
+
+    //- Disable caching of times/processor dirs etc. Cause massive parallel
+    //  problems when e.g decomposing.
+    fileOperation::cacheLevel(0);
+
+
     // Handle arguments
     // ~~~~~~~~~~~~~~~~
     // (replacement for setRootCase that does not abort)
@@ -1287,16 +1264,17 @@ int main(int argc, char *argv[])
     // - reconstruct - reads parallel, write on master only and to parent
     //                 directory
 
-    //- Disable caching of times etc. Cause massive parallel problems when e.g
-    //  decomposing.
-    fileOperation::cacheLevel(0);
-
     // Detect if running data-distributed (processors cannot see all other
     // processors' files)
     const bool nfs = !fileHandler().distributed();
 
-    // Switch off parallel synchronisation of cached time directories
-    //fileHandler().distributed(true);
+
+    // Set up loose processorsXXX directory matching (for collated) so e.g.
+    // when checking for -latestTime we don't miss anything. Once we know
+    // the time, actual number of processors etc we switch back to strict
+    // matching.
+    fileOperation::nProcsFilter(0);
+
 
     // Read handler on processors with a volMesh
     refPtr<fileOperation> volMeshReadHandler;
@@ -1304,11 +1282,12 @@ int main(int argc, char *argv[])
     // Read handler on processors with an areaMesh
     refPtr<fileOperation> areaMeshReadHandler;
 
-    // Handler for master-only operation
-    refPtr<fileOperation> masterOnlyHandler
-    (
-        getNewHandler(boolList(one{}, true))
-    );
+    // Handler for master-only operation (read/writing from/to undecomposed)
+    refPtr<fileOperation> masterOnlyHandler;
+    if (UPstream::master(UPstream::worldComm))
+    {
+        masterOnlyHandler = fileOperation::NewUncollated();
+    }
 
 
     // Need this line since we don't include "setRootCase.H"
@@ -1362,6 +1341,25 @@ int main(int argc, char *argv[])
             << nl << endl;
     }
 
+    if (decompose || reconstruct)
+    {
+        // The UPstream::nProcs is either the source or destination procs
+        fileOperation::nProcsFilter(UPstream::nProcs());
+        InfoOrPout<< "Switching to exact matching for "
+            << fileOperation::processorsBaseDir + Foam::name(UPstream::nProcs())
+            << " processor directories"
+            << nl << endl;
+    }
+    else
+    {
+        // Redistribute mode. Accept any processorsXXX naming since we don't
+        // know yet what the source/target number of processors is
+        fileOperation::nProcsFilter(0);
+        InfoOrPout<< "Switching to matching any "
+            << fileOperation::processorsBaseDir << " directory" << nl << endl;
+    }
+
+
     if ((decompose || reconstruct) && !overwrite)
     {
         overwrite = true;
@@ -1400,6 +1398,7 @@ int main(int argc, char *argv[])
         fileHandler().mkDir(args.globalPath());
     }
 
+
     // Check if we have processor directories. Ideally would like to
     // use fileHandler().dirPath here but we don't have runTime yet and
     // want to delay constructing runTime until we've synced all time
@@ -1427,6 +1426,17 @@ int main(int argc, char *argv[])
     }
     // If master changed to decompose mode make sure all nodes know about it
     Pstream::broadcast(decompose);
+    if (decompose)
+    {
+         // The UPstream::nProcs is either the source or destination procs
+         fileOperation::nProcsFilter(UPstream::nProcs());
+         InfoOrPout<< "Switching to exact matching for "
+             << fileOperation::processorsBaseDir + Foam::name(UPstream::nProcs())
+             << " processor directories"
+             << nl << endl;
+     }
+
+
 
     // If running distributed we have problem of new processors not finding
     // a system/controlDict. However if we switch on the master-only reading
@@ -1444,13 +1454,6 @@ int main(int argc, char *argv[])
     // Construct time
     // ~~~~~~~~~~~~~~
 
-    // Disable master-only file checking since
-    // - uncollated : interferes with running read-handler on less processors
-    //                than nProcs (when redistributing from less to more)
-    // - (masterUn)collated : does not use timeStampMaster anyway
-    Info<< "Disabling uncollated master reading" << nl << endl;
-    IOobject::fileModificationChecking = IOobject::timeStamp;
-
     // Replace #include "createTime.H" with our own version
     // that has MUST_READ instead of READ_MODIFIED
 
@@ -1616,7 +1619,7 @@ int main(int argc, char *argv[])
                 // - check for 'faces' file (polyMesh)
                 // - check for 'faceLabels' file (faMesh)
                 boolList volMeshOnProc;
-                boolList areaMeshOnProc;
+                boolList areaMeshOnProc(UPstream::nProcs(), false);
 
                 volMeshOnProc = haveMeshFile
                 (
@@ -1759,6 +1762,7 @@ int main(int argc, char *argv[])
                         << endl;
 
                     const bool oldParRun = Pstream::parRun(false);
+                    const label oldNumProcs(fileHandler().nProcs());
 
                     // Volume
                     {
@@ -1794,6 +1798,10 @@ int main(int argc, char *argv[])
                         areaMeshHaveUndecomposed = labelsIO.headerOk();
                     }
 
+                    const_cast<fileOperation&>
+                    (
+                        fileHandler()
+                    ).nProcs(oldNumProcs);
                     Pstream::parRun(oldParRun);  // Restore parallel state
                 }
 
@@ -1990,11 +1998,16 @@ int main(int argc, char *argv[])
                         runTime.caseName() = baseRunTime.caseName();
                         const bool oldProcCase(runTime.processorCase(false));
                         const bool oldParRun(Pstream::parRun(false));
+                        const label oldNumProcs(fileHandler().nProcs());
 
                         areaBaseMeshPtr().write();
 
                         // Now we've written all. Reset caseName on master
                         InfoOrPout<< "Restoring caseName" << endl;
+                        const_cast<fileOperation&>
+                        (
+                            fileHandler()
+                        ).nProcs(oldNumProcs);
                         Pstream::parRun(oldParRun);
                         runTime.caseName() = proc0CaseName;
                         runTime.processorCase(oldProcCase);
@@ -2421,26 +2434,19 @@ int main(int argc, char *argv[])
         //              slave  : dummy mesh
         //  redistribute : all read mesh or dummy mesh
 
+        Time& readRunTime =
+        (
+            (decompose)
+          ? baseRunTime
+          : runTime
+        );
+
         // Time coming from processor0 (or undecomposed if no processor0)
-        scalar masterTime;
-        if (decompose)
-        {
-            // Use base time. This is to handle e.g. startTime = latestTime
-            // which will not do anything if there are no processor directories
-            masterTime = timeSelector::selectIfPresent
-            (
-                baseRunTime,
-                args
-            )[0].value();
-        }
-        else
-        {
-            masterTime = timeSelector::selectIfPresent
-            (
-                runTime,
-                args
-            )[0].value();
-        }
+        scalar masterTime = timeSelector::selectIfPresent
+        (
+            readRunTime,
+            args
+        )[0].value();
         Pstream::broadcast(masterTime);
         InfoOrPout
             << "Setting time to that of master or undecomposed case : "
@@ -2448,6 +2454,9 @@ int main(int argc, char *argv[])
         runTime.setTime(masterTime, 0);
         baseRunTime.setTime(masterTime, 0);
 
+
+
+
         // Save old time name (since might be incremented)
         const word oldTimeName(runTime.timeName());
 
@@ -2490,7 +2499,8 @@ int main(int argc, char *argv[])
                 }
 
                 const bool oldParRun = Pstream::parRun(false);
-                volMeshMasterInstance = runTime.findInstance
+                const label oldNumProcs(fileHandler().nProcs());
+                volMeshMasterInstance = readRunTime.findInstance
                 (
                     volMeshSubDir,
                     "faces",
@@ -2499,7 +2509,7 @@ int main(int argc, char *argv[])
 
                 if (doFiniteArea)
                 {
-                    areaMeshMasterInstance = runTime.findInstance
+                    areaMeshMasterInstance = readRunTime.findInstance
                     (
                         areaMeshSubDir,
                         "faceLabels",
@@ -2515,7 +2525,7 @@ int main(int argc, char *argv[])
                         (
                             haveMeshFile
                             (
-                                runTime,
+                                readRunTime,
                                 areaMeshMasterInstance/areaMeshSubDir,
                                 "faceLabels",
                                 false  // verbose=false
@@ -2529,6 +2539,7 @@ int main(int argc, char *argv[])
                     }
                 }
 
+                const_cast<fileOperation&>(fileHandler()).nProcs(oldNumProcs);
                 Pstream::parRun(oldParRun);  // Restore parallel state
 
                 if (decompose)
@@ -2564,25 +2575,59 @@ int main(int argc, char *argv[])
             boolList volMeshOnProc;
             boolList areaMeshOnProc;
 
-            volMeshOnProc = haveMeshFile
-            (
-                runTime,
-                volMeshMasterInstance/volMeshSubDir,
-                "faces"
-            );
+            if (decompose)
+            {
+                // Already determined above that master can read 'faces' file.
+                // This avoids doing all the casename setting/restoring again.
+                volMeshOnProc.setSize(UPstream::nProcs(), false);
+                volMeshOnProc[UPstream::masterNo()] = volMeshHaveUndecomposed;
+            }
+            else
+            {
+                // All check if can read 'faces' file
+                volMeshOnProc = haveMeshFile
+                (
+                    runTime,
+                    volMeshMasterInstance/volMeshSubDir,
+                    "faces"
+                );
+            }
 
             // Create handler for reading
             newHandler(volMeshOnProc, volMeshReadHandler);
 
 
+            // Now we've determined which processors are reading switch back
+            // to exact matching of 'processorsXXX' directory names.
+            // - this determines the processorsXXX fileNames
+            // - the XXX comes from the number of read processors
+            // - also adapt the masterOnlyReader (used in copyUniform)
+
+            fileOperation::nProcsFilter
+            (
+                findIndices(volMeshOnProc, true).size()
+            );
+
+
             if (doFiniteArea)
             {
-                areaMeshOnProc = haveMeshFile
-                (
-                    runTime,
-                    areaMeshMasterInstance/areaMeshSubDir,
-                    "faceLabels"
-                );
+                if (decompose)
+                {
+                    // Already determined above that master can read
+                    // 'faceLabels' file.
+                    areaMeshOnProc.setSize(UPstream::nProcs(), false);
+                    areaMeshOnProc[UPstream::masterNo()] =
+                    areaMeshHaveUndecomposed;
+                }
+                else
+                {
+                    areaMeshOnProc = haveMeshFile
+                    (
+                        runTime,
+                        areaMeshMasterInstance/areaMeshSubDir,
+                        "faceLabels"
+                    );
+                }
 
                 // Create handler for reading
                 if (areaMeshOnProc == volMeshOnProc)
@@ -2599,6 +2644,7 @@ int main(int argc, char *argv[])
                 }
             }
 
+
             // Prior to loadOrCreateMesh, note which meshes already exist
             // for the current file handler.
             // - where mesh would be written if it didn't exist already.
@@ -2723,18 +2769,6 @@ int main(int argc, char *argv[])
                 )
             )
             {
-                //areaMeshPtr = faMeshTools::loadOrCreateMesh
-                //(
-                //    IOobject
-                //    (
-                //        regionName,
-                //        areaMeshMasterInstance,
-                //        runTime,
-                //        IOobjectOption::MUST_READ
-                //    ),
-                //    mesh,  // <- The referenced polyMesh (from above)
-                //    decompose
-                //);
                 areaMeshPtr = faMeshTools::loadOrCreateMesh
                 (
                     IOobject
@@ -2762,12 +2796,6 @@ int main(int argc, char *argv[])
 
             const label nOldCells = mesh.nCells();
 
-            // const label nOldAreaFaces =
-            //     (areaMeshPtr ? areaMeshPtr().nFaces() : 0);
-            //
-            //Pout<< "Loaded mesh : nCells:" << nOldCells
-            //    << " nPatches:" << mesh.boundaryMesh().size() << endl;
-            //Pout<< "Loaded area mesh : nFaces:" << nOldAreaFaces << endl;
 
             // Determine decomposition
             // ~~~~~~~~~~~~~~~~~~~~~~~
@@ -2788,7 +2816,6 @@ int main(int argc, char *argv[])
                 finalDecomp
             );
 
-
             if (dryrun)
             {
                 continue;
@@ -2850,6 +2877,7 @@ int main(int argc, char *argv[])
                 runTime.processorCase(oldProcCase);
             }
 
+
             // Load fields, do all distribution (mesh and fields)
             // - but not lagrangian fields; these are done later
             autoPtr<mapDistributePolyMesh> distMap = redistributeAndWrite
@@ -2875,7 +2903,6 @@ int main(int argc, char *argv[])
                 mesh
             );
 
-
             // Redistribute any clouds
             redistributeLagrangian
             (
@@ -2918,11 +2945,16 @@ int main(int argc, char *argv[])
                         runTime.caseName() = baseRunTime.caseName();
                         const bool oldProcCase(runTime.processorCase(false));
                         const bool oldParRun = UPstream::parRun(false);
+                        const label oldNumProcs(fileHandler().nProcs());
 
                         areaProcMeshPtr->write();
 
                         // Now we've written all. Reset caseName on master
                         InfoOrPout<< "Restoring caseName" << endl;
+                        const_cast<fileOperation&>
+                        (
+                            fileHandler()
+                        ).nProcs(oldNumProcs);
                         UPstream::parRun(oldParRun);
                         runTime.caseName() = proc0CaseName;
                         runTime.processorCase(oldProcCase);
@@ -2981,11 +3013,19 @@ int main(int argc, char *argv[])
 
             // Get reference to standard write handler
             refPtr<fileOperation> defaultHandler;
-            defaultHandler.ref(const_cast<fileOperation&>(fileHandler()));
+            if (writeHandler)
+            {
+                defaultHandler.ref(writeHandler.ref());
+            }
+            else
+            {
+                defaultHandler.ref(const_cast<fileOperation&>(fileHandler()));
+            }
+
 
             copyUniform
             (
-                masterOnlyHandler,  // read handler
+                volMeshReadHandler, // read handler
                 defaultHandler,     //TBD: should be all IOranks
 
                 reconstruct,        // reconstruct
@@ -3001,11 +3041,18 @@ int main(int argc, char *argv[])
 
         // Get reference to standard write handler
         refPtr<fileOperation> defaultHandler;
-        defaultHandler.ref(const_cast<fileOperation&>(fileHandler()));
+        if (writeHandler)
+        {
+            defaultHandler.ref(writeHandler.ref());
+        }
+        else
+        {
+            defaultHandler.ref(const_cast<fileOperation&>(fileHandler()));
+        }
 
         copyUniform
         (
-            masterOnlyHandler,  // read handler
+            volMeshReadHandler, // read handler
             defaultHandler,     //TBD: should be all IOranks
 
             reconstruct,        // reconstruct (=false)
@@ -3013,11 +3060,7 @@ int main(int argc, char *argv[])
 
             oldTimeName,        // provided read time
             (decompose ? baseRunTime.caseName() : proc0CaseName),
-            (                   // read location
-                decompose
-              ? baseRunTime
-              : runTime
-            ),
+            readRunTime,
             runTime             // writing location
         );
     }
diff --git a/src/OpenFOAM/global/argList/argList.C b/src/OpenFOAM/global/argList/argList.C
index 91bdfd6d96..e47fb46342 100644
--- a/src/OpenFOAM/global/argList/argList.C
+++ b/src/OpenFOAM/global/argList/argList.C
@@ -1526,6 +1526,9 @@ void Foam::argList::parse
                 // also have to protect the actual dictionary parsing since
                 // it might trigger file access (e.g. #include, #codeStream)
                 const bool oldParRun = Pstream::parRun(false);
+                // Note: non-parallel running might update
+                // fileOperation::nProcs() so store & restore below
+                const label nOldProcs = fileHandler().nProcs();
 
                 autoPtr<ISstream> dictStream
                 (
@@ -1584,6 +1587,7 @@ void Foam::argList::parse
                 }
 
                 Pstream::parRun(oldParRun);  // Restore parallel state
+                const_cast<fileOperation&>(fileHandler()).nProcs(nOldProcs);
 
                 if (Pstream::nProcs() == 1)
                 {
@@ -1819,6 +1823,9 @@ void Foam::argList::parse
                 if (UPstream::master())
                 {
                     const bool oldParRun = Pstream::parRun(false);
+                    // Note: non-parallel running might update
+                    // fileOperation::nProcs() so store & restore below
+                    const label nOldProcs = fileHandler().nProcs();
 
                     label rank = 0;
                     for (label proci = 1; proci < pathDirs.size(); ++proci)
@@ -1855,6 +1862,7 @@ void Foam::argList::parse
                     }
 
                     UPstream::parRun(oldParRun);
+                    const_cast<fileOperation&>(fileHandler()).nProcs(nOldProcs);
                 }
                 Pstream::broadcast(rankToDirs);
 
diff --git a/src/OpenFOAM/global/fileOperations/fileOperation/fileOperation.C b/src/OpenFOAM/global/fileOperations/fileOperation/fileOperation.C
index bbca362e0d..c82ed16c62 100644
--- a/src/OpenFOAM/global/fileOperations/fileOperation/fileOperation.C
+++ b/src/OpenFOAM/global/fileOperations/fileOperation/fileOperation.C
@@ -80,7 +80,7 @@ Foam::word Foam::fileOperation::processorsBaseDir = "processors";
 //- Caching (e.g. of time directories) - enabled by default
 int Foam::fileOperation::cacheLevel_(1);
 
-int Foam::fileOperation::nProcsFilter_(0);
+int Foam::fileOperation::nProcsFilter_(-1);
 
 // * * * * * * * * * * * * * * * Local Functions * * * * * * * * * * * * * * //
 
@@ -429,6 +429,11 @@ Foam::fileOperation::lookupAndCacheProcessorsPath
     //             if directory status gets synchronised
     // - distributed() : different processors have different roots
     // - fileModificationChecking : (uncollated only) do IO on master only
+    // - nProcsFilter_ : if set to
+    //       0 : accept any directory (e.g. for redistributePar where we don't
+    //           know yet number of read/write procs)
+    //      -1 : accept only processorsDDD where DDD is nProcs(worldComm)
+    //      >0 : accept the exact mentioned number of prcessors
 
 
     // Collated : check whether/how to filter processorsXXX directory names
@@ -556,7 +561,7 @@ Foam::fileOperation::lookupAndCacheProcessorsPath
                 {
                     // "processorsNN"
 
-                    if (proci < rNum)
+                    if (proci < rNum || (nProcsFilter_ == 0))
                     {
                         // And it is also in range.
                         // Eg for "processors4": 3 is ok, 10 is not
@@ -565,7 +570,7 @@ Foam::fileOperation::lookupAndCacheProcessorsPath
                         pathTypeIdx.second() = proci;
                     }
                 }
-                else if (group.contains(proci))
+                else if (group.contains(proci) || (nProcsFilter_ == 0))
                 {
                     // "processorsNN_start-end"
                     // - save the local proc offset
@@ -581,6 +586,10 @@ Foam::fileOperation::lookupAndCacheProcessorsPath
             }
         }
 
+        // Sort processor directory names (natural order)
+        sortProcessorDirs(procDirs);
+
+
         // Global check of empty/exists.
         // 1 : empty directory
         // 2 : non-empty directory
@@ -675,9 +684,6 @@ Foam::fileOperation::lookupAndCacheProcessorsPath
             }
         }
 
-        // Sort processor directory names (natural order)
-        sortProcessorDirs(procDirs);
-
         if (procDirsStatus & 2u)
         {
             if (cacheLevel() > 0)
diff --git a/src/OpenFOAM/global/fileOperations/fileOperation/fileOperationNew.C b/src/OpenFOAM/global/fileOperations/fileOperation/fileOperationNew.C
index 7b0af4fdfd..5b05fe46c8 100644
--- a/src/OpenFOAM/global/fileOperations/fileOperation/fileOperationNew.C
+++ b/src/OpenFOAM/global/fileOperations/fileOperation/fileOperationNew.C
@@ -373,6 +373,11 @@ Foam::fileOperation::New_impl
         // Allocate new handler with same type and similar IO ranks
         // but with different sub-ranks (and communicator)
 
+
+        // Temporarily override world comm to get through the initialisation
+        // (e.g. fileOperation::printRanks() which uses worldComm)
+        const label oldWorldComm = UPstream::commWorld(commAndIORanks.first());
+
         newHandler = fileOperation::New
         (
             origHandler.type(),
@@ -381,6 +386,8 @@ Foam::fileOperation::New_impl
             verbose
         );
 
+        UPstream::commWorld(oldWorldComm);
+
         if (newHandler)
         {
             // Make sure that the output format uses the correct number of
@@ -402,32 +409,9 @@ Foam::fileOperation::New
     bool verbose
 )
 {
-    labelList subProcs = getSelectedProcs(useProc);
+    const labelList subProcs = getSelectedProcs(useProc);
 
-    // useProc subsets worldComm ranks. So we cannot use worldComm
-    // anywhere in file handler. Workaround: use temporary communicator
-    // so we can at least construct. TBD.
-
-    label oldWorldComm = -1;
-    autoPtr<UPstream::communicator> subWorldComm;
-    if (subProcs.size() != UPstream::nProcs(UPstream::worldComm))
-    {
-        subWorldComm.reset
-        (
-            new UPstream::communicator
-            (
-                UPstream::worldComm,
-                subProcs
-            )
-        );
-        oldWorldComm = UPstream::commWorld(subWorldComm());
-    }
-
-    autoPtr<fileOperation> fh(New_impl(origHandler, subProcs, verbose));
-
-    UPstream::commWorld(oldWorldComm);
-
-    return fh;
+    return New_impl(origHandler, subProcs, verbose);
 }
 
 
@@ -439,33 +423,9 @@ Foam::fileOperation::New
     bool verbose
 )
 {
-    labelList subProcs = getSelectedProcs(useProc);
+    const labelList subProcs = getSelectedProcs(useProc);
 
-    label oldWorldComm = -1;
-    autoPtr<UPstream::communicator> subWorldComm;
-
-    // useProc subsets worldComm ranks. So we cannot use worldComm
-    // anywhere in file handler. Workaround: use temporary communicator
-    // so we can at least construct. TBD.
-
-    if (subProcs.size() != UPstream::nProcs(UPstream::worldComm))
-    {
-        subWorldComm.reset
-        (
-            new UPstream::communicator
-            (
-                UPstream::worldComm,
-                subProcs
-            )
-        );
-        oldWorldComm = UPstream::commWorld(subWorldComm());
-    }
-
-    autoPtr<fileOperation> fh(New_impl(origHandler, subProcs, verbose));
-
-    UPstream::commWorld(oldWorldComm);
-
-    return fh;
+    return New_impl(origHandler, subProcs, verbose);
 }
 
 
diff --git a/src/OpenFOAM/global/fileOperations/masterUncollatedFileOperation/masterUncollatedFileOperation.C b/src/OpenFOAM/global/fileOperations/masterUncollatedFileOperation/masterUncollatedFileOperation.C
index f48b12c8d6..ac90550014 100644
--- a/src/OpenFOAM/global/fileOperations/masterUncollatedFileOperation/masterUncollatedFileOperation.C
+++ b/src/OpenFOAM/global/fileOperations/masterUncollatedFileOperation/masterUncollatedFileOperation.C
@@ -799,6 +799,23 @@ masterUncollatedFileOperation
     managedComm_(getManagedComm(comm_))  // Possibly locally allocated
 {
     init(verbose);
+
+    if (comm_ == -1)
+    {
+        FatalErrorInFunction<< "Problem comm_:" << comm_ << exit(FatalError);
+    }
+    if (UPstream::nProcs(comm_) == -1)
+    {
+        FatalErrorInFunction<< "Problem comm_:" << comm_
+            << " nProcs:" << UPstream::nProcs(comm_)
+            << exit(FatalError);
+    }
+    if (UPstream::myProcNo(comm_) == -1)
+    {
+        FatalErrorInFunction<< "Problem comm_:" << comm_
+            << " myProcNo:" << UPstream::myProcNo(comm_)
+            << exit(FatalError);
+    }
 }
 
 
@@ -814,6 +831,23 @@ masterUncollatedFileOperation
     managedComm_(-1)  // Externally managed
 {
     init(verbose);
+
+    if (comm_ == -1)
+    {
+        FatalErrorInFunction<< "Problem comm_:" << comm_ << exit(FatalError);
+    }
+    if (UPstream::nProcs(comm_) == -1)
+    {
+        FatalErrorInFunction<< "Problem comm_:" << comm_
+            << " nProcs:" << UPstream::nProcs(comm_)
+            << exit(FatalError);
+    }
+    if (UPstream::myProcNo(comm_) == -1)
+    {
+        FatalErrorInFunction<< "Problem comm_:" << comm_
+            << " myProcNo:" << UPstream::myProcNo(comm_)
+            << exit(FatalError);
+    }
 }
 
 
@@ -1159,6 +1193,7 @@ Foam::fileName Foam::fileOperations::masterUncollatedFileOperation::filePath
     {
         const bool oldParRun = UPstream::parRun(false);
         const int oldCache = fileOperation::cacheLevel(0);
+        const label oldNProcs = fileOperation::nProcs();
 
         // All masters search locally. Note that global objects might
         // fail (except on master). This gets handled later on (in PARENTOBJECT)
@@ -1175,6 +1210,7 @@ Foam::fileName Foam::fileOperations::masterUncollatedFileOperation::filePath
                 newInstancePath
             );
 
+        const_cast<masterUncollatedFileOperation&>(*this).nProcs(oldNProcs);
         fileOperation::cacheLevel(oldCache);
         UPstream::parRun(oldParRun);
 
@@ -1315,6 +1351,7 @@ Foam::fileName Foam::fileOperations::masterUncollatedFileOperation::dirPath
     {
         const bool oldParRun = UPstream::parRun(false);
         const int oldCache = fileOperation::cacheLevel(0);
+        const label oldNProcs = fileOperation::nProcs();
 
         objPath = filePathInfo
         (
@@ -1328,6 +1365,7 @@ Foam::fileName Foam::fileOperations::masterUncollatedFileOperation::dirPath
             newInstancePath
         );
 
+        const_cast<masterUncollatedFileOperation&>(*this).nProcs(oldNProcs);
         fileOperation::cacheLevel(oldCache);
         UPstream::parRun(oldParRun);
 
@@ -1517,11 +1555,13 @@ Foam::fileOperations::masterUncollatedFileOperation::findInstance
     {
         const bool oldParRun = UPstream::parRun(false);
         const int oldCache = fileOperation::cacheLevel(0);
+        const label oldNProcs = fileOperation::nProcs();
 
         if (exists(pDirs, io))
         {
             foundInstance = io.instance();
         }
+        const_cast<masterUncollatedFileOperation&>(*this).nProcs(oldNProcs);
         fileOperation::cacheLevel(oldCache);
         UPstream::parRun(oldParRun);
     }
@@ -1556,6 +1596,7 @@ Foam::fileOperations::masterUncollatedFileOperation::findInstance
     {
         const bool oldParRun = UPstream::parRun(false);
         const int oldCache = fileOperation::cacheLevel(0);
+        const label oldNProcs = fileOperation::nProcs();
 
         label instIndex = ts.size()-1;
 
@@ -1666,6 +1707,7 @@ Foam::fileOperations::masterUncollatedFileOperation::findInstance
             }
         }
 
+        const_cast<masterUncollatedFileOperation&>(*this).nProcs(oldNProcs);
         fileOperation::cacheLevel(oldCache);
         UPstream::parRun(oldParRun);  // Restore parallel state
     }
@@ -1741,6 +1783,7 @@ Foam::fileOperations::masterUncollatedFileOperation::readObjects
         // (through call to filePath which triggers parallel )
         const bool oldParRun = UPstream::parRun(false);
         const int oldCache = fileOperation::cacheLevel(0);
+        const label oldNProcs = fileOperation::nProcs();
 
         //- Use non-time searching version
         objectNames = fileOperation::readObjects
@@ -1784,6 +1827,7 @@ Foam::fileOperations::masterUncollatedFileOperation::readObjects
             }
         }
 
+        const_cast<masterUncollatedFileOperation&>(*this).nProcs(oldNProcs);
         fileOperation::cacheLevel(oldCache);
         UPstream::parRun(oldParRun);  // Restore parallel state
     }
@@ -2228,11 +2272,13 @@ bool Foam::fileOperations::masterUncollatedFileOperation::read
             // Do master-only reading always.
             const bool oldParRun = UPstream::parRun(false);
             const int oldCache = fileOperation::cacheLevel(0);
+            const label oldNProcs = fileOperation::nProcs();
 
             auto& is = io.readStream(typeName);
             ok = io.readData(is);
             io.close();
 
+            const_cast<masterUncollatedFileOperation&>(*this).nProcs(oldNProcs);
             fileOperation::cacheLevel(oldCache);
             UPstream::parRun(oldParRun);  // Restore parallel state
         }
@@ -2362,9 +2408,11 @@ Foam::instantList Foam::fileOperations::masterUncollatedFileOperation::findTimes
             // Do master-only reading always.
             const bool oldParRun = UPstream::parRun(false);
             const int oldCache = fileOperation::cacheLevel(0);
+            const label oldNProcs = fileOperation::nProcs();
 
             times = fileOperation::findTimes(directory, constantName);
 
+            const_cast<masterUncollatedFileOperation&>(*this).nProcs(oldNProcs);
             fileOperation::cacheLevel(oldCache);
             UPstream::parRun(oldParRun);  // Restore parallel state
         }
@@ -2774,6 +2822,8 @@ void Foam::fileOperations::masterUncollatedFileOperation::addWatches
     {
         // Switch off comms inside findWatch/addWatch etc.
         const bool oldParRun = UPstream::parRun(false);
+        const int oldCache = fileOperation::cacheLevel(0);
+        const label oldNProcs = fileOperation::nProcs();
 
         labelHashSet removedWatches(watchIndices);
 
@@ -2799,6 +2849,8 @@ void Foam::fileOperations::masterUncollatedFileOperation::addWatches
             removeWatch(watchIndices[index]);
         }
 
+        const_cast<masterUncollatedFileOperation&>(*this).nProcs(oldNProcs);
+        fileOperation::cacheLevel(oldCache);
         UPstream::parRun(oldParRun);
     }
     Pstream::broadcast(newWatchIndices);

From 31825dd981d6b2145cc927053b57767c3aed866c Mon Sep 17 00:00:00 2001
From: mattijs <mattijs>
Date: Wed, 25 Oct 2023 15:05:53 +0100
Subject: [PATCH 11/12] BUG: redistributePar: cell/face/pointProcAddressing
 written to incorrect dir

---
 .../redistributePar/redistributePar.C         |  4 +-
 .../fvMesh/fvMeshTools/fvMeshTools.C          |  9 +++-
 .../fvMesh/fvMeshTools/fvMeshTools.H          |  1 +
 .../fvMesh/fvMeshTools/fvMeshToolsProcAddr.C  | 51 +++++++++++++------
 4 files changed, 48 insertions(+), 17 deletions(-)

diff --git a/applications/utilities/parallelProcessing/redistributePar/redistributePar.C b/applications/utilities/parallelProcessing/redistributePar/redistributePar.C
index 556057d439..a69ed17a8e 100644
--- a/applications/utilities/parallelProcessing/redistributePar/redistributePar.C
+++ b/applications/utilities/parallelProcessing/redistributePar/redistributePar.C
@@ -1040,7 +1040,8 @@ autoPtr<mapDistributePolyMesh> redistributeAndWrite
             mesh,
             distMap(),
             decompose,
-            writeHandler
+            mesh.facesInstance(),    //oldFacesInstance,
+            writeHandler             // to write *ProcAddressing
         );
     }
     else if (reconstruct)
@@ -1054,6 +1055,7 @@ autoPtr<mapDistributePolyMesh> redistributeAndWrite
             mesh,
             distMap(),
             decompose,
+            volMeshInstance,    //oldFacesInstance,
             readHandler //writeHandler
         );
     }
diff --git a/src/finiteVolume/fvMesh/fvMeshTools/fvMeshTools.C b/src/finiteVolume/fvMesh/fvMeshTools/fvMeshTools.C
index b9280f819b..9128a1b318 100644
--- a/src/finiteVolume/fvMesh/fvMeshTools/fvMeshTools.C
+++ b/src/finiteVolume/fvMesh/fvMeshTools/fvMeshTools.C
@@ -760,12 +760,19 @@ Foam::fvMeshTools::loadOrCreateMeshImpl
         const label oldNumProcs = fileHandler().nProcs();
         const int oldCache = fileOperation::cacheLevel(0);
 
+        const fileName facesInstance = io.time().findInstance
+        (
+            meshSubDir,
+            "faces",
+            IOobject::MUST_READ
+        );
+
         patchEntries = polyBoundaryMeshEntries
         (
             IOobject
             (
                 "boundary",
-                io.instance(),
+                facesInstance,
                 meshSubDir,
                 io.db(),
                 IOobject::MUST_READ,
diff --git a/src/finiteVolume/fvMesh/fvMeshTools/fvMeshTools.H b/src/finiteVolume/fvMesh/fvMeshTools/fvMeshTools.H
index 2c899adb9b..0a85225664 100644
--- a/src/finiteVolume/fvMesh/fvMeshTools/fvMeshTools.H
+++ b/src/finiteVolume/fvMesh/fvMeshTools/fvMeshTools.H
@@ -220,6 +220,7 @@ public:
         const fvMesh& procMesh,
         const mapDistributePolyMesh& map,
         const bool decompose,
+        const fileName& writeHandlerInstance,
         refPtr<fileOperation>& writeHandler
     );
 };
diff --git a/src/finiteVolume/fvMesh/fvMeshTools/fvMeshToolsProcAddr.C b/src/finiteVolume/fvMesh/fvMeshTools/fvMeshToolsProcAddr.C
index 1cd8e06dda..18fea740cc 100644
--- a/src/finiteVolume/fvMesh/fvMeshTools/fvMeshToolsProcAddr.C
+++ b/src/finiteVolume/fvMesh/fvMeshTools/fvMeshToolsProcAddr.C
@@ -303,6 +303,7 @@ void Foam::fvMeshTools::writeProcAddressing
     const fvMesh& mesh,
     const mapDistributePolyMesh& map,
     const bool decompose,
+    const fileName& writeHandlerInstance,
     refPtr<fileOperation>& writeHandler
 )
 {
@@ -317,7 +318,7 @@ void Foam::fvMeshTools::writeProcAddressing
     // been done independently (as a registered object)
     IOobject ioAddr
     (
-        "procAddressing",
+        "proc-addressing",
         mesh.facesInstance(),
         polyMesh::meshSubDir,
         mesh.thisDb(),
@@ -421,23 +422,43 @@ void Foam::fvMeshTools::writeProcAddressing
         );
     }
 
-    auto oldHandler = fileOperation::fileHandler(writeHandler);
 
-    const bool cellOk = cellMap.write();
-    const bool faceOk = faceMap.write();
-    const bool pointOk = pointMap.write();
-    const bool patchOk = patchMap.write();
+    // Switch to using the correct
+    // - fileHandler
+    // - instance
+    // to write to the original mesh/time in the original format. Clunky!
+    // Bypass regIOobject writing to avoid taking over the current time
+    // as instance so instead of e.g. 'celllMap.write()' directly call
+    // the chosen file-handler.
 
-    writeHandler = fileOperation::fileHandler(oldHandler);
-
-    if (!cellOk || !faceOk || !pointOk || !patchOk)
+    const auto& tm = cellMap.time();
+    const IOstreamOption opt(tm.writeFormat(), tm.writeCompression());
     {
-        WarningInFunction
-            << "Failed to write some of "
-            << cellMap.objectRelPath() << ", "
-            << faceMap.objectRelPath() << ", "
-            << pointMap.objectRelPath() << ", "
-            << patchMap.objectRelPath() << endl;
+        auto oldHandler = fileOperation::fileHandler(writeHandler);
+
+        cellMap.instance() = writeHandlerInstance;
+        const bool cellOk = fileHandler().writeObject(cellMap, opt, true);
+
+        faceMap.instance() = writeHandlerInstance;
+        const bool faceOk = fileHandler().writeObject(faceMap, opt, true);
+
+        pointMap.instance() = writeHandlerInstance;
+        const bool pointOk = fileHandler().writeObject(pointMap, opt, true);
+
+        patchMap.instance() = writeHandlerInstance;
+        const bool patchOk = fileHandler().writeObject(patchMap, opt, true);
+
+        writeHandler = fileOperation::fileHandler(oldHandler);
+
+        if (!cellOk || !faceOk || !pointOk || !patchOk)
+        {
+            WarningInFunction
+                << "Failed to write some of "
+                << cellMap.objectRelPath() << ", "
+                << faceMap.objectRelPath() << ", "
+                << pointMap.objectRelPath() << ", "
+                << patchMap.objectRelPath() << endl;
+        }
     }
 }
 

From 5be652d03c4b37be305e6ace310a0ee56a925f4a Mon Sep 17 00:00:00 2001
From: mattijs <mattijs>
Date: Thu, 9 Nov 2023 16:00:53 +0000
Subject: [PATCH 12/12] ENH: hostCollated: naming without ranges if running on
 single rank

---
 .../collatedFileOperation/collatedFileOperation.C | 15 ++++++++++-----
 1 file changed, 10 insertions(+), 5 deletions(-)

diff --git a/src/OpenFOAM/global/fileOperations/collatedFileOperation/collatedFileOperation.C b/src/OpenFOAM/global/fileOperations/collatedFileOperation/collatedFileOperation.C
index be88c0c135..5bfe78a1f4 100644
--- a/src/OpenFOAM/global/fileOperations/collatedFileOperation/collatedFileOperation.C
+++ b/src/OpenFOAM/global/fileOperations/collatedFileOperation/collatedFileOperation.C
@@ -573,11 +573,16 @@ Foam::word Foam::fileOperations::collatedFileOperation::processorsDir
                         break;
                     }
                 }
-                procDir +=
-                  + "_"
-                  + Foam::name(minProc)
-                  + "-"
-                  + Foam::name(maxProc);
+
+                // Add range if not all processors
+                if (maxProc-minProc+1 != nProcs_)
+                {
+                    procDir +=
+                      + "_"
+                      + Foam::name(minProc)
+                      + "-"
+                      + Foam::name(maxProc);
+                }
             }
         }