ENH: improve handling of finiteArea mesh with distributed roots

- in makeFaMesh, the serial fields are now only read on the master
  process and broadcast to the other ranks. The read+distribute is
  almost identical to that used in redistributePar, except that in
  this case entire fields are sent and not a zero-sized subset.

- improved internal faMesh checking for files so that the TryNew
  method works with distributed roots.

applications/utilities/finiteArea/checkFaMesh/checkFaMesh.C

@@ -95,7 +95,8 @@ int main(int argc, char *argv[])
 
     Info<< "Time = " << runTime.timeName() << nl << endl;
 
-    #include "printMeshSummary.H"
+    // Mesh information (verbose)
+    faMeshTools::printMeshChecks(aMesh);
 
     if (args.found("write-vtk"))
     {

applications/utilities/finiteArea/makeFaMesh/decomposeFaFields.H

@@ -30,44 +30,63 @@ do
         break;
     }
 
-    reconstructor.writeAddressing();
+    reconstructor.writeMesh();          // Writes on master only
+    reconstructor.writeAddressing();    // Writes per-proc
 
-    Info<< "Wrote proc-addressing" << nl << endl;
+    Info<< "Wrote proc-addressing and serial mesh" << nl << endl;
 
     // Handle area fields
     // ------------------
 
     faFieldDecomposer::fieldsCache areaFieldsCache;
 
-    const faMesh& fullMesh = reconstructor.mesh();
+    const faMesh& serialMesh = reconstructor.mesh();
 
+    if (doDecompFields)
     {
-        // Use uncollated (or master uncollated) file handler here.
+        // The serial finite-area mesh exists and is identical on all
-        // - each processor is reading in the identical serial fields.
+        // processors, but its fields can only reliably be read on the
-        // - nothing should be parallel-coordinated.
+        // master (eg, running with distributed roots).
+        //
+        // - mark mesh fields as readable on master only (haveMeshOnProc)
+        // - 'subset' entire serial mesh so that a full copy will be
+        //   broadcast to other ranks (subsetterPtr)
+        // - scan available IOobjects on the master only
 
-        // Similarly, if we write the serial finite-area mesh, this is only
+        bitSet haveMeshOnProc;
-        // done from one processor!
+        std::unique_ptr<faMeshSubset> subsetter;
+        IOobjectList objects(0);
 
-        reconstructor.writeMesh();
+        const bool oldDistributed = fileHandler().distributed();
+        auto oldHandler = fileHandler(fileOperation::NewUncollated());
+        fileHandler().distributed(true);
 
-        if (doDecompFields)
+        if (Pstream::master())
         {
-            const bool oldDistributed = fileHandler().distributed();
+            haveMeshOnProc.set(Pstream::myProcNo());
-            auto oldHandler = fileHandler(fileOperation::NewUncollated());
+            subsetter.reset(new faMeshSubset(serialMesh));
-            fileHandler().distributed(true);
 
-            IOobjectList objects(fullMesh.time(), runTime.timeName());
+            const bool oldParRun = Pstream::parRun(false);
 
-            areaFieldsCache.readAllFields(fullMesh, objects);
+            objects = IOobjectList(serialMesh.time(), runTime.timeName());
 
-            // Restore old settings
+            Pstream::parRun(oldParRun);
-            if (oldHandler)
-            {
-                fileHandler(std::move(oldHandler));
-            }
-            fileHandler().distributed(oldDistributed);
         }
+
+        // Restore old settings
+        if (oldHandler)
+        {
+            fileHandler(std::move(oldHandler));
+        }
+        fileHandler().distributed(oldDistributed);
+
+        areaFieldsCache.readAllFields
+        (
+            haveMeshOnProc,
+            subsetter.get(),
+            serialMesh,
+            objects
+        );
     }
 
     const label nAreaFields = areaFieldsCache.size();
@@ -78,7 +97,7 @@ do
 
         faFieldDecomposer fieldDecomposer
         (
-            fullMesh,
+            serialMesh,
             aMesh,
             reconstructor.edgeProcAddressing(),
             reconstructor.faceProcAddressing(),

applications/utilities/finiteArea/makeFaMesh/makeFaMesh.C

@@ -6,7 +6,7 @@
      \\/     M anipulation  |
 -------------------------------------------------------------------------------
     Copyright (C) 2016-2017 Wikki Ltd
-    Copyright (C) 2021 OpenCFD Ltd.
+    Copyright (C) 2021-2022 OpenCFD Ltd.
 -------------------------------------------------------------------------------
 License
     This file is part of OpenFOAM.
@@ -42,13 +42,14 @@ Original Authors
 #include "argList.H"
 #include "OSspecific.H"
 #include "faMesh.H"
+#include "faMeshTools.H"
 #include "IOdictionary.H"
 #include "IOobjectList.H"
-
 #include "areaFields.H"
 #include "edgeFields.H"
 #include "faFieldDecomposer.H"
 #include "faMeshReconstructor.H"
+#include "faMeshSubset.H"
 #include "PtrListOps.H"
 #include "foamVtkIndPatchWriter.H"
 #include "OBJstream.H"
@@ -131,8 +132,8 @@ int main(int argc, char *argv[])
     // Create
     faMesh aMesh(mesh, meshDefDict);
 
-    // Mesh information
+    // Mesh information (less verbose)
-    #include "printMeshSummary.H"
+    faMeshTools::printMeshChecks(aMesh, 0);
 
     if (args.found("write-edges-obj"))
     {

applications/utilities/finiteArea/makeFaMesh/printMeshSummary.H

@@ -1,104 +0,0 @@
-/*---------------------------------------------------------------------------*\
-  =========                 |
-  \\      /  F ield         | OpenFOAM: The Open Source CFD Toolbox
-   \\    /   O peration     |
-    \\  /    A nd           | www.openfoam.com
-     \\/     M anipulation  |
--------------------------------------------------------------------------------
-    Copyright (C) 2021-2022 OpenCFD Ltd.
--------------------------------------------------------------------------------
-License
-    This file is part of OpenFOAM, distributed under GPL-3.0-or-later.
-
-Description
-    Summary of faMesh information
-
-\*---------------------------------------------------------------------------*/
-
-{
-    const faBoundaryMesh& patches = aMesh.boundary();
-    const label nNonProcessor = patches.nNonProcessor();
-    const label nPatches = patches.size();
-
-    Info<< "----------------" << nl
-        << "Mesh Information" << nl
-        << "----------------" << nl
-        << "  " << "boundingBox: " << boundBox(aMesh.points()) << nl
-        << "  " << "nFaces: " << returnReduce(aMesh.nFaces(), sumOp<label>())
-        << nl;
-
-
-    Info<< "----------------" << nl
-        << "Patches" << nl
-        << "----------------" << nl;
-
-    for (label patchi = 0; patchi < nNonProcessor; ++patchi)
-    {
-        const faPatch& p = patches[patchi];
-
-        // Report physical size (nEdges) not virtual size
-        Info<< "  " << "patch " << p.index()
-            << " (size: " << returnReduce(p.nEdges(), sumOp<label>())
-            << ") name: " << p.name()
-            << nl;
-    }
-
-    Info<< "----------------" << nl
-        << "Used polyPatches: " << flatOutput(aMesh.whichPolyPatches()) << nl;
-
-
-    // Geometry information
-    Info<< nl;
-    {
-        scalarMinMax limit(gMinMax(aMesh.S().field()));
-        Info<< "Face area:" << nl
-            << "    min = " << limit.min() << " max = "  << limit.max() << nl;
-    }
-
-    {
-        scalarMinMax limit(minMax(aMesh.magLe().primitiveField()));
-
-        // Include processor boundaries into 'internal' edges
-        if (Pstream::parRun())
-        {
-            for (label patchi = nNonProcessor; patchi < nPatches; ++patchi)
-            {
-                limit.add(minMax(aMesh.magLe().boundaryField()[patchi]));
-            }
-
-            reduce(limit, minMaxOp<scalar>());
-        }
-
-        Info<< "Edge length (internal):" << nl
-            << "    min = " << limit.min() << " max = "  << limit.max() << nl;
-
-
-        // Include (non-processor) boundaries
-        for (label patchi = 0; patchi < nNonProcessor; ++patchi)
-        {
-            limit.add(minMax(aMesh.magLe().boundaryField()[patchi]));
-        }
-
-        if (Pstream::parRun())
-        {
-            reduce(limit, minMaxOp<scalar>());
-        }
-
-        Info<< "Edge length:" << nl
-            << "    min = " << limit.min()
-            << " max = "  << limit.max() << nl;
-    }
-
-    // Not particularly meaningful
-    #if 0
-    {
-        MinMax<vector> limit(gMinMax(aMesh.faceAreaNormals().field()));
-
-        Info<< "Face area normals:" << nl
-            << "    min = " << limit.min() << " max = " << limit.max() << nl;
-    }
-    #endif
-}
-
-
-// ************************************************************************* //

src/OpenFOAM/parallel/fieldsDistributor/fieldsDistributor.H

@@ -38,6 +38,7 @@ SourceFiles
 #define Foam_fieldsDistributor_H
 
 #include "IOobjectList.H"
+#include "bitSet.H"
 #include "boolList.H"
 #include "PtrList.H"
 #include "GeometricField.H"
@@ -53,6 +54,22 @@ namespace Foam
 
 class fieldsDistributor
 {
+    // Private Methods
+
+    //- Read volume/surface/point/area fields that may or may not exist
+    //- on all processors
+    template<class BoolListType, class GeoField, class MeshSubsetter>
+    static void readFieldsImpl
+    (
+        const BoolListType& haveMeshOnProc,
+        const MeshSubsetter* subsetter,
+        const typename GeoField::Mesh& mesh,
+        IOobjectList& allObjects,
+        PtrList<GeoField>& fields,
+        const bool deregister
+    );
+
+
 public:
 
     // Reading helpers
@@ -103,6 +120,32 @@ public:
     );
 
 
+    //- Read volume/surface/point/area fields that may or may not exist
+    //- on all processors
+    template<class GeoField, class MeshSubsetter>
+    static void readFields
+    (
+        const bitSet& haveMeshOnProc,
+        const MeshSubsetter* subsetter,
+        const typename GeoField::Mesh& mesh,
+        IOobjectList& allObjects,
+        PtrList<GeoField>& fields,
+        const bool deregister = false
+    );
+
+    //- Read volume/surface/point/area fields that may or may not exist
+    //- on all processors
+    template<class GeoField, class MeshSubsetter>
+    static void readFields
+    (
+        const boolList& haveMeshOnProc,
+        const MeshSubsetter* subsetter,
+        const typename GeoField::Mesh& mesh,
+        IOobjectList& allObjects,
+        PtrList<GeoField>& fields,
+        const bool deregister = false
+    );
+
     //- Read volume/surface/point/area fields that may or may not exist
     //- on all processors
     template<class GeoField, class MeshSubsetter>
@@ -110,7 +153,7 @@ public:
     (
         const boolList& haveMeshOnProc,
         const typename GeoField::Mesh& mesh,
-        const autoPtr<MeshSubsetter>& subsetterPtr,
+        const autoPtr<MeshSubsetter>& subsetter,
         IOobjectList& allObjects,
         PtrList<GeoField>& fields,
         const bool deregister = false

src/OpenFOAM/parallel/fieldsDistributor/fieldsDistributorTemplates.C

@@ -101,12 +101,12 @@ void Foam::fieldsDistributor::readFields
 }
 
 
-template<class GeoField, class MeshSubsetter>
+template<class BoolListType, class GeoField, class MeshSubsetter>
-void Foam::fieldsDistributor::readFields
+void Foam::fieldsDistributor::readFieldsImpl
 (
-    const boolList& haveMeshOnProc,
+    const BoolListType& haveMeshOnProc,
+    const MeshSubsetter* subsetter,
     const typename GeoField::Mesh& mesh,
-    const autoPtr<MeshSubsetter>& subsetterPtr,
     IOobjectList& allObjects,
     PtrList<GeoField>& fields,
     const bool deregister
@@ -122,7 +122,7 @@ void Foam::fieldsDistributor::readFields
     wordList masterNames(objectNames);
     Pstream::broadcast(masterNames);
 
-    if (haveMeshOnProc[Pstream::myProcNo()] && objectNames != masterNames)
+    if (haveMeshOnProc.test(Pstream::myProcNo()) && objectNames != masterNames)
     {
         FatalErrorInFunction
             << "Objects not synchronised across processors." << nl
@@ -173,9 +173,10 @@ void Foam::fieldsDistributor::readFields
         bool decompose = true;
         for (const int proci : Pstream::subProcs())
         {
-            if (haveMeshOnProc[proci])
+            if (haveMeshOnProc.test(proci))
             {
                 decompose = false;
+                break;
             }
         }
 
@@ -197,7 +198,7 @@ void Foam::fieldsDistributor::readFields
 
         Pstream::parRun(oldParRun);  // Restore any changes
     }
-    else if (haveMeshOnProc[Pstream::myProcNo()])
+    else if (haveMeshOnProc.test(Pstream::myProcNo()))
     {
         // Have mesh so just try to load
         forAll(masterNames, i)
@@ -226,20 +227,18 @@ void Foam::fieldsDistributor::readFields
 
         OPBstream toProcs(UPstream::masterNo());  // worldComm
 
-        const label nDicts = (subsetterPtr ? fields.size() : label(0));
+        const label nDicts = (subsetter ? fields.size() : label(0));
 
         toProcs << nDicts << token::BEGIN_LIST;  // Begin list
 
-        if (nDicts)
+        if (nDicts && subsetter)
         {
             // Disable communication for interpolate() method
             const bool oldParRun = Pstream::parRun(false);
 
-            const auto& subsetter = subsetterPtr();
+            for (const auto& fld : fields)
-
-            forAll(fields, i)
             {
-                tmp<GeoField> tsubfld = subsetter.interpolate(fields[i]);
+                tmp<GeoField> tsubfld = subsetter->interpolate(fld);
 
                 // Surround each with {} as dictionary entry
                 toProcs.beginBlock();
@@ -258,15 +257,14 @@ void Foam::fieldsDistributor::readFields
         IPBstream fromMaster(UPstream::masterNo());  // worldComm
 
         // But only consume where needed...
-        if (!haveMeshOnProc[Pstream::myProcNo()])
+        if (!haveMeshOnProc.test(Pstream::myProcNo()))
         {
             fromMaster >> fieldDicts;
         }
     }
 
 
-    // Use the received dictionaries to create fields
+    // Use the received dictionaries (if any) to create missing fields.
-    // (will be empty if we didn't require them)
 
     // Disable communication when constructing from dictionary
     const bool oldParRun = Pstream::parRun(false);
@@ -327,4 +325,76 @@ void Foam::fieldsDistributor::readFields
 }
 
 
+template<class GeoField, class MeshSubsetter>
+void Foam::fieldsDistributor::readFields
+(
+    const bitSet& haveMeshOnProc,
+    const MeshSubsetter* subsetter,
+    const typename GeoField::Mesh& mesh,
+    IOobjectList& allObjects,
+    PtrList<GeoField>& fields,
+    const bool deregister
+)
+{
+    readFieldsImpl
+    (
+        haveMeshOnProc,
+        subsetter,
+
+        mesh,
+        allObjects,
+        fields,
+        deregister
+    );
+}
+
+
+template<class GeoField, class MeshSubsetter>
+void Foam::fieldsDistributor::readFields
+(
+    const boolList& haveMeshOnProc,
+    const MeshSubsetter* subsetter,
+    const typename GeoField::Mesh& mesh,
+    IOobjectList& allObjects,
+    PtrList<GeoField>& fields,
+    const bool deregister
+)
+{
+    readFieldsImpl
+    (
+        haveMeshOnProc,
+        subsetter,
+
+        mesh,
+        allObjects,
+        fields,
+        deregister
+    );
+}
+
+
+template<class GeoField, class MeshSubsetter>
+void Foam::fieldsDistributor::readFields
+(
+    const boolList& haveMeshOnProc,
+    const typename GeoField::Mesh& mesh,
+    const autoPtr<MeshSubsetter>& subsetter,
+    IOobjectList& allObjects,
+    PtrList<GeoField>& fields,
+    const bool deregister
+)
+{
+    readFieldsImpl
+    (
+        haveMeshOnProc,
+        subsetter.get(),
+
+        mesh,
+        allObjects,
+        fields,
+        deregister
+    );
+}
+
+
 // ************************************************************************* //

src/finiteArea/Make/files

@@ -11,6 +11,7 @@ faMesh/faBoundaryMesh/faBoundaryMeshEntries.C
 
 faMesh/faMeshSubset/faMeshSubset.C
 faMesh/faMeshTools/faMeshTools.C
+faMesh/faMeshTools/faMeshToolsChecks.C
 faMesh/faMeshTools/faMeshToolsProcAddr.C
 
 faPatches = faMesh/faPatches

src/finiteArea/faMesh/faMesh.C

@@ -978,7 +978,7 @@ bool Foam::faMesh::movePoints()
                     mesh(),
                     IOobject::NO_READ,
                     IOobject::NO_WRITE,
-                    false
+                    IOobject::NO_REGISTER
                 ),
                 S()
             );

src/finiteArea/faMesh/faMeshNew.C

@@ -64,7 +64,7 @@ bool Foam::faMesh::hasSystemFiles(const polyMesh& pMesh)
                     pMesh,
                     IOobject::MUST_READ,
                     IOobject::NO_WRITE,
-                    false
+                    IOobject::NO_REGISTER
                 ),
                 expect // typeName (ununsed?)
             )
@@ -76,6 +76,7 @@ bool Foam::faMesh::hasSystemFiles(const polyMesh& pMesh)
         }
     }
 
+    // Only needed on master
     Pstream::broadcast(looksValid);
 
     return looksValid;
@@ -87,8 +88,8 @@ bool Foam::faMesh::hasFiles(const polyMesh& pMesh)
     // As well as system/{faSchemes,faSolution}
     //
     // expect these:
-    // - timeValue/faMesh/faBoundary
+    // - instance/faMesh/faceLabels
-    // - timeValue/instance/faMesh/faceLabels
+    // - instance/faMesh/faBoundary
 
     bool looksValid = hasSystemFiles(pMesh);
 
@@ -96,15 +97,23 @@ bool Foam::faMesh::hasFiles(const polyMesh& pMesh)
     {
         const fileOperation& fp = Foam::fileHandler();
 
-        fileName subDir(pMesh.dbDir()/faMesh::meshSubDir);
+        // The geometry instance for faMesh/faceLabels
+        // Must use READ_IF_PRESENT to avoid aborting if not available
+
+        const word instance = pMesh.time().findInstance
+        (
+            pMesh.dbDir()/faMesh::meshSubDir,
+            "faceLabels",
+            IOobject::READ_IF_PRESENT
+        );
 
         for
         (
             const wordPair& expect
           : List<wordPair>
             ({
-                {"faBoundary", "faBoundaryMesh"},
+                {"faceLabels", "labelList"},
-                {"faceLabels", "labelList"}
+                {"faBoundary", "faBoundaryMesh"}
             })
         )
         {
@@ -115,18 +124,18 @@ bool Foam::faMesh::hasFiles(const polyMesh& pMesh)
             (
                 fp.filePath
                 (
-                    false,  // non-global
+                    false,      // non-global
                     IOobject
                     (
                         dataFile,
-                        pMesh.time().findInstance(subDir, dataFile),
+                        instance,
                         faMesh::meshSubDir,
                         pMesh,
-                        IOobject::MUST_READ,
+                        IOobject::READ_IF_PRESENT,
                         IOobject::NO_WRITE,
-                        false
+                        IOobject::NO_REGISTER
                     ),
-                    dataClass // typeName (ununsed?)
+                    dataClass   // typeName (ununsed?)
                 )
             );
 
@@ -136,7 +145,8 @@ bool Foam::faMesh::hasFiles(const polyMesh& pMesh)
             }
         }
 
-        Pstream::broadcast(looksValid);
+        // Everybody needs it, or they all fail
+        Pstream::reduceAnd(looksValid);
     }
 
     return looksValid;

src/finiteArea/faMesh/faMeshTools/faMeshTools.C

@@ -109,7 +109,7 @@ Foam::autoPtr<Foam::faMesh> Foam::faMeshTools::newMesh
                 io.db(),
                 IOobject::MUST_READ,
                 IOobject::NO_WRITE,
-                false
+                IOobject::NO_REGISTER
             )
         );
 
@@ -282,7 +282,7 @@ Foam::autoPtr<Foam::faMesh> Foam::faMeshTools::loadOrCreateMesh
                 io.db(),
                 IOobject::MUST_READ,
                 IOobject::NO_WRITE,
-                false
+                IOobject::NO_REGISTER
             )
         );
 

src/finiteArea/faMesh/faMeshTools/faMeshTools.H

@@ -31,6 +31,7 @@ Description
 
 SourceFiles
     faMeshTools.C
+    faMeshToolsChecks.C
     faMeshToolsProcAddr.C
     faMeshToolsTemplates.C
 
@@ -134,6 +135,13 @@ public:
         const GeometricField<Type, faePatchField, edgeMesh>& fld,
         const bool primitiveOrdering = false
     );
+
+    //- Report mesh information
+    static void printMeshChecks
+    (
+        const faMesh& mesh,
+        const int verbose = 1
+    );
 };
 
 

src/finiteArea/faMesh/faMeshTools/faMeshToolsChecks.C

@@ -8,15 +8,37 @@
     Copyright (C) 2021-2022 OpenCFD Ltd.
 -------------------------------------------------------------------------------
 License
-    This file is part of OpenFOAM, distributed under GPL-3.0-or-later.
+    This file is part of OpenFOAM.
 
-Description
+    OpenFOAM is free software: you can redistribute it and/or modify it
-    Summary of faMesh information
+    under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    OpenFOAM is distributed in the hope that it will be useful, but WITHOUT
+    ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+    FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+    for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with OpenFOAM.  If not, see <http://www.gnu.org/licenses/>.
 
 \*---------------------------------------------------------------------------*/
 
+#include "faMeshTools.H"
+#include "areaFields.H"
+#include "edgeFields.H"
+#include "processorFaPatch.H"
+
+// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
+
+void Foam::faMeshTools::printMeshChecks
+(
+    const faMesh& mesh,
+    const int verbose
+)
 {
-    const faBoundaryMesh& patches = aMesh.boundary();
+    const faBoundaryMesh& patches = mesh.boundary();
     const label nNonProcessor = patches.nNonProcessor();
     const label nPatches = patches.size();
 
@@ -36,19 +58,19 @@ Description
 
     const labelList nFaces
     (
-        UPstream::listGatherValues<label>(aMesh.nFaces())
+        UPstream::listGatherValues<label>(mesh.nFaces())
     );
     const labelList nPoints
     (
-        UPstream::listGatherValues<label>(aMesh.nPoints())
+        UPstream::listGatherValues<label>(mesh.nPoints())
     );
     const labelList nEdges
     (
-        UPstream::listGatherValues<label>(aMesh.nEdges())
+        UPstream::listGatherValues<label>(mesh.nEdges())
     );
     const labelList nIntEdges
     (
-        UPstream::listGatherValues<label>(aMesh.nInternalEdges())
+        UPstream::listGatherValues<label>(mesh.nInternalEdges())
     );
 
     // The "real" (non-processor) boundary edges
@@ -56,7 +78,7 @@ Description
     (
         UPstream::listGatherValues<label>
         (
-            aMesh.nBoundaryEdges() - nLocalProcEdges
+            mesh.nBoundaryEdges() - nLocalProcEdges
         )
     );
     const labelList nProcEdges
@@ -70,10 +92,10 @@ Description
     //         per-proc: (...)
 
     const auto reporter =
-        [&](const char* tag, const labelList& list)
+        [&,verbose](const char* tag, const labelList& list)
         {
             Info<< "  Number of " << tag << ": " << sum(list) << nl;
-            if (Pstream::parRun())
+            if (Pstream::parRun() && verbose)
             {
                 int padding = static_cast<int>
                 (
@@ -92,7 +114,7 @@ Description
     Info<< "----------------" << nl
         << "Mesh Information" << nl
         << "----------------" << nl
-        << "  " << "boundingBox: " << boundBox(aMesh.points()) << nl;
+        << "  " << "boundingBox: " << boundBox(mesh.points()) << nl;
 
     if (Pstream::master())
     {
@@ -124,26 +146,26 @@ Description
     }
 
     Info<< "----------------" << nl
-        << "Used polyPatches: " << flatOutput(aMesh.whichPolyPatches()) << nl;
+        << "Used polyPatches: " << flatOutput(mesh.whichPolyPatches()) << nl;
 
 
     // Geometry information
     Info<< nl;
     {
-        scalarMinMax limit(gMinMax(aMesh.S().field()));
+        scalarMinMax limit(gMinMax(mesh.S().field()));
         Info<< "Face area:" << nl
             << "    min = " << limit.min() << " max = "  << limit.max() << nl;
     }
 
     {
-        scalarMinMax limit(minMax(aMesh.magLe().primitiveField()));
+        scalarMinMax limit(minMax(mesh.magLe().primitiveField()));
 
         // Include processor boundaries into 'internal' edges
         if (Pstream::parRun())
         {
             for (label patchi = nNonProcessor; patchi < nPatches; ++patchi)
             {
-                limit.add(minMax(aMesh.magLe().boundaryField()[patchi]));
+                limit.add(minMax(mesh.magLe().boundaryField()[patchi]));
             }
 
             reduce(limit, minMaxOp<scalar>());
@@ -156,7 +178,7 @@ Description
         // Include (non-processor) boundaries
         for (label patchi = 0; patchi < nNonProcessor; ++patchi)
         {
-            limit.add(minMax(aMesh.magLe().boundaryField()[patchi]));
+            limit.add(minMax(mesh.magLe().boundaryField()[patchi]));
         }
 
         if (Pstream::parRun())
@@ -171,7 +193,7 @@ Description
     // Not particularly meaningful
     #if 0
     {
-        MinMax<vector> limit(gMinMax(aMesh.faceAreaNormals().field()));
+        MinMax<vector> limit(gMinMax(mesh.faceAreaNormals().field()));
 
         Info<< "Face area normals:" << nl
             << "    min = " << limit.min() << " max = " << limit.max() << nl;

src/finiteArea/faMesh/faMeshTools/faMeshToolsProcAddr.C

@@ -238,7 +238,7 @@ Foam::faMeshTools::readProcAddressing
         mesh.thisDb(),
         IOobject::READ_IF_PRESENT,
         IOobject::NO_WRITE,
-        false  // no register
+        IOobject::NO_REGISTER
     );
 
     //if (ioAddr.typeHeaderOk<labelIOList>(true))
@@ -327,7 +327,7 @@ void Foam::faMeshTools::writeProcAddressing
         (procMesh && !decompose ? procMesh->thisDb() : mesh.thisDb()),
         IOobject::NO_READ,
         IOobject::NO_WRITE,
-        false  // no register
+        IOobject::NO_REGISTER
     );
 
 
@@ -418,7 +418,7 @@ void Foam::faMeshTools::writeProcAddressing
             mesh.thisDb(),
             IOobject::NO_READ,
             IOobject::NO_WRITE,
-            false  // no register
+            IOobject::NO_REGISTER
         ),
         map
     );

src/parallel/decompose/faDecompose/faFieldDecomposer.H

@@ -45,6 +45,7 @@ SourceFiles
 #define Foam_faFieldDecomposer_H
 
 #include "faMesh.H"
+#include "faMeshSubset.H"
 #include "faPatchFieldMapper.H"
 #include "edgeFields.H"
 
@@ -446,6 +447,26 @@ public:
             const IOobjectList& objects
         );
 
+        //- Read all fields given mesh and objects.
+        //- Supports reading/sending fields
+        void readAllFields
+        (
+            const bitSet& haveMeshOnProc,
+            const faMeshSubset* subsetter,
+            const faMesh& mesh,
+            IOobjectList& objects
+        );
+
+        //- Read all fields given mesh and objects.
+        //- Supports reading/sending fields
+        void readAllFields
+        (
+            const boolList& haveMeshOnProc,
+            const faMeshSubset* subsetter,
+            const faMesh& mesh,
+            IOobjectList& objects
+        );
+
         //- Decompose and write all fields
         void decomposeAllFields
         (

src/parallel/decompose/faDecompose/faFieldDecomposerCache.C

@@ -77,7 +77,11 @@ public:
 
     bool empty() const noexcept { return !size(); }
 
-    void readAll(const faMesh& mesh, const IOobjectList& objects)
+    void readAll
+    (
+        const faMesh& mesh,
+        const IOobjectList& objects
+    )
     {
         #undef  doLocalCode
         #define doLocalCode(Type)                                     \
@@ -113,6 +117,45 @@ public:
         #undef doLocalCode
     }
 
+    template<class BoolListType>
+    void readAll
+    (
+        const BoolListType& haveMeshOnProc,
+        const faMeshSubset*& subsetter,
+        const faMesh& mesh,
+        IOobjectList& objects
+    )
+    {
+        #undef  doLocalCode
+        #define doLocalCode(Type)                                     \
+        {                                                             \
+            fieldsDistributor::readFields                             \
+            (                                                         \
+                haveMeshOnProc,                                       \
+                subsetter,                                            \
+                mesh,                                                 \
+                objects,                                              \
+                Type##AreaFields_                                     \
+            );                                                        \
+            fieldsDistributor::readFields                             \
+            (                                                         \
+                haveMeshOnProc,                                       \
+                subsetter,                                            \
+                mesh,                                                 \
+                objects,                                              \
+                Type##EdgeFields_                                     \
+            );                                                        \
+        }
+
+        doLocalCode(scalar);
+        doLocalCode(vector);
+        doLocalCode(sphericalTensor);
+        doLocalCode(symmTensor);
+        doLocalCode(tensor);
+
+        #undef doLocalCode
+    }
+
     template<class GeoField>
     static void decompose
     (
@@ -205,6 +248,36 @@ void Foam::faFieldDecomposer::fieldsCache::readAllFields
 }
 
 
+void Foam::faFieldDecomposer::fieldsCache::readAllFields
+(
+    const bitSet& haveMeshOnProc,
+    const faMeshSubset* subsetter,
+    const faMesh& mesh,
+    IOobjectList& objects
+)
+{
+    if (cache_)
+    {
+        cache_->readAll(haveMeshOnProc, subsetter, mesh, objects);
+    }
+}
+
+
+void Foam::faFieldDecomposer::fieldsCache::readAllFields
+(
+    const boolList& haveMeshOnProc,
+    const faMeshSubset* subsetter,
+    const faMesh& mesh,
+    IOobjectList& objects
+)
+{
+    if (cache_)
+    {
+        cache_->readAll(haveMeshOnProc, subsetter, mesh, objects);
+    }
+}
+
+
 void Foam::faFieldDecomposer::fieldsCache::decomposeAllFields
 (
     const faFieldDecomposer& decomposer,

tutorials/incompressible/pimpleFoam/laminar/filmPanel0/Allrun-distributed

@@ -0,0 +1,90 @@
+#!/bin/sh
+cd "${0%/*}" || exit                                # Run from this directory
+. ${WM_PROJECT_DIR:?}/bin/tools/RunFunctions        # Tutorial run functions
+#------------------------------------------------------------------------------
+
+## ./Allclean
+
+rootDir="test-distributed"
+caseName="${PWD##*/}"
+
+#not yet: fileHandler="-fileHandler collated"
+unset fileHandler
+
+restore0Dir
+
+runApplication blockMesh
+
+if [ -d "$rootDir" ]
+then
+    echo "Directory already exists: $rootDir"
+fi
+
+masterDecompParDict="$rootDir/machineA/$caseName/system/decomposeParDict"
+caseOption="-case $rootDir/machineA/$caseName"
+
+for subdir in machineA machineB machineC machineD
+do
+    mkdir -p "$rootDir/$subdir/$caseName"
+done
+
+# Master
+masterCase="$rootDir/machineA/$caseName"
+
+for instance in 0 constant system
+do
+    if [ -d "$masterCase/$instance" ]
+    then
+        echo "    Directory exists: $targetDir/$instance"
+    else
+        echo "Copy $instance/ to master root: $masterCase"
+        cp -R "$instance" "$masterCase"
+    fi
+done
+
+# others (nothing to copy)
+
+
+# Add distributed roots
+# - seems to be fine with relative root
+## rootDir="$PWD/$rootDir"
+
+cat<< CASE_ROOTS >> "$masterCase/system/decomposeParDict"
+
+distributed true;
+
+roots
+(
+    //master: "$rootDir/machineA"
+    "$rootDir/machineA"
+    "$rootDir/machineA"
+
+    "$rootDir/machineB"
+    "$rootDir/machineB"
+    "$rootDir/machineB"
+
+    "$rootDir/machineC"
+    "$rootDir/machineC"
+    "$rootDir/machineC"
+
+    "$rootDir/machineD"
+    "$rootDir/machineD"
+    "$rootDir/machineD"
+);
+CASE_ROOTS
+
+#------------------------------------------------------------------------------
+
+#export FOAM_ABORT=true
+
+runParallel -s decompose redistributePar -decompose -overwrite \
+    -no-finite-area $caseOption $fileHandler
+
+runParallel checkMesh $caseOption $fileHandler
+
+runParallel makeFaMesh $caseOption $fileHandler
+
+runParallel $(getApplication) $caseOption $fileHandler
+
+
+#------------------------------------------------------------------------------

tutorials/incompressible/pimpleFoam/laminar/filmPanel0/Test-distributed

@@ -1,58 +0,0 @@
-#!/bin/sh
-cd "${0%/*}" || exit                                # Run from this directory
-. ${WM_PROJECT_DIR:?}/bin/tools/RunFunctions        # Tutorial run functions
-#------------------------------------------------------------------------------
-
-## ./Allclean
-
-restore0Dir
-
-runApplication blockMesh
-
-rm -rf test-distribute
-masterDecompParDict="test-distribute/machineA/testcase/system/decomposeParDict"
-
-for subdir in machineA machineB machineC machineD
-do
-    mkdir -p test-distribute/"$subdir"/testcase
-done
-
-# master
-cp -R 0 constant system test-distribute/machineA/testcase
-# others (nothing to copy)
-
-
-cat<< CASE_ROOTS >> "$masterDecompParDict"
-
-distributed true;
-
-roots
-(
-    //master: "$PWD/test-distribute/machineA"
-    "$PWD/test-distribute/machineA"
-    "$PWD/test-distribute/machineA"
-
-    "$PWD/test-distribute/machineB"
-    "$PWD/test-distribute/machineB"
-    "$PWD/test-distribute/machineB"
-
-    "$PWD/test-distribute/machineC"
-    "$PWD/test-distribute/machineC"
-    "$PWD/test-distribute/machineC"
-
-    "$PWD/test-distribute/machineD"
-    "$PWD/test-distribute/machineD"
-    "$PWD/test-distribute/machineD"
-);
-CASE_ROOTS
-
-#export FOAM_ABORT=true
-
-runParallel -s decompose redistributePar -decompose -case test-distribute/machineA/testcase
-
-# Currently fails (OpenFOAM-v2206)
-runParallel checkFaMesh -case test-distribute/machineA/testcase
-
-exit 0
-
-#------------------------------------------------------------------------------