BUG: redistributePar: single-step. See #1211

- override casename, procesorCase flags to guarantee reconstructed
case to be written to the undecomposed directory
- alternative is to construct a Zero mesh on the undecomposed
runTime and add all other bits to that but that has not been
pursued

applications/utilities/parallelProcessing/reconstructParMesh/reconstructParMesh.C

@@ -1033,18 +1033,6 @@ int main(int argc, char *argv[])
                 meshes.set(proci, &fvMeshes[proci]);
             }
 
-            // Collect statistics
-            label nCells = 0;
-            label nFaces = 0;
-            label nPoints = 0;
-            forAll(meshes, proci)
-            {
-                const polyMesh& mesh = meshes[proci];
-                nCells += mesh.nCells();
-                nFaces += mesh.nFaces();
-                nPoints += mesh.nPoints();
-            }
-
             // Get pairs of patches to stitch. These pairs have to
             // - have ordered, opposite faces (so one to one correspondence)
             List<DynamicList<label>> localPatch;
@@ -1167,16 +1155,25 @@ int main(int argc, char *argv[])
             masterOwner = masterMesh.faceOwner();
 
             // Write reconstructed mesh
-            const word oldCaseName = masterMesh.time().caseName();
+            // Override:
-            const_cast<Time&>(masterMesh.time()).caseName() =
+            //  - caseName
-                runTime.caseName();
+            //  - processorCase flag
+            // so the resulting mesh goes to the correct location (even with
+            // collated). The better way of solving this is to construct
+            // (zero) mesh on the undecomposed runTime.
+            Time& masterTime = const_cast<Time&>(masterMesh.time());
+
+            const word oldCaseName = masterTime.caseName();
+            masterTime.caseName() = runTime.caseName();
+            const bool oldProcCase(masterTime.processorCase(false));
 
             writeMesh(masterMesh, cellProcAddressing);
             if (writeCellDist)
             {
                 writeDistribution(runTime, masterMesh, cellProcAddressing);
             }
-            const_cast<Time&>(masterMesh.time()).caseName() = oldCaseName;
+            masterTime.caseName() = oldCaseName;
+            masterTime.processorCase(oldProcCase);
         }
 
 

applications/utilities/parallelProcessing/redistributePar/redistributePar.C

@@ -366,9 +366,11 @@ void determineDecomposition
                 Time& tm = const_cast<Time&>(mesh.time());
 
                 tm.caseName() = baseRunTime.caseName();
+                const bool oldProcCase(tm.processorCase(false));
                 writeDecomposition("cellDist", mesh, decomp);
                 Info<< "Restoring caseName to " << proc0CaseName << endl;
                 tm.caseName() = proc0CaseName;
+                tm.processorCase(oldProcCase);
             }
         }
         else
@@ -1178,6 +1180,7 @@ autoPtr<mapDistributePolyMesh> redistributeAndWrite
             Info<< "Setting caseName to " << baseRunTime.caseName()
                 << " to write reconstructed mesh and fields." << endl;
             runTime.caseName() = baseRunTime.caseName();
+            const bool oldProcCase(runTime.processorCase(false));
 
             mesh.write();
             topoSet::removeFiles(mesh);
@@ -1198,6 +1201,7 @@ autoPtr<mapDistributePolyMesh> redistributeAndWrite
             // Now we've written all. Reset caseName on master
             Info<< "Restoring caseName to " << proc0CaseName << endl;
             runTime.caseName() = proc0CaseName;
+            runTime.processorCase(oldProcCase);
         }
     }
     else
@@ -1280,12 +1284,14 @@ autoPtr<mapDistributePolyMesh> redistributeAndWrite
                 Info<< "Setting caseName to " << baseRunTime.caseName()
                     << " to write reconstructed refinement data." << endl;
                 runTime.caseName() = baseRunTime.caseName();
+                const bool oldProcCase(runTime.processorCase(false));
 
                 refData.write();
 
                 // Now we've written all. Reset caseName on master
                 Info<< "Restoring caseName to " << proc0CaseName << endl;
                 runTime.caseName() = proc0CaseName;
+                runTime.processorCase(oldProcCase);
             }
         }
         else
@@ -1323,6 +1329,7 @@ autoPtr<mapDistributePolyMesh> redistributeAndWrite
     //            Info<< "Setting caseName to " << baseRunTime.caseName()
     //                << " to write reconstructed refinement data." << endl;
     //            runTime.caseName() = baseRunTime.caseName();
+    //            const bool oldProcCase(runTime.processorCase(false));
     //
     //            forAll(cellSets, i)
     //            {
@@ -1332,6 +1339,7 @@ autoPtr<mapDistributePolyMesh> redistributeAndWrite
     //            // Now we've written all. Reset caseName on master
     //            Info<< "Restoring caseName to " << proc0CaseName << endl;
     //            runTime.caseName() = proc0CaseName;
+    //            runTime.processorCase(oldProcCase);
     //        }
     //    }
     //    else
@@ -2383,12 +2391,17 @@ int main(int argc, char *argv[])
             << nl << endl;
     }
 
-    if (isDir(args.path()))
+    // Check if we have processor directories. Ideally would like to
+    // use fileHandler().dirPath here but we don't have runTime yet and
+    // want to delay constructing runTime until we've synced all time
+    // directories...
+    const fileName procDir(fileHandler().filePath(args.path()));
+    if (isDir(procDir))
     {
         if (decompose)
         {
             Info<< "Removing existing processor directories" << endl;
-            rmDir(args.path());
+            rmDir(procDir);
         }
     }
     else
@@ -2408,19 +2421,20 @@ int main(int argc, char *argv[])
     // e.g. latestTime will pick up a different time (which causes createTime.H
     // to abort). So for now make sure to have master times on all
     // processors
+    if (!procDir.empty())
     {
         Info<< "Creating time directories on all processors" << nl << endl;
         instantList timeDirs;
         if (Pstream::master())
         {
             const bool oldParRun = Pstream::parRun(false);
-            timeDirs = Time::findTimes(args.path(), "constant");
+            timeDirs = Time::findTimes(procDir, "constant");
             Pstream::parRun(oldParRun);  // Restore parallel state
         }
         Pstream::scatter(timeDirs);
         for (const instant& t : timeDirs)
         {
-            mkDir(args.path()/t.name());
+            mkDir(procDir/t.name());
         }
     }
 

src/OpenFOAM/db/Time/TimePaths.H

@@ -6,7 +6,7 @@
      \\/     M anipulation  |
 -------------------------------------------------------------------------------
     Copyright (C) 2011-2014 OpenFOAM Foundation
-    Copyright (C) 2016-2018 OpenCFD Ltd.
+    Copyright (C) 2016-2021 OpenCFD Ltd.
 -------------------------------------------------------------------------------
 License
     This file is part of OpenFOAM.
@@ -118,6 +118,9 @@ public:
         //- Return true if this is a processor case
         inline bool processorCase() const;
 
+        //- Set processor case status. Return old one
+        inline bool processorCase(const bool isProcessorCase);
+
         //- Return root path
         inline const fileName& rootPath() const;
 

src/OpenFOAM/db/Time/TimePathsI.H

@@ -5,7 +5,7 @@
     \\  /    A nd           | www.openfoam.com
      \\/     M anipulation  |
 -------------------------------------------------------------------------------
-    Copyright (C) 2018 OpenCFD Ltd.
+    Copyright (C) 2018-2021 OpenCFD Ltd.
 -------------------------------------------------------------------------------
 License
     This file is part of OpenFOAM.
@@ -39,6 +39,14 @@ inline bool Foam::TimePaths::processorCase() const
 }
 
 
+inline bool Foam::TimePaths::processorCase(const bool newProcCase)
+{
+    const bool oldProcCase = processorCase_;
+    processorCase_ = newProcCase;
+    return oldProcCase;
+}
+
+
 inline const Foam::fileName& Foam::TimePaths::rootPath() const
 {
     return rootPath_;

tutorials/heatTransfer/buoyantBoussinesqSimpleFoam/iglooWithFridges/Allrun

@@ -20,7 +20,7 @@ runApplication $(getApplication)
 #)
 #
 #runParallel $(getApplication) -fileHandler collated
-#runApplication reconstructParMesh -constant  -mergeTol 1e-6
+#runApplication reconstructParMesh -constant
 #runApplication reconstructPar
 
 #------------------------------------------------------------------------------

tutorials/incompressible/pisoFoam/LES/motorBike/lesFiles/Allrun

@@ -14,7 +14,7 @@ cd "${0%/*}" || exit                                # Run from this directory
 
 runParallel pisoFoam
 
-runApplication reconstructParMesh -constant -mergeTol 1e-6
+runApplication reconstructParMesh -constant
 
 runApplication reconstructPar
 

tutorials/mesh/snappyHexMesh/iglooWithFridgesDirectionalRefinement/Allrun

@@ -12,6 +12,6 @@ runApplication blockMesh
 ## Parallel
 runApplication decomposePar -fileHandler collated
 runParallel snappyHexMesh -overwrite -fileHandler collated
-runApplication reconstructParMesh -constant -fileHandler collated -mergeTol 1e-6
+runApplication reconstructParMesh -constant -fileHandler collated
 
 #------------------------------------------------------------------------------