diff --git a/src/finiteArea/faMesh/faMeshTools/faMeshTools.C b/src/finiteArea/faMesh/faMeshTools/faMeshTools.C index cce1e3c6d7..102dd4046b 100644 --- a/src/finiteArea/faMesh/faMeshTools/faMeshTools.C +++ b/src/finiteArea/faMesh/faMeshTools/faMeshTools.C @@ -30,6 +30,8 @@ License #include "faBoundaryMeshEntries.H" #include "areaFields.H" #include "edgeFields.H" +#include "fileOperation.H" +#include "BitOps.H" #include "polyMesh.H" #include "processorFaPatch.H" @@ -66,7 +68,8 @@ void Foam::faMeshTools::forceDemandDriven(faMesh& mesh) } -Foam::autoPtr Foam::faMeshTools::newMesh +Foam::autoPtr +Foam::faMeshTools::newMesh ( const IOobject& io, const polyMesh& pMesh, @@ -248,9 +251,11 @@ Foam::autoPtr Foam::faMeshTools::newMesh } -Foam::autoPtr Foam::faMeshTools::loadOrCreateMesh +Foam::autoPtr +Foam::faMeshTools::loadOrCreateMeshImpl ( const IOobject& io, + refPtr* readHandlerPtr, // Can be nullptr const polyMesh& pMesh, const bool decompose, const bool verbose @@ -270,9 +275,11 @@ Foam::autoPtr Foam::faMeshTools::loadOrCreateMesh // Read and scatter master patches (without reading master mesh!) PtrList patchEntries; - if (Pstream::master()) + if (UPstream::master()) { - const bool oldParRun = Pstream::parRun(false); + const bool oldParRun = UPstream::parRun(false); + const label oldNumProcs = fileHandler().nProcs(); + const int oldCache = fileOperation::cacheLevel(0); patchEntries = faBoundaryMeshEntries ( @@ -288,46 +295,75 @@ Foam::autoPtr Foam::faMeshTools::loadOrCreateMesh ) ); - Pstream::parRun(oldParRun); + fileOperation::cacheLevel(oldCache); + if (oldParRun) + { + const_cast(fileHandler()).nProcs(oldNumProcs); + } + UPstream::parRun(oldParRun); } // Broadcast: send patches to all - Pstream::broadcast(patchEntries); // == worldComm; + Pstream::broadcast(patchEntries, UPstream::worldComm); - /// Info<< patchEntries << nl; - - // Dummy meshes - // ~~~~~~~~~~~~ // Check who has or needs a mesh. - // For 'decompose', only need mesh on master. - // Otherwise check for presence of the "faceLabels" file + bool haveLocalMesh = false; - bool haveMesh = - ( - decompose - ? Pstream::master() - : fileHandler().isFile + if (readHandlerPtr) + { + // Non-null reference when a mesh exists on given processor + haveLocalMesh = (*readHandlerPtr).good(); + } + else + { + // No file handler. + // For 'decompose', only need mesh on master. + // Otherwise check for presence of the "faceLabels" file + + haveLocalMesh = ( - fileHandler().filePath + decompose + ? UPstream::master() + : fileHandler().isFile ( - io.time().path()/io.instance()/meshSubDir/"faceLabels" + fileHandler().filePath + ( + io.time().path()/io.instance()/meshSubDir/"faceLabels" + ) ) - ) + ); + } + + + // Globally consistent information about who has a mesh + boolList haveMesh + ( + UPstream::allGatherValues(haveLocalMesh) ); - if (!haveMesh) + autoPtr meshPtr; + + if (!haveLocalMesh) { - const bool oldParRun = Pstream::parRun(false); + // No local mesh - need to synthesize one + + const bool oldParRun = UPstream::parRun(false); + const label oldNumProcs = fileHandler().nProcs(); + const int oldCache = fileOperation::cacheLevel(0); // Create dummy mesh - on procs that don't already have a mesh - faMesh dummyMesh + meshPtr.reset ( - pMesh, - labelList(), - IOobject(io, IOobject::NO_READ, IOobject::AUTO_WRITE) + new faMesh + ( + pMesh, + labelList(), + IOobject(io, IOobject::NO_READ, IOobject::AUTO_WRITE) + ) ); + faMesh& mesh = *meshPtr; // Add patches faPatchList patches(patchEntries.size()); @@ -361,39 +397,119 @@ Foam::autoPtr Foam::faMeshTools::loadOrCreateMesh name, patchDict, nPatches++, - dummyMesh.boundary() + mesh.boundary() ) ); } } - patches.resize(nPatches); - dummyMesh.addFaPatches(patches, false); // No parallel comms + mesh.addFaPatches(patches, false); // No parallel comms - // Bad hack, but the underlying polyMesh is NO_WRITE - // so it does not create the faMesh subDir for us... - Foam::mkDir(dummyMesh.boundary().path()); + if (!readHandlerPtr) + { + // The 'old' way of doing things. + // Write the dummy mesh to disk for subsequent re-reading. + // + // This is not particularly elegant. - //Pout<< "Writing dummy mesh to " << dummyMesh.boundary().path() - // << endl; - dummyMesh.write(); + // Bad hack, but the underlying polyMesh is NO_WRITE + // so it does not create the faMesh subDir for us... + Foam::mkDir(mesh.boundary().path()); - Pstream::parRun(oldParRun); // Restore parallel state + //Pout<< "Writing dummy mesh to " << mesh.boundary().path() << nl; + mesh.write(); + + // Discard - it will be re-read later + meshPtr.reset(nullptr); + } + + fileOperation::cacheLevel(oldCache); + if (oldParRun) + { + const_cast(fileHandler()).nProcs(oldNumProcs); + } + UPstream::parRun(oldParRun); // Restore parallel state + } + else if (readHandlerPtr && haveLocalMesh) + { + const labelList meshProcIds(BitOps::sortedToc(haveMesh)); + + UPstream::communicator newCommunicator; + const label oldWorldComm = UPstream::commWorld(); + + auto& readHandler = *readHandlerPtr; + auto oldHandler = fileOperation::fileHandler(readHandler); + + // With IO ranks the communicator of the fileOperation will + // only include the ranks for the current IO rank. + // Instead allocate a new communicator for everyone with a mesh + + const auto& handlerProcIds = UPstream::procID(fileHandler().comm()); + + // Comparing global ranks in the communicator. + // Use std::equal for the List