From 68b692fdfcf36686ba421b56e06b3ea7f6dbaecb Mon Sep 17 00:00:00 2001 From: Mark Olesen Date: Mon, 2 May 2022 17:36:22 +0200 Subject: [PATCH] ENH: combine loadOrCreateMesh from redistributePar into fvMeshTools. - similar functionality as newMesh etc. Relocated to finiteVolume since there are no dynamicMesh dependencies. - use simpler procAddressing (with updated mapDistributeBase). separated from redistributePar --- .../redistributePar/loadOrCreateMesh.C | 391 +----- .../redistributePar/loadOrCreateMesh.H | 27 +- .../redistributePar/redistributePar.C | 624 +-------- src/dynamicMesh/Make/files | 2 - .../fvMeshDistribute/fvMeshDistribute.C | 3 +- .../fvMeshDistribute/fvMeshDistribute.H | 4 +- src/dynamicMesh/fvMeshTools/fvMeshTools.C | 815 ------------ src/finiteVolume/Make/files | 2 + .../fvMesh/fvMeshTools/fvMeshTools.C | 1124 +++++++++++++++++ .../fvMesh}/fvMeshTools/fvMeshTools.H | 67 +- .../fvMesh/fvMeshTools/fvMeshToolsProcAddr.C | 452 +++++++ .../fvMeshTools/fvMeshToolsTemplates.C | 52 +- 12 files changed, 1754 insertions(+), 1809 deletions(-) delete mode 100644 src/dynamicMesh/fvMeshTools/fvMeshTools.C create mode 100644 src/finiteVolume/fvMesh/fvMeshTools/fvMeshTools.C rename src/{dynamicMesh => finiteVolume/fvMesh}/fvMeshTools/fvMeshTools.H (73%) create mode 100644 src/finiteVolume/fvMesh/fvMeshTools/fvMeshToolsProcAddr.C rename src/{dynamicMesh => finiteVolume/fvMesh}/fvMeshTools/fvMeshToolsTemplates.C (80%) diff --git a/applications/utilities/parallelProcessing/redistributePar/loadOrCreateMesh.C b/applications/utilities/parallelProcessing/redistributePar/loadOrCreateMesh.C index 27827c98ed..7d789e5625 100644 --- a/applications/utilities/parallelProcessing/redistributePar/loadOrCreateMesh.C +++ b/applications/utilities/parallelProcessing/redistributePar/loadOrCreateMesh.C @@ -27,377 +27,62 @@ License \*---------------------------------------------------------------------------*/ #include "loadOrCreateMesh.H" -#include "processorPolyPatch.H" -#include "processorCyclicPolyPatch.H" -#include "Time.H" -#include "polyBoundaryMeshEntries.H" -#include "IOobjectList.H" -#include "pointSet.H" -#include "faceSet.H" -#include "cellSet.H" -#include "basicFvGeometryScheme.H" +#include "Pstream.H" +#include "OSspecific.H" -// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * // +// * * * * * * * * * * * * * * * Global Functions * * * * * * * * * * * * * // -// Read mesh if available. Otherwise create empty mesh with same non-proc -// patches as proc0 mesh. Requires: -// - all processors to have all patches (and in same order). -// - io.instance() set to facesInstance -Foam::autoPtr Foam::loadOrCreateMesh +Foam::boolList Foam::haveMeshFile ( - const bool decompose, - const IOobject& io + const Time& runTime, + const fileName& meshPath, + const word& meshFile, + const bool verbose ) { - // Region name - // ~~~~~~~~~~~ - - fileName meshSubDir; - - if (io.name() == polyMesh::defaultRegion) - { - meshSubDir = polyMesh::meshSubDir; - } - else - { - meshSubDir = io.name()/polyMesh::meshSubDir; - } - - - // Patch types - // ~~~~~~~~~~~ - // Read and scatter master patches (without reading master mesh!) - - PtrList patchEntries; - if (Pstream::master()) - { - const bool oldParRun = Pstream::parRun(false); - - patchEntries = polyBoundaryMeshEntries + boolList haveFileOnProc + ( + UPstream::listGatherValues ( - IOobject + fileHandler().isFile ( - "boundary", - io.instance(), //facesInstance, - meshSubDir, - io.db(), - IOobject::MUST_READ, - IOobject::NO_WRITE, - false - ) - ); - - Pstream::parRun(oldParRun); - } - - // Broadcast: send patches to all - Pstream::scatter(patchEntries); // == worldComm; - - - // Dummy meshes - // ~~~~~~~~~~~~ - - // Check who has a mesh - bool haveMesh; - if (decompose) - { - // Mesh needs to be present on the master only - haveMesh = Pstream::master(); - } - else - { - const fileName facesFile - ( - io.time().path() - /io.instance() //facesInstance - /meshSubDir - /"faces" - ); - - // Check presence of the searched-for faces file - haveMesh = fileHandler().isFile(fileHandler().filePath(facesFile)); - } - - if (!haveMesh) - { - const bool oldParRun = Pstream::parRun(false); - - - // Create dummy mesh. Only used on procs that don't have mesh. - IOobject noReadIO(io); - noReadIO.readOpt(IOobject::NO_READ); - noReadIO.writeOpt(IOobject::AUTO_WRITE); - fvMesh dummyMesh(noReadIO, Zero, false); - - // Add patches - List patches(patchEntries.size()); - label nPatches = 0; - - forAll(patchEntries, patchi) - { - const entry& e = patchEntries[patchi]; - const word type(e.dict().get("type")); - const word& name = e.keyword(); - - if - ( - type != processorPolyPatch::typeName - && type != processorCyclicPolyPatch::typeName - ) - { - dictionary patchDict(e.dict()); - patchDict.set("nFaces", 0); - patchDict.set("startFace", 0); - - patches[patchi] = polyPatch::New + fileHandler().filePath ( - name, - patchDict, - nPatches++, - dummyMesh.boundaryMesh() - ).ptr(); - } - } - patches.setSize(nPatches); - dummyMesh.addFvPatches(patches, false); // no parallel comms - - - // Add some dummy zones so upon reading it does not read them - // from the undecomposed case. Should be done as extra argument to - // regIOobject::readStream? - List pz - ( - 1, - new pointZone - ( - "dummyPointZone", - 0, - dummyMesh.pointZones() + runTime.path()/meshPath/meshFile + ) ) - ); - List fz - ( - 1, - new faceZone - ( - "dummyFaceZone", - 0, - dummyMesh.faceZones() - ) - ); - List cz - ( - 1, - new cellZone - ( - "dummyCellZone", - 0, - dummyMesh.cellZones() - ) - ); - dummyMesh.addZones(pz, fz, cz); - dummyMesh.pointZones().clear(); - dummyMesh.faceZones().clear(); - dummyMesh.cellZones().clear(); - //Pout<< "Writing dummy mesh to " << dummyMesh.polyMesh::objectPath() - // << endl; - dummyMesh.write(); + ) + ); - Pstream::parRun(oldParRun); // Restore parallel state - } - - - - // Read mesh - // ~~~~~~~~~ - // Now all processors have a (possibly zero size) mesh so read in - // parallel - - //Pout<< "Reading mesh from " << io.objectPath() << endl; - auto meshPtr = autoPtr::New(io); - fvMesh& mesh = *meshPtr; - - // Make sure to use a non-parallel geometry calculation method + if (verbose) { - tmp basicGeometry - ( - fvGeometryScheme::New - ( - mesh, - dictionary(), - basicFvGeometryScheme::typeName - ) - ); - mesh.geometry(basicGeometry); + Info<< "Per processor availability of \"" + << meshFile << "\" file in " << meshPath << nl + << " " << flatOutput(haveFileOnProc) << nl << endl; } + Pstream::broadcast(haveFileOnProc); + return haveFileOnProc; +} - // Sync patches - // ~~~~~~~~~~~~ - if (!Pstream::master() && haveMesh) +void Foam::removeProcAddressing(const polyMesh& mesh) +{ + IOobject ioAddr + ( + "procAddressing", + mesh.facesInstance(), + polyMesh::meshSubDir, + mesh.thisDb() + ); + + for (const auto prefix : {"boundary", "cell", "face", "point"}) { - // Check master names against mine + ioAddr.rename(prefix + word("ProcAddressing")); - const polyBoundaryMesh& patches = mesh.boundaryMesh(); - - forAll(patchEntries, patchi) - { - const entry& e = patchEntries[patchi]; - const word type(e.dict().get("type")); - const word& name = e.keyword(); - - if - ( - type == processorPolyPatch::typeName - || type == processorCyclicPolyPatch::typeName - ) - { - break; - } - - if (patchi >= patches.size()) - { - FatalErrorInFunction - << "Non-processor patches not synchronised." - << endl - << "Processor " << Pstream::myProcNo() - << " has only " << patches.size() - << " patches, master has " - << patchi - << exit(FatalError); - } - - if - ( - type != patches[patchi].type() - || name != patches[patchi].name() - ) - { - FatalErrorInFunction - << "Non-processor patches not synchronised." - << endl - << "Master patch " << patchi - << " name:" << type - << " type:" << type << endl - << "Processor " << Pstream::myProcNo() - << " patch " << patchi - << " has name:" << patches[patchi].name() - << " type:" << patches[patchi].type() - << exit(FatalError); - } - } + const fileName procFile(ioAddr.objectPath()); + Foam::rm(procFile); } - - - // Determine zones - // ~~~~~~~~~~~~~~~ - - wordList pointZoneNames(mesh.pointZones().names()); - Pstream::scatter(pointZoneNames); - wordList faceZoneNames(mesh.faceZones().names()); - Pstream::scatter(faceZoneNames); - wordList cellZoneNames(mesh.cellZones().names()); - Pstream::scatter(cellZoneNames); - - if (!haveMesh) - { - // Add the zones. Make sure to remove the old dummy ones first - mesh.pointZones().clear(); - mesh.faceZones().clear(); - mesh.cellZones().clear(); - - List pz(pointZoneNames.size()); - forAll(pointZoneNames, i) - { - pz[i] = new pointZone - ( - pointZoneNames[i], - i, - mesh.pointZones() - ); - } - List fz(faceZoneNames.size()); - forAll(faceZoneNames, i) - { - fz[i] = new faceZone - ( - faceZoneNames[i], - i, - mesh.faceZones() - ); - } - List cz(cellZoneNames.size()); - forAll(cellZoneNames, i) - { - cz[i] = new cellZone - ( - cellZoneNames[i], - i, - mesh.cellZones() - ); - } - mesh.addZones(pz, fz, cz); - } - - - // Determine sets - // ~~~~~~~~~~~~~~ - - wordList pointSetNames; - wordList faceSetNames; - wordList cellSetNames; - if (Pstream::master()) - { - // Read sets - const bool oldParRun = Pstream::parRun(false); - IOobjectList objects(mesh, mesh.facesInstance(), "polyMesh/sets"); - Pstream::parRun(oldParRun); - - pointSetNames = objects.sortedNames(pointSet::typeName); - faceSetNames = objects.sortedNames(faceSet::typeName); - cellSetNames = objects.sortedNames(cellSet::typeName); - } - Pstream::scatter(pointSetNames); - Pstream::scatter(faceSetNames); - Pstream::scatter(cellSetNames); - - if (!haveMesh) - { - for (const word& setName : pointSetNames) - { - pointSet(mesh, setName, 0).write(); - } - for (const word& setName : faceSetNames) - { - faceSet(mesh, setName, 0).write(); - } - for (const word& setName : cellSetNames) - { - cellSet(mesh, setName, 0).write(); - } - } - - - // Force recreation of globalMeshData. - mesh.globalData(); - - - // Do some checks. - - // Check if the boundary definition is unique - mesh.boundaryMesh().checkDefinition(true); - // Check if the boundary processor patches are correct - mesh.boundaryMesh().checkParallelSync(true); - // Check names of zones are equal - mesh.cellZones().checkDefinition(true); - mesh.cellZones().checkParallelSync(true); - mesh.faceZones().checkDefinition(true); - mesh.faceZones().checkParallelSync(true); - mesh.pointZones().checkDefinition(true); - mesh.pointZones().checkParallelSync(true); - - return meshPtr; } diff --git a/applications/utilities/parallelProcessing/redistributePar/loadOrCreateMesh.H b/applications/utilities/parallelProcessing/redistributePar/loadOrCreateMesh.H index 9125fc19f5..bd468348e3 100644 --- a/applications/utilities/parallelProcessing/redistributePar/loadOrCreateMesh.H +++ b/applications/utilities/parallelProcessing/redistributePar/loadOrCreateMesh.H @@ -6,6 +6,7 @@ \\/ M anipulation | ------------------------------------------------------------------------------- Copyright (C) 2012 OpenFOAM Foundation + Copyright (C) 2022 OpenCFD Ltd. ------------------------------------------------------------------------------- License This file is part of OpenFOAM. @@ -27,29 +28,35 @@ InNamespace Foam Description - Load or create (0 size) a mesh. Used in distributing meshes to a - larger number of processors + Miscellaneous file handling for meshes. SourceFiles loadOrCreateMesh.C \*---------------------------------------------------------------------------*/ -#ifndef loadOrCreateMesh_H -#define loadOrCreateMesh_H +#ifndef Foam_loadOrCreateMesh_H +#define Foam_loadOrCreateMesh_H -#include "fvMesh.H" +#include "fvMeshTools.H" // * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * // namespace Foam { -//- Load (if it exists) or create zero cell mesh given an IOobject: -// name : regionName -// instance : exact directory where to find mesh (i.e. does not -// do a findInstance -autoPtr loadOrCreateMesh(const bool decompose, const IOobject& io); +//- Check for availability of specified mesh file (default: "faces") +boolList haveMeshFile +( + const Time& runTime, + const fileName& meshPath, + const word& meshFile = "faces", + const bool verbose = true +); + + +//- Remove procAddressing +void removeProcAddressing(const polyMesh& mesh); //- Remove empty directory. Return true if removed. bool removeEmptyDir(const fileName& path); diff --git a/applications/utilities/parallelProcessing/redistributePar/redistributePar.C b/applications/utilities/parallelProcessing/redistributePar/redistributePar.C index 62328a3eeb..316d6749ae 100644 --- a/applications/utilities/parallelProcessing/redistributePar/redistributePar.C +++ b/applications/utilities/parallelProcessing/redistributePar/redistributePar.C @@ -92,7 +92,6 @@ Usage #include "zeroGradientFvPatchFields.H" #include "topoSet.H" #include "regionProperties.H" -#include "basicFvGeometryScheme.H" #include "parFvFieldReconstructor.H" #include "parLagrangianRedistributor.H" @@ -257,49 +256,6 @@ void copyUniform } -boolList haveFacesFile(const fileName& meshPath) -{ - const fileName facesPath(meshPath/"faces"); - Info<< "Checking for mesh in " << facesPath << nl << endl; - boolList haveMesh - ( - UPstream::listGatherValues - ( - fileHandler().isFile(fileHandler().filePath(facesPath)) - ) - ); - Info<< "Per processor mesh availability:" << nl - << " " << flatOutput(haveMesh) << nl << endl; - - Pstream::broadcast(haveMesh); - return haveMesh; -} - - -void setBasicGeometry(fvMesh& mesh) -{ - // Set the fvGeometryScheme to basic since it does not require - // any parallel communication to construct the geometry. During - // redistributePar one might temporarily end up with processors - // with zero procBoundaries. Normally procBoundaries trigger geometry - // calculation (e.g. send over cellCentres) so on the processors without - // procBoundaries this will not happen. The call to the geometry calculation - // is not synchronised and this might lead to a hang for geometry schemes - // that do require synchronisation - - tmp basicGeometry - ( - fvGeometryScheme::New - ( - mesh, - dictionary(), - basicFvGeometryScheme::typeName - ) - ); - mesh.geometry(basicGeometry); -} - - void printMeshData(const polyMesh& mesh) { // Collect all data on master @@ -557,213 +513,6 @@ void determineDecomposition } -// Write addressing if decomposing (1 to many) or reconstructing (many to 1) -void writeProcAddressing -( - autoPtr&& writeHandler, - const fvMesh& mesh, - const mapDistributePolyMesh& map, - const bool decompose -) -{ - Info<< "Writing procAddressing files to " << mesh.facesInstance() - << endl; - - labelIOList cellMap - ( - IOobject - ( - "cellProcAddressing", - mesh.facesInstance(), - polyMesh::meshSubDir, - mesh, - IOobject::NO_READ - ), - 0 - ); - - labelIOList faceMap - ( - IOobject - ( - "faceProcAddressing", - mesh.facesInstance(), - polyMesh::meshSubDir, - mesh, - IOobject::NO_READ - ), - 0 - ); - - labelIOList pointMap - ( - IOobject - ( - "pointProcAddressing", - mesh.facesInstance(), - polyMesh::meshSubDir, - mesh, - IOobject::NO_READ - ), - 0 - ); - - labelIOList patchMap - ( - IOobject - ( - "boundaryProcAddressing", - mesh.facesInstance(), - polyMesh::meshSubDir, - mesh, - IOobject::NO_READ - ), - 0 - ); - - // Decomposing: see how cells moved from undecomposed case - if (decompose) - { - cellMap = identity(map.nOldCells()); - map.distributeCellData(cellMap); - - faceMap = identity(map.nOldFaces()); - { - const mapDistribute& faceDistMap = map.faceMap(); - - if (faceDistMap.subHasFlip() || faceDistMap.constructHasFlip()) - { - // Offset by 1 - faceMap = faceMap + 1; - } - // Apply face flips - mapDistributeBase::distribute - ( - Pstream::commsTypes::nonBlocking, - List(), - faceDistMap.constructSize(), - faceDistMap.subMap(), - faceDistMap.subHasFlip(), - faceDistMap.constructMap(), - faceDistMap.constructHasFlip(), - faceMap, - flipLabelOp() - ); - } - - pointMap = identity(map.nOldPoints()); - map.distributePointData(pointMap); - - patchMap = identity(map.oldPatchSizes().size()); - const mapDistribute& patchDistMap = map.patchMap(); - // Use explicit distribute since we need to provide a null value - // (for new patches) and this is the only call that allow us to - // provide one ... - mapDistributeBase::distribute - ( - Pstream::commsTypes::nonBlocking, - List(), - patchDistMap.constructSize(), - patchDistMap.subMap(), - patchDistMap.subHasFlip(), - patchDistMap.constructMap(), - patchDistMap.constructHasFlip(), - patchMap, - label(-1), - eqOp