diff --git a/applications/utilities/parallelProcessing/redistributePar/parLagrangianDistributor.C b/applications/utilities/parallelProcessing/redistributePar/parLagrangianDistributor.C index b269a43a3c..626cb9c642 100644 --- a/applications/utilities/parallelProcessing/redistributePar/parLagrangianDistributor.C +++ b/applications/utilities/parallelProcessing/redistributePar/parLagrangianDistributor.C @@ -52,7 +52,7 @@ Foam::parLagrangianDistributor::parLagrangianDistributor const mapDistribute& cellMap = distMap_.cellMap(); // Get destination processors and cells - destinationProcID_ = labelList(tgtMesh_.nCells(), Pstream::myProcNo()); + destinationProcID_ = labelList(tgtMesh_.nCells(), UPstream::myProcNo()); cellMap.reverseDistribute(nSrcCells, destinationProcID_); destinationCell_ = identity(tgtMesh_.nCells()); @@ -67,21 +67,39 @@ Foam::parLagrangianDistributor::parLagrangianDistributor void Foam::parLagrangianDistributor::findClouds ( const fvMesh& mesh, - wordList& cloudNames, - List& objectNames + wordList& cloudNames, // All cloud names on any processor + boolList& haveClouds, // Per cloud name, whether my processor has it + List& objectNames // Per cloud name, the field name ) { - fileNameList localCloudDirs + const IOobject io ( - readDir + cloud::prefix, + mesh.time().timeName(), + mesh, + IOobjectOption::MUST_READ, + IOobjectOption::NO_WRITE + ); + + // Using the fileHandler: + // - use fileHandler to synthesise correct processor directory + // - cannot use readObjects since assumes all processors have same + // files (i.e. it only checks processor0) + const fileNameList localCloudDirs + ( + fileHandler().readDir ( - mesh.time().timePath() - / mesh.dbDir() - / cloud::prefix, + fileHandler().objectPath + ( + io, + word::null // typeName: not used currently + ), fileName::DIRECTORY ) ); + + // Copy out the local cloud names (and fileName -> word) cloudNames.resize_nocopy(localCloudDirs.size()); forAll(localCloudDirs, i) { @@ -92,18 +110,34 @@ void Foam::parLagrangianDistributor::findClouds Pstream::combineReduce(cloudNames, ListOps::uniqueEqOp()); Foam::sort(cloudNames); // Consistent order - objectNames.clear(); + + // See which of the global cloudNames I have + haveClouds.resize_nocopy(cloudNames.size()); + haveClouds = false; + + for (const fileName& localCloudName : localCloudDirs) + { + const label cloudi = cloudNames.find(localCloudName); + if (cloudi >= 0) + { + haveClouds[cloudi] = true; + } + } + + // Collect fields per cloud objectNames.resize(cloudNames.size()); for (const fileName& localCloudName : localCloudDirs) { // Do local scan for valid cloud objects + const bool oldParRun = UPstream::parRun(false); IOobjectList localObjs ( mesh, mesh.time().timeName(), cloud::prefix/localCloudName ); + UPstream::parRun(oldParRun); bool isCloud = false; if (localObjs.erase("coordinates")) @@ -141,8 +175,7 @@ Foam::parLagrangianDistributor::distributeLagrangianPositions ) const { //Debug(lpi.size()); - - const label oldLpi = lpi.size(); + //const label oldLpi = lpi.size(); labelListList sendMap; @@ -154,7 +187,7 @@ Foam::parLagrangianDistributor::distributeLagrangianPositions // neighbour processors List> particleTransferLists ( - Pstream::nProcs() + UPstream::nProcs() ); // Per particle the destination processor @@ -233,13 +266,15 @@ Foam::parLagrangianDistributor::distributeLagrangianPositions } } - if (lagrangianPositions.size()) + const bool writeOnProc = lagrangianPositions.size(); + + //if (writeOnProc) { // Write coordinates file IOPosition ( lagrangianPositions - ).write(); + ).write(writeOnProc); // Optionally write positions file in v1706 format and earlier if (particle::writeLagrangianPositions) @@ -248,35 +283,33 @@ Foam::parLagrangianDistributor::distributeLagrangianPositions ( lagrangianPositions, cloud::geometryType::POSITIONS - ).write(); + ).write(writeOnProc); } } - else if (oldLpi) - { - // When running with -overwrite it should also delete the old - // files. Below works but is not optimal. - - // Remove any existing coordinates - const fileName oldCoords - ( - IOPosition - ( - lagrangianPositions - ).objectPath() - ); - Foam::rm(oldCoords); - - // Remove any existing positions - const fileName oldPos - ( - IOPosition - ( - lagrangianPositions, - cloud::geometryType::POSITIONS - ).objectPath() - ); - Foam::rm(oldPos); - } + //else if (!writeOnProc && oldLpi) + //{ + // // When running with -overwrite it should also delete the old + // // files. Below works but is not optimal. + // + // // Remove any existing coordinates + // Foam::rm + // ( + // IOPosition + // ( + // lagrangianPositions + // ).objectPath() + // ); + // + // // Remove any existing positions + // Foam::rm + // ( + // IOPosition + // ( + // lagrangianPositions, + // cloud::geometryType::POSITIONS + // ).objectPath() + // ); + //} // Restore cloud name lpi.rename(cloudName); diff --git a/applications/utilities/parallelProcessing/redistributePar/parLagrangianDistributor.H b/applications/utilities/parallelProcessing/redistributePar/parLagrangianDistributor.H index 6ce206e795..f79054a9f8 100644 --- a/applications/utilities/parallelProcessing/redistributePar/parLagrangianDistributor.H +++ b/applications/utilities/parallelProcessing/redistributePar/parLagrangianDistributor.H @@ -6,7 +6,7 @@ \\/ M anipulation | ------------------------------------------------------------------------------- Copyright (C) 2015 OpenFOAM Foundation - Copyright (C) 2018-2022 OpenCFD Ltd. + Copyright (C) 2018-2023 OpenCFD Ltd. ------------------------------------------------------------------------------- License This file is part of OpenFOAM. @@ -114,7 +114,14 @@ public: static void findClouds ( const fvMesh&, + + //! All cloud names on any processor wordList& cloudNames, + + //! Per cloud name, whether my processor has it + boolList& haveClouds, + + //! Per cloud nmae, the field names List& objectNames ); @@ -131,6 +138,7 @@ public: static label readFields ( const passivePositionParticleCloud& cloud, + const bool haveCloud, const IOobjectList& objects, const wordRes& selectedFields = wordRes() ); @@ -139,18 +147,11 @@ public: static label readAllFields ( const passivePositionParticleCloud& cloud, + const bool haveCloud, const IOobjectList& objects, const wordRes& selectedFields = wordRes() ); - //- Read and store all fields for known cloud field types - // Uses the current cloud instance to obtain the IOobjectList - static label readAllFields - ( - const passivePositionParticleCloud& cloud, - const wordRes& selectedFields = wordRes() - ); - // Member Functions @@ -171,6 +172,7 @@ public: ( const mapDistributeBase& lagrangianMap, const word& cloudName, + const bool haveCloud, const IOobjectList& cloudObjs, const wordRes& selectedFields ) const; @@ -189,6 +191,7 @@ public: ( const mapDistributeBase& map, const word& cloudName, + const bool haveCloud, const IOobjectList& objects, const wordRes& selectedFields = wordRes() ) const; @@ -199,11 +202,14 @@ public: ( const mapDistributeBase& map, const word& cloudName, + const bool haveCloud, const IOobjectList& objects, const wordRes& selectedFields = wordRes() ) const; - //- Redistribute and write stored lagrangian fields + //- Redistribute and write stored lagrangian fields. + // Note: does no reading so no need to check for existence + // of lagrangian files template label distributeStoredFields ( diff --git a/applications/utilities/parallelProcessing/redistributePar/parLagrangianDistributorFields.C b/applications/utilities/parallelProcessing/redistributePar/parLagrangianDistributorFields.C index 730c3fcc1b..9c16c360b5 100644 --- a/applications/utilities/parallelProcessing/redistributePar/parLagrangianDistributorFields.C +++ b/applications/utilities/parallelProcessing/redistributePar/parLagrangianDistributorFields.C @@ -5,7 +5,7 @@ \\ / A nd | www.openfoam.com \\/ M anipulation | ------------------------------------------------------------------------------- - Copyright (C) 2022 OpenCFD Ltd. + Copyright (C) 2022-2023 OpenCFD Ltd. ------------------------------------------------------------------------------- License This file is part of OpenFOAM. @@ -33,6 +33,7 @@ License Foam::label Foam::parLagrangianDistributor::readAllFields ( const passivePositionParticleCloud& cloud, + const bool haveCloud, const IOobjectList& cloudObjs, const wordRes& selectedFields ) @@ -48,6 +49,7 @@ Foam::label Foam::parLagrangianDistributor::readAllFields > \ ( \ cloud, \ + haveCloud, \ cloudObjs, \ selectedFields \ ); \ @@ -56,6 +58,7 @@ Foam::label Foam::parLagrangianDistributor::readAllFields >> \ ( \ cloud, \ + haveCloud, \ cloudObjs, \ selectedFields \ ); \ @@ -64,6 +67,7 @@ Foam::label Foam::parLagrangianDistributor::readAllFields , Type>> \ ( \ cloud, \ + haveCloud, \ cloudObjs, \ selectedFields \ ); \ @@ -84,21 +88,22 @@ Foam::label Foam::parLagrangianDistributor::readAllFields } -Foam::label Foam::parLagrangianDistributor::readAllFields -( - const passivePositionParticleCloud& cloud, - const wordRes& selectedFields -) -{ - IOobjectList cloudObjs(cloud, cloud.time().timeName()); - return readAllFields(cloud, cloudObjs, selectedFields); -} +//Foam::label Foam::parLagrangianDistributor::readAllFields +//( +// const passivePositionParticleCloud& cloud, +// const wordRes& selectedFields +//) +//{ +// IOobjectList cloudObjs(cloud, cloud.time().timeName()); +// return readAllFields(cloud, cloudObjs, selectedFields); +//} Foam::label Foam::parLagrangianDistributor::distributeAllFields ( const mapDistributeBase& lagrangianMap, const word& cloudName, + const bool haveCloud, const IOobjectList& cloudObjs, const wordRes& selectedFields ) const @@ -114,6 +119,7 @@ Foam::label Foam::parLagrangianDistributor::distributeAllFields ( \ lagrangianMap, \ cloudName, \ + haveCloud, \ cloudObjs, \ selectedFields \ ); \ @@ -122,6 +128,7 @@ Foam::label Foam::parLagrangianDistributor::distributeAllFields ( \ lagrangianMap, \ cloudName, \ + haveCloud, \ cloudObjs, \ selectedFields \ ); \ diff --git a/applications/utilities/parallelProcessing/redistributePar/parLagrangianDistributorTemplates.C b/applications/utilities/parallelProcessing/redistributePar/parLagrangianDistributorTemplates.C index 3ea8ec4a50..e44e949c44 100644 --- a/applications/utilities/parallelProcessing/redistributePar/parLagrangianDistributorTemplates.C +++ b/applications/utilities/parallelProcessing/redistributePar/parLagrangianDistributorTemplates.C @@ -64,91 +64,85 @@ Foam::label Foam::parLagrangianDistributor::distributeFields ( const mapDistributeBase& map, const word& cloudName, + const bool haveCloud, const IOobjectList& objects, const wordRes& selectedFields ) const { - typedef IOField fieldType; + typedef IOField Container; const wordList fieldNames ( - filterObjects + filterObjects> ( objects, selectedFields ) ); + // Read if present + IOobject srcIOobject + ( + "none", + srcMesh_.time().timeName(), + cloud::prefix/cloudName, + srcMesh_, + IOobjectOption::LAZY_READ, + IOobjectOption::NO_WRITE, + IOobjectOption::NO_REGISTER + ); - bool reconstruct = false; + IOobject tgtIOobject + ( + "none", + tgtMesh_.time().timeName(), + cloud::prefix/cloudName, + tgtMesh_, + IOobjectOption::NO_READ, + IOobjectOption::NO_WRITE, + IOobjectOption::NO_REGISTER + ); - label nFields = 0; - for (const word& objectName : fieldNames) + //bool reconstruct = false; + + if (verbose_ && fieldNames.size()) { - if (!nFields) - { - // Performing an all-to-one (reconstruct)? - reconstruct = - returnReduceAnd(!map.constructSize() || Pstream::master()); - } - - if (verbose_) - { - if (!nFields) - { - Info<< " Distributing lagrangian " - << fieldType::typeName << "s\n" << nl; - } - Info<< " " << objectName << nl; - } - ++nFields; - - // Read if present - IOField field - ( - IOobject - ( - objectName, - srcMesh_.time().timeName(), - cloud::prefix/cloudName, - srcMesh_, - IOobject::READ_IF_PRESENT, - IOobject::NO_WRITE, - IOobject::NO_REGISTER - ), - label(0) - ); - - map.distribute(field); - - - const IOobject fieldIO - ( - objectName, - tgtMesh_.time().timeName(), - cloud::prefix/cloudName, - tgtMesh_, - IOobject::NO_READ, - IOobject::NO_WRITE, - IOobject::NO_REGISTER - ); - - if (field.size()) - { - IOField(fieldIO, std::move(field)).write(); - } - else if (!reconstruct) - { - // When running with -overwrite it should also delete the old - // files. Below works but is not optimal. - - const fileName fldName(fieldIO.objectPath()); - Foam::rm(fldName); - } + Info<< " Distributing lagrangian " + << Container::typeName << "s\n" << nl; } - if (nFields && verbose_) Info<< endl; - return nFields; + for (const word& objectName : fieldNames) + { + if (verbose_) + { + Info<< " " << objectName << nl; + } + + srcIOobject.resetHeader(objectName); + tgtIOobject.resetHeader(objectName); + + // Read if present (ie, haveCloud means readOnProc) + Container field(srcIOobject, haveCloud); + + // Distribute + map.distribute(field); + + const bool writeOnProc = field.size(); + + // Write + Container(tgtIOobject, std::move(field)).write(writeOnProc); + + //if (!writeOnProc && !reconstruct) + //{ + // // When running with -overwrite it should also delete the old + // // files. Below works but is not optimal. + // + // Foam::rm(tgtIOobject.objectPath()); + //} + } + + if (verbose_ && fieldNames.size()) Info<< endl; + return fieldNames.size(); } @@ -157,25 +151,27 @@ Foam::label Foam::parLagrangianDistributor::distributeFieldFields ( const mapDistributeBase& map, const word& cloudName, + const bool haveCloud, const IOobjectList& objects, const wordRes& selectedFields ) const { - typedef CompactIOField, Type> fieldType; + typedef CompactIOField, Type> Container; DynamicList fieldNames; - fieldNames.append + // CompactIOField Field names + fieldNames.push_back ( - filterObjects + filterObjects, Type>> ( objects, selectedFields ) ); - // Append IOField Field names - fieldNames.append + // IOField Field names + fieldNames.push_back ( filterObjects>> ( @@ -184,80 +180,69 @@ Foam::label Foam::parLagrangianDistributor::distributeFieldFields ) ); - bool reconstruct = false; + // Read if present + IOobject srcIOobject + ( + "none", + srcMesh_.time().timeName(), + cloud::prefix/cloudName, + srcMesh_, + IOobjectOption::LAZY_READ, + IOobjectOption::NO_WRITE, + IOobjectOption::NO_REGISTER + ); + + IOobject tgtIOobject + ( + "none", + tgtMesh_.time().timeName(), + cloud::prefix/cloudName, + tgtMesh_, + IOobjectOption::NO_READ, + IOobjectOption::NO_WRITE, + IOobjectOption::NO_REGISTER + ); + + //bool reconstruct = false; + + if (verbose_ && fieldNames.size()) + { + Info<< " Distributing lagrangian " + << Container::typeName << "s\n" << nl; + } - label nFields = 0; for (const word& objectName : fieldNames) { - if (!nFields) - { - // Performing an all-to-one (reconstruct)? - reconstruct = - returnReduceAnd(!map.constructSize() || Pstream::master()); - } - if (verbose_) { - if (!nFields) - { - Info<< " Distributing lagrangian " - << fieldType::typeName << "s\n" << nl; - } Info<< " " << objectName << nl; } - ++nFields; - // Read if present - CompactIOField, Type> field - ( - IOobject - ( - objectName, - srcMesh_.time().timeName(), - cloud::prefix/cloudName, - srcMesh_, - IOobject::READ_IF_PRESENT, - IOobject::NO_WRITE, - IOobject::NO_REGISTER - ), - label(0) - ); + srcIOobject.resetHeader(objectName); + tgtIOobject.resetHeader(objectName); + + // Read if present (ie, haveCloud means readOnProc) + Container field(srcIOobject, haveCloud); // Distribute map.distribute(field); + const bool writeOnProc = field.size(); + // Write - const IOobject fieldIO - ( - objectName, - tgtMesh_.time().timeName(), - cloud::prefix/cloudName, - tgtMesh_, - IOobject::NO_READ, - IOobject::NO_WRITE, - IOobject::NO_REGISTER - ); + Container(tgtIOobject, std::move(field)).write(writeOnProc); - if (field.size()) - { - CompactIOField, Type> - ( - fieldIO, - std::move(field) - ).write(); - } - else if (!reconstruct) - { - // When running with -overwrite it should also delete the old - // files. Below works but is not optimal. - - const fileName fldName(fieldIO.objectPath()); - Foam::rm(fldName); - } + //if (!writeOnProc && !reconstruct) + //{ + // // When running with -overwrite it should also delete the old + // // files. Below works but is not optimal. + // + // Foam::rm(tgtIOobject.objectPath()); + //} } - if (nFields && verbose_) Info<< endl; - return nFields; + if (verbose_ && fieldNames.size()) Info<< endl; + return fieldNames.size(); } @@ -265,12 +250,11 @@ template Foam::label Foam::parLagrangianDistributor::readFields ( const passivePositionParticleCloud& cloud, + const bool haveCloud, const IOobjectList& objects, const wordRes& selectedFields ) { - const word fieldClassName(Container::typeName); - const wordList fieldNames ( filterObjects @@ -280,40 +264,40 @@ Foam::label Foam::parLagrangianDistributor::readFields ) ); - label nFields = 0; + // Read if present + IOobject readIO + ( + "none", + cloud.time().timeName(), + cloud, + IOobjectOption::LAZY_READ, + IOobjectOption::NO_WRITE, + IOobjectOption::REGISTER + ); + + if (verbose_ && fieldNames.size()) + { + Info<< " Reading lagrangian " + << Container::typeName << "s\n" << nl; + } + for (const word& objectName : fieldNames) { if (verbose_) { - if (!nFields) - { - Info<< " Reading lagrangian " - << Container::typeName << "s\n" << nl; - } Info<< " " << objectName << nl; } - ++nFields; - // Read if present - Container* fieldPtr = new Container - ( - IOobject - ( - objectName, - cloud.time().timeName(), - cloud, - IOobject::LAZY_READ, - IOobject::NO_WRITE, - IOobject::REGISTER - ), - label(0) - ); + readIO.resetHeader(objectName); + + // Read if present (ie, haveCloud means readOnProc) + Container* fieldPtr = new Container(readIO, haveCloud); fieldPtr->store(); } - if (nFields && verbose_) Info<< endl; - return nFields; + if (verbose_ && fieldNames.size()) Info<< endl; + return fieldNames.size(); } @@ -324,58 +308,55 @@ Foam::label Foam::parLagrangianDistributor::distributeStoredFields passivePositionParticleCloud& cloud ) const { - bool reconstruct = false; - label nFields = 0; + // Parallel-consistent + UPtrList fields(cloud.sorted()); - for (Container& field : cloud.sorted()) + IOobject writeIO + ( + "none", + tgtMesh_.time().timeName(), + cloud::prefix/cloud.name(), + tgtMesh_, + IOobjectOption::NO_READ, + IOobjectOption::NO_WRITE, + IOobjectOption::NO_REGISTER + ); + + //bool reconstruct = false; + + if (verbose_ && fields.size()) { - if (!nFields) - { - // Performing an all-to-one (reconstruct)? - reconstruct = - returnReduceAnd(!map.constructSize() || Pstream::master()); - } + Info<< " Distributing lagrangian " + << Container::typeName << "s\n" << nl; + } + for (Container& field : fields) + { if (verbose_) { - if (!nFields) - { - Info<< " Distributing lagrangian " - << Container::typeName << "s\n" << nl; - } Info<< " " << field.name() << nl; } - ++nFields; map.distribute(field); - const IOobject fieldIO - ( - field.name(), - tgtMesh_.time().timeName(), - cloud::prefix/cloud.name(), - tgtMesh_, - IOobject::NO_READ, - IOobject::NO_WRITE, - IOobject::NO_REGISTER - ); + writeIO.resetHeader(field.name()); - if (field.size()) - { - Container(fieldIO, std::move(field)).write(); - } - else if (!reconstruct) - { - // When running with -overwrite it should also delete the old - // files. Below works but is not optimal. + const bool writeOnProc = field.size(); - const fileName fldName(fieldIO.objectPath()); - Foam::rm(fldName); - } + // Write + Container(writeIO, std::move(field)).write(writeOnProc); + + //if (!writeOnProc && !reconstruct) + //{ + // // When running with -overwrite it should also delete the old + // // files. Below works but is not optimal. + // + // Foam::rm(writeIO.objectPath()); + //} } - if (nFields && verbose_) Info<< endl; - return nFields; + if (verbose_ && fields.size()) Info<< endl; + return fields.size(); } diff --git a/applications/utilities/parallelProcessing/redistributePar/redistributeLagrangian.H b/applications/utilities/parallelProcessing/redistributePar/redistributeLagrangian.H index 99f08b3c29..bb118558f5 100644 --- a/applications/utilities/parallelProcessing/redistributePar/redistributeLagrangian.H +++ b/applications/utilities/parallelProcessing/redistributePar/redistributeLagrangian.H @@ -6,7 +6,7 @@ \\/ M anipulation | ------------------------------------------------------------------------------- Copyright (C) 2011-2017 OpenFOAM Foundation - Copyright (C) 2015-2022 OpenCFD Ltd. + Copyright (C) 2015-2023 OpenCFD Ltd. ------------------------------------------------------------------------------- License This file is part of OpenFOAM. @@ -48,6 +48,7 @@ readLagrangian ( const fvMesh& mesh, const wordList& cloudNames, + const boolUList& haveClouds, const wordRes& selectedFields ) { @@ -78,11 +79,10 @@ readLagrangian IOobjectList cloudObjs(clouds[i], clouds[i].time().timeName()); - //Pout<< "Found cloud objects:" << cloudObjs.names() << endl; - parLagrangianDistributor::readAllFields ( clouds[i], + haveClouds[i], cloudObjs, selectedFields ); @@ -101,11 +101,19 @@ readLagrangian ) { wordList cloudNames; + boolList haveClouds; List fieldNames; - // Find all cloudNames on all processors - parLagrangianDistributor::findClouds(mesh, cloudNames, fieldNames); - return readLagrangian(mesh, cloudNames, selectedFields); + // Find all cloudNames on all processors + parLagrangianDistributor::findClouds + ( + mesh, + cloudNames, + haveClouds, + fieldNames + ); + + return readLagrangian(mesh, cloudNames, haveClouds, selectedFields); } @@ -121,9 +129,17 @@ void reconstructLagrangian // Clouds (note: might not be present on all processors) wordList cloudNames; + boolList haveClouds; List fieldNames; + // Find all cloudNames on all processors - parLagrangianDistributor::findClouds(mesh, cloudNames, fieldNames); + parLagrangianDistributor::findClouds + ( + mesh, + cloudNames, + haveClouds, + fieldNames + ); if (cloudNames.empty()) { @@ -142,13 +158,16 @@ void reconstructLagrangian baseMesh, mesh.nCells(), // range of cell indices in clouds distMap + //writeHandler // Which processors to write ) ); } const auto& distributor = *distributorPtr; - for (const word& cloudName : cloudNames) + forAll(cloudNames, cloudi) { + const word& cloudName = cloudNames[cloudi]; + Info<< "Reconstructing lagrangian fields for cloud " << cloudName << nl << endl; @@ -169,6 +188,7 @@ void reconstructLagrangian ( lagrangianMapPtr(), cloudName, + haveClouds[cloudi], cloudObjs, selectedFields );