zhyang-dev/openfoam
1
0
Fork 0
mirror of https://develop.openfoam.com/Development/openfoam.git synced 2025-11-28 03:28:01 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

ENH: Update redistribute clouds with readOnProc/writeOnProc

Browse Source 
- when reading, detect all clouds on all processors and uses this when
  reading fields. Similarly, when writing it uses writeOnProc to skip
  clouds that are empty on any particular processor.

Co-authored-by: Mark Olesen <>
This commit is contained in:
mattijs
2023-05-12 18:43:32 +02:00
committed by Andrew Heather
parent eea72282ab
commit bcd873ccfe
5 changed files with 309 additions and 262 deletions
Download Patch File Download Diff File Expand all files Collapse all files

115
applications/utilities/parallelProcessing/redistributePar/parLagrangianDistributor.C
View File

@ -52,7 +52,7 @@ Foam::parLagrangianDistributor::parLagrangianDistributor
const mapDistribute& cellMap = distMap_.cellMap();
// Get destination processors and cells
destinationProcID_ = labelList(tgtMesh_.nCells(), Pstream::myProcNo());
destinationProcID_ = labelList(tgtMesh_.nCells(), UPstream::myProcNo());
cellMap.reverseDistribute(nSrcCells, destinationProcID_);
destinationCell_ = identity(tgtMesh_.nCells());
@ -67,21 +67,39 @@ Foam::parLagrangianDistributor::parLagrangianDistributor
void Foam::parLagrangianDistributor::findClouds
(
const fvMesh& mesh,
wordList& cloudNames,
List<wordList>& objectNames
wordList& cloudNames, // All cloud names on any processor
boolList& haveClouds, // Per cloud name, whether my processor has it
List<wordList>& objectNames // Per cloud name, the field name
)
{
fileNameList localCloudDirs
const IOobject io
(
readDir
cloud::prefix,
mesh.time().timeName(),
mesh,
IOobjectOption::MUST_READ,
IOobjectOption::NO_WRITE
);
// Using the fileHandler:
// - use fileHandler to synthesise correct processor directory
// - cannot use readObjects since assumes all processors have same
// files (i.e. it only checks processor0)
const fileNameList localCloudDirs
(
fileHandler().readDir
(
mesh.time().timePath()
/ mesh.dbDir()
/ cloud::prefix,
fileHandler().objectPath
(
io,
word::null // typeName: not used currently
),
fileName::DIRECTORY
)
);
// Copy out the local cloud names (and fileName -> word)
cloudNames.resize_nocopy(localCloudDirs.size());
forAll(localCloudDirs, i)
{
@ -92,18 +110,34 @@ void Foam::parLagrangianDistributor::findClouds
Pstream::combineReduce(cloudNames, ListOps::uniqueEqOp<word>());
Foam::sort(cloudNames); // Consistent order
objectNames.clear();
// See which of the global cloudNames I have
haveClouds.resize_nocopy(cloudNames.size());
haveClouds = false;
for (const fileName& localCloudName : localCloudDirs)
{
const label cloudi = cloudNames.find(localCloudName);
if (cloudi >= 0)
{
haveClouds[cloudi] = true;
}
}
// Collect fields per cloud
objectNames.resize(cloudNames.size());
for (const fileName& localCloudName : localCloudDirs)
{
// Do local scan for valid cloud objects
const bool oldParRun = UPstream::parRun(false);
IOobjectList localObjs
(
mesh,
mesh.time().timeName(),
cloud::prefix/localCloudName
);
UPstream::parRun(oldParRun);
bool isCloud = false;
if (localObjs.erase("coordinates"))
@ -141,8 +175,7 @@ Foam::parLagrangianDistributor::distributeLagrangianPositions
) const
{
//Debug(lpi.size());
const label oldLpi = lpi.size();
//const label oldLpi = lpi.size();
labelListList sendMap;
@ -154,7 +187,7 @@ Foam::parLagrangianDistributor::distributeLagrangianPositions
// neighbour processors
List<IDLList<passivePositionParticle>> particleTransferLists
(
Pstream::nProcs()
UPstream::nProcs()
);
// Per particle the destination processor
@ -233,13 +266,15 @@ Foam::parLagrangianDistributor::distributeLagrangianPositions
}
}
if (lagrangianPositions.size())
const bool writeOnProc = lagrangianPositions.size();
//if (writeOnProc)
{
// Write coordinates file
IOPosition<passivePositionParticleCloud>
(
lagrangianPositions
).write();
).write(writeOnProc);
// Optionally write positions file in v1706 format and earlier
if (particle::writeLagrangianPositions)
@ -248,35 +283,33 @@ Foam::parLagrangianDistributor::distributeLagrangianPositions
(
lagrangianPositions,
cloud::geometryType::POSITIONS
).write();
).write(writeOnProc);
}
}
else if (oldLpi)
{
// When running with -overwrite it should also delete the old
// files. Below works but is not optimal.
// Remove any existing coordinates
const fileName oldCoords
(
IOPosition<passivePositionParticleCloud>
(
lagrangianPositions
).objectPath()
);
Foam::rm(oldCoords);
// Remove any existing positions
const fileName oldPos
(
IOPosition<passivePositionParticleCloud>
(
lagrangianPositions,
cloud::geometryType::POSITIONS
).objectPath()
);
Foam::rm(oldPos);
}
//else if (!writeOnProc && oldLpi)
//{
// // When running with -overwrite it should also delete the old
// // files. Below works but is not optimal.
//
// // Remove any existing coordinates
// Foam::rm
// (
// IOPosition<passivePositionParticleCloud>
// (
// lagrangianPositions
// ).objectPath()
// );
//
// // Remove any existing positions
// Foam::rm
// (
// IOPosition<passivePositionParticleCloud>
// (
// lagrangianPositions,
// cloud::geometryType::POSITIONS
// ).objectPath()
// );
//}
// Restore cloud name
lpi.rename(cloudName);

26
applications/utilities/parallelProcessing/redistributePar/parLagrangianDistributor.H
View File

@ -6,7 +6,7 @@
\\/ M anipulation |
-------------------------------------------------------------------------------
Copyright (C) 2015 OpenFOAM Foundation
Copyright (C) 2018-2022 OpenCFD Ltd.
Copyright (C) 2018-2023 OpenCFD Ltd.
-------------------------------------------------------------------------------
License
This file is part of OpenFOAM.
@ -114,7 +114,14 @@ public:
static void findClouds
(
const fvMesh&,
//! All cloud names on any processor
wordList& cloudNames,
//! Per cloud name, whether my processor has it
boolList& haveClouds,
//! Per cloud nmae, the field names
List<wordList>& objectNames
);
@ -131,6 +138,7 @@ public:
static label readFields
(
const passivePositionParticleCloud& cloud,
const bool haveCloud,
const IOobjectList& objects,
const wordRes& selectedFields = wordRes()
);
@ -139,18 +147,11 @@ public:
static label readAllFields
(
const passivePositionParticleCloud& cloud,
const bool haveCloud,
const IOobjectList& objects,
const wordRes& selectedFields = wordRes()
);
//- Read and store all fields for known cloud field types
// Uses the current cloud instance to obtain the IOobjectList
static label readAllFields
(
const passivePositionParticleCloud& cloud,
const wordRes& selectedFields = wordRes()
);
// Member Functions
@ -171,6 +172,7 @@ public:
(
const mapDistributeBase& lagrangianMap,
const word& cloudName,
const bool haveCloud,
const IOobjectList& cloudObjs,
const wordRes& selectedFields
) const;
@ -189,6 +191,7 @@ public:
(
const mapDistributeBase& map,
const word& cloudName,
const bool haveCloud,
const IOobjectList& objects,
const wordRes& selectedFields = wordRes()
) const;
@ -199,11 +202,14 @@ public:
(
const mapDistributeBase& map,
const word& cloudName,
const bool haveCloud,
const IOobjectList& objects,
const wordRes& selectedFields = wordRes()
) const;
//- Redistribute and write stored lagrangian fields
//- Redistribute and write stored lagrangian fields.
// Note: does no reading so no need to check for existence
// of lagrangian files
template<class Container>
label distributeStoredFields
(

27
applications/utilities/parallelProcessing/redistributePar/parLagrangianDistributorFields.C
View File

@ -5,7 +5,7 @@
\\ / A nd | www.openfoam.com
\\/ M anipulation |
-------------------------------------------------------------------------------
Copyright (C) 2022 OpenCFD Ltd.
Copyright (C) 2022-2023 OpenCFD Ltd.
-------------------------------------------------------------------------------
License
This file is part of OpenFOAM.
@ -33,6 +33,7 @@ License
Foam::label Foam::parLagrangianDistributor::readAllFields
(
const passivePositionParticleCloud& cloud,
const bool haveCloud,
const IOobjectList& cloudObjs,
const wordRes& selectedFields
)
@ -48,6 +49,7 @@ Foam::label Foam::parLagrangianDistributor::readAllFields
<IOField<Type>> \
( \
cloud, \
haveCloud, \
cloudObjs, \
selectedFields \
); \
@ -56,6 +58,7 @@ Foam::label Foam::parLagrangianDistributor::readAllFields
<IOField<Field<Type>>> \
( \
cloud, \
haveCloud, \
cloudObjs, \
selectedFields \
); \
@ -64,6 +67,7 @@ Foam::label Foam::parLagrangianDistributor::readAllFields
<CompactIOField<Field<Type>, Type>> \
( \
cloud, \
haveCloud, \
cloudObjs, \
selectedFields \
); \
@ -84,21 +88,22 @@ Foam::label Foam::parLagrangianDistributor::readAllFields
}
Foam::label Foam::parLagrangianDistributor::readAllFields
(
const passivePositionParticleCloud& cloud,
const wordRes& selectedFields
)
{
IOobjectList cloudObjs(cloud, cloud.time().timeName());
return readAllFields(cloud, cloudObjs, selectedFields);
}
//Foam::label Foam::parLagrangianDistributor::readAllFields
//(
// const passivePositionParticleCloud& cloud,
// const wordRes& selectedFields
//)
//{
// IOobjectList cloudObjs(cloud, cloud.time().timeName());
// return readAllFields(cloud, cloudObjs, selectedFields);
//}
Foam::label Foam::parLagrangianDistributor::distributeAllFields
(
const mapDistributeBase& lagrangianMap,
const word& cloudName,
const bool haveCloud,
const IOobjectList& cloudObjs,
const wordRes& selectedFields
) const
@ -114,6 +119,7 @@ Foam::label Foam::parLagrangianDistributor::distributeAllFields
( \
lagrangianMap, \
cloudName, \
haveCloud, \
cloudObjs, \
selectedFields \
); \
@ -122,6 +128,7 @@ Foam::label Foam::parLagrangianDistributor::distributeAllFields
( \
lagrangianMap, \
cloudName, \
haveCloud, \
cloudObjs, \
selectedFields \
); \

367
applications/utilities/parallelProcessing/redistributePar/parLagrangianDistributorTemplates.C
View File

@ -64,91 +64,85 @@ Foam::label Foam::parLagrangianDistributor::distributeFields
(
const mapDistributeBase& map,
const word& cloudName,
const bool haveCloud,
const IOobjectList& objects,
const wordRes& selectedFields
) const
{
typedef IOField<Type> fieldType;
typedef IOField<Type> Container;
const wordList fieldNames
(
filterObjects<fieldType>
filterObjects<IOField<Type>>
(
objects,
selectedFields
)
);
// Read if present
IOobject srcIOobject
(
"none",
srcMesh_.time().timeName(),
cloud::prefix/cloudName,
srcMesh_,
IOobjectOption::LAZY_READ,
IOobjectOption::NO_WRITE,
IOobjectOption::NO_REGISTER
);
bool reconstruct = false;
IOobject tgtIOobject
(
"none",
tgtMesh_.time().timeName(),
cloud::prefix/cloudName,
tgtMesh_,
IOobjectOption::NO_READ,
IOobjectOption::NO_WRITE,
IOobjectOption::NO_REGISTER
);
label nFields = 0;
for (const word& objectName : fieldNames)
//bool reconstruct = false;
if (verbose_ && fieldNames.size())
{
if (!nFields)
{
// Performing an all-to-one (reconstruct)?
reconstruct =
returnReduceAnd(!map.constructSize() || Pstream::master());
}
if (verbose_)
{
if (!nFields)
{
Info<< " Distributing lagrangian "
<< fieldType::typeName << "s\n" << nl;
}
Info<< " " << objectName << nl;
}
++nFields;
// Read if present
IOField<Type> field
(
IOobject
(
objectName,
srcMesh_.time().timeName(),
cloud::prefix/cloudName,
srcMesh_,
IOobject::READ_IF_PRESENT,
IOobject::NO_WRITE,
IOobject::NO_REGISTER
),
label(0)
);
map.distribute(field);
const IOobject fieldIO
(
objectName,
tgtMesh_.time().timeName(),
cloud::prefix/cloudName,
tgtMesh_,
IOobject::NO_READ,
IOobject::NO_WRITE,
IOobject::NO_REGISTER
);
if (field.size())
{
IOField<Type>(fieldIO, std::move(field)).write();
}
else if (!reconstruct)
{
// When running with -overwrite it should also delete the old
// files. Below works but is not optimal.
const fileName fldName(fieldIO.objectPath());
Foam::rm(fldName);
}
Info<< " Distributing lagrangian "
<< Container::typeName << "s\n" << nl;
}
if (nFields && verbose_) Info<< endl;
return nFields;
for (const word& objectName : fieldNames)
{
if (verbose_)
{
Info<< " " << objectName << nl;
}
srcIOobject.resetHeader(objectName);
tgtIOobject.resetHeader(objectName);
// Read if present (ie, haveCloud means readOnProc)
Container field(srcIOobject, haveCloud);
// Distribute
map.distribute(field);
const bool writeOnProc = field.size();
// Write
Container(tgtIOobject, std::move(field)).write(writeOnProc);
//if (!writeOnProc && !reconstruct)
//{
// // When running with -overwrite it should also delete the old
// // files. Below works but is not optimal.
//
// Foam::rm(tgtIOobject.objectPath());
//}
}
if (verbose_ && fieldNames.size()) Info<< endl;
return fieldNames.size();
}
@ -157,25 +151,27 @@ Foam::label Foam::parLagrangianDistributor::distributeFieldFields
(
const mapDistributeBase& map,
const word& cloudName,
const bool haveCloud,
const IOobjectList& objects,
const wordRes& selectedFields
) const
{
typedef CompactIOField<Field<Type>, Type> fieldType;
typedef CompactIOField<Field<Type>, Type> Container;
DynamicList<word> fieldNames;
fieldNames.append
// CompactIOField Field names
fieldNames.push_back
(
filterObjects<fieldType>
filterObjects<CompactIOField<Field<Type>, Type>>
(
objects,
selectedFields
)
);
// Append IOField Field names
fieldNames.append
// IOField Field names
fieldNames.push_back
(
filterObjects<IOField<Field<Type>>>
(
@ -184,80 +180,69 @@ Foam::label Foam::parLagrangianDistributor::distributeFieldFields
)
);
bool reconstruct = false;
// Read if present
IOobject srcIOobject
(
"none",
srcMesh_.time().timeName(),
cloud::prefix/cloudName,
srcMesh_,
IOobjectOption::LAZY_READ,
IOobjectOption::NO_WRITE,
IOobjectOption::NO_REGISTER
);
IOobject tgtIOobject
(
"none",
tgtMesh_.time().timeName(),
cloud::prefix/cloudName,
tgtMesh_,
IOobjectOption::NO_READ,
IOobjectOption::NO_WRITE,
IOobjectOption::NO_REGISTER
);
//bool reconstruct = false;
if (verbose_ && fieldNames.size())
{
Info<< " Distributing lagrangian "
<< Container::typeName << "s\n" << nl;
}
label nFields = 0;
for (const word& objectName : fieldNames)
{
if (!nFields)
{
// Performing an all-to-one (reconstruct)?
reconstruct =
returnReduceAnd(!map.constructSize() || Pstream::master());
}
if (verbose_)
{
if (!nFields)
{
Info<< " Distributing lagrangian "
<< fieldType::typeName << "s\n" << nl;
}
Info<< " " << objectName << nl;
}
++nFields;
// Read if present
CompactIOField<Field<Type>, Type> field
(
IOobject
(
objectName,
srcMesh_.time().timeName(),
cloud::prefix/cloudName,
srcMesh_,
IOobject::READ_IF_PRESENT,
IOobject::NO_WRITE,
IOobject::NO_REGISTER
),
label(0)
);
srcIOobject.resetHeader(objectName);
tgtIOobject.resetHeader(objectName);
// Read if present (ie, haveCloud means readOnProc)
Container field(srcIOobject, haveCloud);
// Distribute
map.distribute(field);
const bool writeOnProc = field.size();
// Write
const IOobject fieldIO
(
objectName,
tgtMesh_.time().timeName(),
cloud::prefix/cloudName,
tgtMesh_,
IOobject::NO_READ,
IOobject::NO_WRITE,
IOobject::NO_REGISTER
);
Container(tgtIOobject, std::move(field)).write(writeOnProc);
if (field.size())
{
CompactIOField<Field<Type>, Type>
(
fieldIO,
std::move(field)
).write();
}
else if (!reconstruct)
{
// When running with -overwrite it should also delete the old
// files. Below works but is not optimal.
const fileName fldName(fieldIO.objectPath());
Foam::rm(fldName);
}
//if (!writeOnProc && !reconstruct)
//{
// // When running with -overwrite it should also delete the old
// // files. Below works but is not optimal.
//
// Foam::rm(tgtIOobject.objectPath());
//}
}
if (nFields && verbose_) Info<< endl;
return nFields;
if (verbose_ && fieldNames.size()) Info<< endl;
return fieldNames.size();
}
@ -265,12 +250,11 @@ template<class Container>
Foam::label Foam::parLagrangianDistributor::readFields
(
const passivePositionParticleCloud& cloud,
const bool haveCloud,
const IOobjectList& objects,
const wordRes& selectedFields
)
{
const word fieldClassName(Container::typeName);
const wordList fieldNames
(
filterObjects<Container>
@ -280,40 +264,40 @@ Foam::label Foam::parLagrangianDistributor::readFields
)
);
label nFields = 0;
// Read if present
IOobject readIO
(
"none",
cloud.time().timeName(),
cloud,
IOobjectOption::LAZY_READ,
IOobjectOption::NO_WRITE,
IOobjectOption::REGISTER
);
if (verbose_ && fieldNames.size())
{
Info<< " Reading lagrangian "
<< Container::typeName << "s\n" << nl;
}
for (const word& objectName : fieldNames)
{
if (verbose_)
{
if (!nFields)
{
Info<< " Reading lagrangian "
<< Container::typeName << "s\n" << nl;
}
Info<< " " << objectName << nl;
}
++nFields;
// Read if present
Container* fieldPtr = new Container
(
IOobject
(
objectName,
cloud.time().timeName(),
cloud,
IOobject::LAZY_READ,
IOobject::NO_WRITE,
IOobject::REGISTER
),
label(0)
);
readIO.resetHeader(objectName);
// Read if present (ie, haveCloud means readOnProc)
Container* fieldPtr = new Container(readIO, haveCloud);
fieldPtr->store();
}
if (nFields && verbose_) Info<< endl;
return nFields;
if (verbose_ && fieldNames.size()) Info<< endl;
return fieldNames.size();
}
@ -324,58 +308,55 @@ Foam::label Foam::parLagrangianDistributor::distributeStoredFields
passivePositionParticleCloud& cloud
) const
{
bool reconstruct = false;
label nFields = 0;
// Parallel-consistent
UPtrList<Container> fields(cloud.sorted<Container>());
for (Container& field : cloud.sorted<Container>())
IOobject writeIO
(
"none",
tgtMesh_.time().timeName(),
cloud::prefix/cloud.name(),
tgtMesh_,
IOobjectOption::NO_READ,
IOobjectOption::NO_WRITE,
IOobjectOption::NO_REGISTER
);
//bool reconstruct = false;
if (verbose_ && fields.size())
{
if (!nFields)
{
// Performing an all-to-one (reconstruct)?
reconstruct =
returnReduceAnd(!map.constructSize() || Pstream::master());
}
Info<< " Distributing lagrangian "
<< Container::typeName << "s\n" << nl;
}
for (Container& field : fields)
{
if (verbose_)
{
if (!nFields)
{
Info<< " Distributing lagrangian "
<< Container::typeName << "s\n" << nl;
}
Info<< " " << field.name() << nl;
}
++nFields;
map.distribute(field);
const IOobject fieldIO
(
field.name(),
tgtMesh_.time().timeName(),
cloud::prefix/cloud.name(),
tgtMesh_,
IOobject::NO_READ,
IOobject::NO_WRITE,
IOobject::NO_REGISTER
);
writeIO.resetHeader(field.name());
if (field.size())
{
Container(fieldIO, std::move(field)).write();
}
else if (!reconstruct)
{
// When running with -overwrite it should also delete the old
// files. Below works but is not optimal.
const bool writeOnProc = field.size();
const fileName fldName(fieldIO.objectPath());
Foam::rm(fldName);
}
// Write
Container(writeIO, std::move(field)).write(writeOnProc);
//if (!writeOnProc && !reconstruct)
//{
// // When running with -overwrite it should also delete the old
// // files. Below works but is not optimal.
//
// Foam::rm(writeIO.objectPath());
//}
}
if (nFields && verbose_) Info<< endl;
return nFields;
if (verbose_ && fields.size()) Info<< endl;
return fields.size();
}

36
applications/utilities/parallelProcessing/redistributePar/redistributeLagrangian.H
View File

@ -6,7 +6,7 @@
\\/ M anipulation |
-------------------------------------------------------------------------------
Copyright (C) 2011-2017 OpenFOAM Foundation
Copyright (C) 2015-2022 OpenCFD Ltd.
Copyright (C) 2015-2023 OpenCFD Ltd.
-------------------------------------------------------------------------------
License
This file is part of OpenFOAM.
@ -48,6 +48,7 @@ readLagrangian
(
const fvMesh& mesh,
const wordList& cloudNames,
const boolUList& haveClouds,
const wordRes& selectedFields
)
{
@ -78,11 +79,10 @@ readLagrangian
IOobjectList cloudObjs(clouds[i], clouds[i].time().timeName());
//Pout<< "Found cloud objects:" << cloudObjs.names() << endl;
parLagrangianDistributor::readAllFields
(
clouds[i],
haveClouds[i],
cloudObjs,
selectedFields
);
@ -101,11 +101,19 @@ readLagrangian
)
{
wordList cloudNames;
boolList haveClouds;
List<wordList> fieldNames;
// Find all cloudNames on all processors
parLagrangianDistributor::findClouds(mesh, cloudNames, fieldNames);
return readLagrangian(mesh, cloudNames, selectedFields);
// Find all cloudNames on all processors
parLagrangianDistributor::findClouds
(
mesh,
cloudNames,
haveClouds,
fieldNames
);
return readLagrangian(mesh, cloudNames, haveClouds, selectedFields);
}
@ -121,9 +129,17 @@ void reconstructLagrangian
// Clouds (note: might not be present on all processors)
wordList cloudNames;
boolList haveClouds;
List<wordList> fieldNames;
// Find all cloudNames on all processors
parLagrangianDistributor::findClouds(mesh, cloudNames, fieldNames);
parLagrangianDistributor::findClouds
(
mesh,
cloudNames,
haveClouds,
fieldNames
);
if (cloudNames.empty())
{
@ -142,13 +158,16 @@ void reconstructLagrangian
baseMesh,
mesh.nCells(), // range of cell indices in clouds
distMap
//writeHandler // Which processors to write
)
);
}
const auto& distributor = *distributorPtr;
for (const word& cloudName : cloudNames)
forAll(cloudNames, cloudi)
{
const word& cloudName = cloudNames[cloudi];
Info<< "Reconstructing lagrangian fields for cloud "
<< cloudName << nl << endl;
@ -169,6 +188,7 @@ void reconstructLagrangian
(
lagrangianMapPtr(),
cloudName,
haveClouds[cloudi],
cloudObjs,
selectedFields
);