zhyang-dev/openfoam
1
0
Fork 0
mirror of https://develop.openfoam.com/Development/openfoam.git synced 2025-11-28 03:28:01 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Merge branch 'update-redistributePar' into 'develop'

Browse Source 
improve fileHandler support for redistributePar

See merge request Development/openfoam!605
This commit is contained in:
Andrew Heather
2023-11-21 11:14:45 +00:00
parent eea72282ab 5be652d03c
commit 0ae6141397
45 changed files with 2462 additions and 967 deletions
Download Patch File Download Diff File Expand all files Collapse all files

243
applications/utilities/parallelProcessing/redistributePar/loadOrCreateMesh.C
View File

@ -6,7 +6,7 @@
\\/ M anipulation |
-------------------------------------------------------------------------------
Copyright (C) 2012-2017 OpenFOAM Foundation
Copyright (C) 2015-2022 OpenCFD Ltd.
Copyright (C) 2015-2023 OpenCFD Ltd.
-------------------------------------------------------------------------------
License
This file is part of OpenFOAM.
@ -30,9 +30,104 @@ License
#include "faMesh.H"
#include "Pstream.H"
#include "OSspecific.H"
#include "decomposedBlockData.H"
// * * * * * * * * * * * * * * * Global Functions * * * * * * * * * * * * * //
bool Foam::checkFileExistence(const fileName& fName)
{
// Trimmed-down version of lookupAndCacheProcessorsPath
// with Foam::exists() check. No caching.
// Check for two conditions:
// - file has to exist
// - if collated the entry has to exist inside the file
// Note: bypass fileOperation::filePath(IOobject&) since has problems
// with going to a different number of processors
// (in collated format). Use file-based searching instead
const auto& handler = Foam::fileHandler();
typedef fileOperation::procRangeType procRangeType;
fileName path, pDir, local;
procRangeType group;
label numProcs;
const label proci =
fileOperation::splitProcessorPath
(fName, path, pDir, local, group, numProcs);
bool found = false;
if (proci != -1)
{
// Read all directories to see any beginning with processor
// ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
const fileNameList dirEntries
(
handler.readDir(path, fileName::Type::DIRECTORY)
);
// Extract info from processorN or processorsNN
// - highest processor number
// - directory+offset containing data for proci
// label nProcs = 0;
for (const fileName& dirN : dirEntries)
{
// Analyse directory name
fileName rp, rd, rl;
label rNum;
const label readProci =
fileOperation::splitProcessorPath
(dirN, rp, rd, rl, group, rNum);
if (proci == readProci)
{
// Found "processorN"
if (Foam::exists(path/dirN/local))
{
found = true;
break;
}
}
else if (rNum != -1)
{
// "processorsNN" or "processorsNN_start-end"
if (group.empty())
{
// "processorsNN"
if (proci < rNum && Foam::exists(path/dirN/local))
{
found = true;
break;
}
}
else if (group.contains(proci))
{
// "processorsNN_start-end"
// - save the local proc offset
if (Foam::exists(path/dirN/local))
{
found = true;
break;
}
}
}
}
}
if (!found)
{
found = Foam::exists(fName);
}
return found;
}
Foam::boolList Foam::haveMeshFile
(
const Time& runTime,
@ -41,18 +136,83 @@ Foam::boolList Foam::haveMeshFile
const bool verbose
)
{
#if 0
// Simple directory scanning - too fragile
bool found = checkFileExistence(runTime.path()/meshPath/meshFile);
#else
// Trimmed-down version of lookupAndCacheProcessorsPath
// with Foam::exists() check. No caching.
// Check for two conditions:
// - file has to exist
// - if collated the entry has to exist inside the file
// Note: bypass fileOperation::filePath(IOobject&) since has problems
// with going to a different number of processors
// (in collated format). Use file-based searching instead
const auto& handler = Foam::fileHandler();
typedef fileOperation::procRangeType procRangeType;
const fileName fName
(
handler.filePath(runTime.path()/meshPath/meshFile)
);
bool found = handler.isFile(fName);
if (returnReduceAnd(found)) // worldComm
{
autoPtr<ISstream> isPtr(fileHandler().NewIFstream(fName));
if (isPtr && isPtr->good())
{
auto& is = *isPtr;
IOobject io(meshFile, meshPath, runTime);
io.readHeader(is);
if (decomposedBlockData::isCollatedType(io))
{
fileName path, pDir, local;
procRangeType group;
label numProcs;
label proci = fileOperation::splitProcessorPath
(
fName,
path,
pDir,
local,
group,
numProcs
);
label myBlockNumber = 0;
if (proci == -1 && group.empty())
{
// 'processorsXXX' format so contains all ranks
// according to worldComm
myBlockNumber = UPstream::myProcNo(UPstream::worldComm);
}
else
{
// 'processorsXXX_n-m' format so check for the
// relative rank
myBlockNumber = UPstream::myProcNo(fileHandler().comm());
}
// Check if block for the local rank is inside file
found = decomposedBlockData::hasBlock(is, myBlockNumber);
}
}
}
#endif
// Globally consistent information about who has a mesh
boolList haveFileOnProc
(
UPstream::listGatherValues<bool>
(
fileHandler().isFile
(
fileHandler().filePath
(
runTime.path()/meshPath/meshFile
)
)
)
UPstream::allGatherValues<bool>(found, UPstream::worldComm)
);
if (verbose)
@ -62,7 +222,6 @@ Foam::boolList Foam::haveMeshFile
<< " " << flatOutput(haveFileOnProc) << nl << endl;
}
Pstream::broadcast(haveFileOnProc);
return haveFileOnProc;
}
@ -107,38 +266,52 @@ void Foam::removeProcAddressing(const polyMesh& mesh)
}
void Foam::removeEmptyDir(const fileName& path)
void Foam::masterMeshInstance
(
const IOobject& io,
fileName& facesInstance,
fileName& pointsInstance
)
{
// Remove directory: silent, emptyOnly
Foam::rmDir(path, true, true);
}
const fileName meshSubDir
(
polyMesh::regionName(io.name()) / polyMesh::meshSubDir
);
void Foam::removeEmptyDirs(const fileName& meshPath)
{
// Delete resulting directory if empty
fileName path(meshPath);
path.clean();
// Do subdirectories
if (UPstream::master())
{
const fileNameList dirs
const bool oldParRun = UPstream::parRun(false);
const label oldNumProcs = fileHandler().nProcs();
const int oldCache = fileOperation::cacheLevel(0);
facesInstance = io.time().findInstance
(
Foam::readDir
(
path,
fileName::DIRECTORY,
false, // filterGz
false // followLink
)
meshSubDir,
"faces",
IOobjectOption::MUST_READ
);
for (const auto& dir : dirs)
pointsInstance = io.time().findInstance
(
meshSubDir,
"points",
IOobjectOption::MUST_READ
);
fileOperation::cacheLevel(oldCache);
if (oldParRun)
{
removeEmptyDirs(path/dir);
const_cast<fileOperation&>(fileHandler()).nProcs(oldNumProcs);
}
UPstream::parRun(oldParRun);
}
removeEmptyDir(path);
// Broadcast information to all
Pstream::broadcasts
(
UPstream::worldComm,
facesInstance,
pointsInstance
);
}

17
applications/utilities/parallelProcessing/redistributePar/loadOrCreateMesh.H
View File

@ -6,7 +6,7 @@
\\/ M anipulation |
-------------------------------------------------------------------------------
Copyright (C) 2012 OpenFOAM Foundation
Copyright (C) 2022 OpenCFD Ltd.
Copyright (C) 2022-2023 OpenCFD Ltd.
-------------------------------------------------------------------------------
License
This file is part of OpenFOAM.
@ -48,6 +48,9 @@ namespace Foam
// Forward Declarations
class faMesh;
//- Check for availability of given file
bool checkFileExistence(const fileName& fName);
//- Check for availability of specified mesh file (default: "faces")
boolList haveMeshFile
(
@ -64,11 +67,13 @@ void removeProcAddressing(const faMesh& mesh);
//- Remove procAddressing
void removeProcAddressing(const polyMesh& mesh);
//- Remove empty directory
void removeEmptyDir(const fileName& path);
//- Remove empty directories from bottom up
void removeEmptyDirs(const fileName& path);
//- Determine master faces instance
void masterMeshInstance
(
const IOobject& io,
fileName& facesInstance,
fileName& pointsInstance
);
}

29
applications/utilities/parallelProcessing/redistributePar/parFaFieldDistributorCache.C
View File

@ -64,6 +64,7 @@ void Foam::parFaFieldDistributorCache::read
const bool decompose, // i.e. read from undecomposed case
const boolList& areaMeshOnProc,
refPtr<fileOperation>& readHandler,
const fileName& areaMeshInstance,
faMesh& mesh
)
@ -76,10 +77,16 @@ void Foam::parFaFieldDistributorCache::read
// Missing an area mesh somewhere?
if (areaMeshOnProc.found(false))
{
const bool oldParRun = UPstream::parRun(false);
const int oldCache = fileOperation::cacheLevel(0);
// A zero-sized mesh with boundaries.
// This is used to create zero-sized fields.
subsetterPtr.reset(new faMeshSubset(mesh, zero{}));
fileOperation::cacheLevel(oldCache);
UPstream::parRun(oldParRun); // Restore parallel state
// Deregister from polyMesh ...
auto& obr = const_cast<objectRegistry&>
(
@ -92,14 +99,26 @@ void Foam::parFaFieldDistributorCache::read
obr.checkOut("faSolution");
}
// Get original objects (before incrementing time!)
if (Pstream::master() && decompose)
if (UPstream::master() && decompose)
{
runTime.caseName() = baseRunTime.caseName();
runTime.processorCase(false);
}
IOobjectList objects(mesh.mesh(), runTime.timeName());
IOobjectList objects; //(mesh.mesh(), runTime.timeName());
if (readHandler)
{
auto oldHandler = fileOperation::fileHandler(readHandler);
const auto oldComm = UPstream::commWorld(fileHandler().comm());
objects = IOobjectList(mesh.mesh(), runTime.timeName());
readHandler = fileOperation::fileHandler(oldHandler);
UPstream::commWorld(oldComm);
}
if (Pstream::master() && decompose)
{
runTime.caseName() = proc0CaseName;
@ -116,11 +135,13 @@ void Foam::parFaFieldDistributorCache::read
runTime.processorCase(false);
}
#undef doFieldReading
#define doFieldReading(Storage) \
fieldsDistributor::readFields \
( \
areaMeshOnProc, mesh, subsetterPtr, objects, Storage, \
areaMeshOnProc, readHandler, mesh, subsetterPtr, objects, \
Storage, \
true /* (deregister field) */ \
);

1
applications/utilities/parallelProcessing/redistributePar/parFaFieldDistributorCache.H
View File

@ -105,6 +105,7 @@ public:
const bool decompose, // i.e. read from undecomposed case
const boolList& areaMeshOnProc,
refPtr<fileOperation>& readHandler,
const fileName& areaMeshInstance,
faMesh& mesh
);

115
applications/utilities/parallelProcessing/redistributePar/parLagrangianDistributor.C
View File

@ -52,7 +52,7 @@ Foam::parLagrangianDistributor::parLagrangianDistributor
const mapDistribute& cellMap = distMap_.cellMap();
// Get destination processors and cells
destinationProcID_ = labelList(tgtMesh_.nCells(), Pstream::myProcNo());
destinationProcID_ = labelList(tgtMesh_.nCells(), UPstream::myProcNo());
cellMap.reverseDistribute(nSrcCells, destinationProcID_);
destinationCell_ = identity(tgtMesh_.nCells());
@ -67,21 +67,39 @@ Foam::parLagrangianDistributor::parLagrangianDistributor
void Foam::parLagrangianDistributor::findClouds
(
const fvMesh& mesh,
wordList& cloudNames,
List<wordList>& objectNames
wordList& cloudNames, // All cloud names on any processor
boolList& haveClouds, // Per cloud name, whether my processor has it
List<wordList>& objectNames // Per cloud name, the field name
)
{
fileNameList localCloudDirs
const IOobject io
(
readDir
cloud::prefix,
mesh.time().timeName(),
mesh,
IOobjectOption::MUST_READ,
IOobjectOption::NO_WRITE
);
// Using the fileHandler:
// - use fileHandler to synthesise correct processor directory
// - cannot use readObjects since assumes all processors have same
// files (i.e. it only checks processor0)
const fileNameList localCloudDirs
(
fileHandler().readDir
(
mesh.time().timePath()
/ mesh.dbDir()
/ cloud::prefix,
fileHandler().objectPath
(
io,
word::null // typeName: not used currently
),
fileName::DIRECTORY
)
);
// Copy out the local cloud names (and fileName -> word)
cloudNames.resize_nocopy(localCloudDirs.size());
forAll(localCloudDirs, i)
{
@ -92,18 +110,34 @@ void Foam::parLagrangianDistributor::findClouds
Pstream::combineReduce(cloudNames, ListOps::uniqueEqOp<word>());
Foam::sort(cloudNames); // Consistent order
objectNames.clear();
// See which of the global cloudNames I have
haveClouds.resize_nocopy(cloudNames.size());
haveClouds = false;
for (const fileName& localCloudName : localCloudDirs)
{
const label cloudi = cloudNames.find(localCloudName);
if (cloudi >= 0)
{
haveClouds[cloudi] = true;
}
}
// Collect fields per cloud
objectNames.resize(cloudNames.size());
for (const fileName& localCloudName : localCloudDirs)
{
// Do local scan for valid cloud objects
const bool oldParRun = UPstream::parRun(false);
IOobjectList localObjs
(
mesh,
mesh.time().timeName(),
cloud::prefix/localCloudName
);
UPstream::parRun(oldParRun);
bool isCloud = false;
if (localObjs.erase("coordinates"))
@ -141,8 +175,7 @@ Foam::parLagrangianDistributor::distributeLagrangianPositions
) const
{
//Debug(lpi.size());
const label oldLpi = lpi.size();
//const label oldLpi = lpi.size();
labelListList sendMap;
@ -154,7 +187,7 @@ Foam::parLagrangianDistributor::distributeLagrangianPositions
// neighbour processors
List<IDLList<passivePositionParticle>> particleTransferLists
(
Pstream::nProcs()
UPstream::nProcs()
);
// Per particle the destination processor
@ -233,13 +266,15 @@ Foam::parLagrangianDistributor::distributeLagrangianPositions
}
}
if (lagrangianPositions.size())
const bool writeOnProc = lagrangianPositions.size();
//if (writeOnProc)
{
// Write coordinates file
IOPosition<passivePositionParticleCloud>
(
lagrangianPositions
).write();
).write(writeOnProc);
// Optionally write positions file in v1706 format and earlier
if (particle::writeLagrangianPositions)
@ -248,35 +283,33 @@ Foam::parLagrangianDistributor::distributeLagrangianPositions
(
lagrangianPositions,
cloud::geometryType::POSITIONS
).write();
).write(writeOnProc);
}
}
else if (oldLpi)
{
// When running with -overwrite it should also delete the old
// files. Below works but is not optimal.
// Remove any existing coordinates
const fileName oldCoords
(
IOPosition<passivePositionParticleCloud>
(
lagrangianPositions
).objectPath()
);
Foam::rm(oldCoords);
// Remove any existing positions
const fileName oldPos
(
IOPosition<passivePositionParticleCloud>
(
lagrangianPositions,
cloud::geometryType::POSITIONS
).objectPath()
);
Foam::rm(oldPos);
}
//else if (!writeOnProc && oldLpi)
//{
// // When running with -overwrite it should also delete the old
// // files. Below works but is not optimal.
//
// // Remove any existing coordinates
// Foam::rm
// (
// IOPosition<passivePositionParticleCloud>
// (
// lagrangianPositions
// ).objectPath()
// );
//
// // Remove any existing positions
// Foam::rm
// (
// IOPosition<passivePositionParticleCloud>
// (
// lagrangianPositions,
// cloud::geometryType::POSITIONS
// ).objectPath()
// );
//}
// Restore cloud name
lpi.rename(cloudName);

26
applications/utilities/parallelProcessing/redistributePar/parLagrangianDistributor.H
View File

@ -6,7 +6,7 @@
\\/ M anipulation |
-------------------------------------------------------------------------------
Copyright (C) 2015 OpenFOAM Foundation
Copyright (C) 2018-2022 OpenCFD Ltd.
Copyright (C) 2018-2023 OpenCFD Ltd.
-------------------------------------------------------------------------------
License
This file is part of OpenFOAM.
@ -114,7 +114,14 @@ public:
static void findClouds
(
const fvMesh&,
//! All cloud names on any processor
wordList& cloudNames,
//! Per cloud name, whether my processor has it
boolList& haveClouds,
//! Per cloud nmae, the field names
List<wordList>& objectNames
);
@ -131,6 +138,7 @@ public:
static label readFields
(
const passivePositionParticleCloud& cloud,
const bool haveCloud,
const IOobjectList& objects,
const wordRes& selectedFields = wordRes()
);
@ -139,18 +147,11 @@ public:
static label readAllFields
(
const passivePositionParticleCloud& cloud,
const bool haveCloud,
const IOobjectList& objects,
const wordRes& selectedFields = wordRes()
);
//- Read and store all fields for known cloud field types
// Uses the current cloud instance to obtain the IOobjectList
static label readAllFields
(
const passivePositionParticleCloud& cloud,
const wordRes& selectedFields = wordRes()
);
// Member Functions
@ -171,6 +172,7 @@ public:
(
const mapDistributeBase& lagrangianMap,
const word& cloudName,
const bool haveCloud,
const IOobjectList& cloudObjs,
const wordRes& selectedFields
) const;
@ -189,6 +191,7 @@ public:
(
const mapDistributeBase& map,
const word& cloudName,
const bool haveCloud,
const IOobjectList& objects,
const wordRes& selectedFields = wordRes()
) const;
@ -199,11 +202,14 @@ public:
(
const mapDistributeBase& map,
const word& cloudName,
const bool haveCloud,
const IOobjectList& objects,
const wordRes& selectedFields = wordRes()
) const;
//- Redistribute and write stored lagrangian fields
//- Redistribute and write stored lagrangian fields.
// Note: does no reading so no need to check for existence
// of lagrangian files
template<class Container>
label distributeStoredFields
(

27
applications/utilities/parallelProcessing/redistributePar/parLagrangianDistributorFields.C
View File

@ -5,7 +5,7 @@
\\ / A nd | www.openfoam.com
\\/ M anipulation |
-------------------------------------------------------------------------------
Copyright (C) 2022 OpenCFD Ltd.
Copyright (C) 2022-2023 OpenCFD Ltd.
-------------------------------------------------------------------------------
License
This file is part of OpenFOAM.
@ -33,6 +33,7 @@ License
Foam::label Foam::parLagrangianDistributor::readAllFields
(
const passivePositionParticleCloud& cloud,
const bool haveCloud,
const IOobjectList& cloudObjs,
const wordRes& selectedFields
)
@ -48,6 +49,7 @@ Foam::label Foam::parLagrangianDistributor::readAllFields
<IOField<Type>> \
( \
cloud, \
haveCloud, \
cloudObjs, \
selectedFields \
); \
@ -56,6 +58,7 @@ Foam::label Foam::parLagrangianDistributor::readAllFields
<IOField<Field<Type>>> \
( \
cloud, \
haveCloud, \
cloudObjs, \
selectedFields \
); \
@ -64,6 +67,7 @@ Foam::label Foam::parLagrangianDistributor::readAllFields
<CompactIOField<Field<Type>, Type>> \
( \
cloud, \
haveCloud, \
cloudObjs, \
selectedFields \
); \
@ -84,21 +88,22 @@ Foam::label Foam::parLagrangianDistributor::readAllFields
}
Foam::label Foam::parLagrangianDistributor::readAllFields
(
const passivePositionParticleCloud& cloud,
const wordRes& selectedFields
)
{
IOobjectList cloudObjs(cloud, cloud.time().timeName());
return readAllFields(cloud, cloudObjs, selectedFields);
}
//Foam::label Foam::parLagrangianDistributor::readAllFields
//(
// const passivePositionParticleCloud& cloud,
// const wordRes& selectedFields
//)
//{
// IOobjectList cloudObjs(cloud, cloud.time().timeName());
// return readAllFields(cloud, cloudObjs, selectedFields);
//}
Foam::label Foam::parLagrangianDistributor::distributeAllFields
(
const mapDistributeBase& lagrangianMap,
const word& cloudName,
const bool haveCloud,
const IOobjectList& cloudObjs,
const wordRes& selectedFields
) const
@ -114,6 +119,7 @@ Foam::label Foam::parLagrangianDistributor::distributeAllFields
( \
lagrangianMap, \
cloudName, \
haveCloud, \
cloudObjs, \
selectedFields \
); \
@ -122,6 +128,7 @@ Foam::label Foam::parLagrangianDistributor::distributeAllFields
( \
lagrangianMap, \
cloudName, \
haveCloud, \
cloudObjs, \
selectedFields \
); \

367
applications/utilities/parallelProcessing/redistributePar/parLagrangianDistributorTemplates.C
View File

@ -64,91 +64,85 @@ Foam::label Foam::parLagrangianDistributor::distributeFields
(
const mapDistributeBase& map,
const word& cloudName,
const bool haveCloud,
const IOobjectList& objects,
const wordRes& selectedFields
) const
{
typedef IOField<Type> fieldType;
typedef IOField<Type> Container;
const wordList fieldNames
(
filterObjects<fieldType>
filterObjects<IOField<Type>>
(
objects,
selectedFields
)
);
// Read if present
IOobject srcIOobject
(
"none",
srcMesh_.time().timeName(),
cloud::prefix/cloudName,
srcMesh_,
IOobjectOption::LAZY_READ,
IOobjectOption::NO_WRITE,
IOobjectOption::NO_REGISTER
);
bool reconstruct = false;
IOobject tgtIOobject
(
"none",
tgtMesh_.time().timeName(),
cloud::prefix/cloudName,
tgtMesh_,
IOobjectOption::NO_READ,
IOobjectOption::NO_WRITE,
IOobjectOption::NO_REGISTER
);
label nFields = 0;
for (const word& objectName : fieldNames)
//bool reconstruct = false;
if (verbose_ && fieldNames.size())
{
if (!nFields)
{
// Performing an all-to-one (reconstruct)?
reconstruct =
returnReduceAnd(!map.constructSize() || Pstream::master());
}
if (verbose_)
{
if (!nFields)
{
Info<< " Distributing lagrangian "
<< fieldType::typeName << "s\n" << nl;
}
Info<< " " << objectName << nl;
}
++nFields;
// Read if present
IOField<Type> field
(
IOobject
(
objectName,
srcMesh_.time().timeName(),
cloud::prefix/cloudName,
srcMesh_,
IOobject::READ_IF_PRESENT,
IOobject::NO_WRITE,
IOobject::NO_REGISTER
),
label(0)
);
map.distribute(field);
const IOobject fieldIO
(
objectName,
tgtMesh_.time().timeName(),
cloud::prefix/cloudName,
tgtMesh_,
IOobject::NO_READ,
IOobject::NO_WRITE,
IOobject::NO_REGISTER
);
if (field.size())
{
IOField<Type>(fieldIO, std::move(field)).write();
}
else if (!reconstruct)
{
// When running with -overwrite it should also delete the old
// files. Below works but is not optimal.
const fileName fldName(fieldIO.objectPath());
Foam::rm(fldName);
}
Info<< " Distributing lagrangian "
<< Container::typeName << "s\n" << nl;
}
if (nFields && verbose_) Info<< endl;
return nFields;
for (const word& objectName : fieldNames)
{
if (verbose_)
{
Info<< " " << objectName << nl;
}
srcIOobject.resetHeader(objectName);
tgtIOobject.resetHeader(objectName);
// Read if present (ie, haveCloud means readOnProc)
Container field(srcIOobject, haveCloud);
// Distribute
map.distribute(field);
const bool writeOnProc = field.size();
// Write
Container(tgtIOobject, std::move(field)).write(writeOnProc);
//if (!writeOnProc && !reconstruct)
//{
// // When running with -overwrite it should also delete the old
// // files. Below works but is not optimal.
//
// Foam::rm(tgtIOobject.objectPath());
//}
}
if (verbose_ && fieldNames.size()) Info<< endl;
return fieldNames.size();
}
@ -157,25 +151,27 @@ Foam::label Foam::parLagrangianDistributor::distributeFieldFields
(
const mapDistributeBase& map,
const word& cloudName,
const bool haveCloud,
const IOobjectList& objects,
const wordRes& selectedFields
) const
{
typedef CompactIOField<Field<Type>, Type> fieldType;
typedef CompactIOField<Field<Type>, Type> Container;
DynamicList<word> fieldNames;
fieldNames.append
// CompactIOField Field names
fieldNames.push_back
(
filterObjects<fieldType>
filterObjects<CompactIOField<Field<Type>, Type>>
(
objects,
selectedFields
)
);
// Append IOField Field names
fieldNames.append
// IOField Field names
fieldNames.push_back
(
filterObjects<IOField<Field<Type>>>
(
@ -184,80 +180,69 @@ Foam::label Foam::parLagrangianDistributor::distributeFieldFields
)
);
bool reconstruct = false;
// Read if present
IOobject srcIOobject
(
"none",
srcMesh_.time().timeName(),
cloud::prefix/cloudName,
srcMesh_,
IOobjectOption::LAZY_READ,
IOobjectOption::NO_WRITE,
IOobjectOption::NO_REGISTER
);
IOobject tgtIOobject
(
"none",
tgtMesh_.time().timeName(),
cloud::prefix/cloudName,
tgtMesh_,
IOobjectOption::NO_READ,
IOobjectOption::NO_WRITE,
IOobjectOption::NO_REGISTER
);
//bool reconstruct = false;
if (verbose_ && fieldNames.size())
{
Info<< " Distributing lagrangian "
<< Container::typeName << "s\n" << nl;
}
label nFields = 0;
for (const word& objectName : fieldNames)
{
if (!nFields)
{
// Performing an all-to-one (reconstruct)?
reconstruct =
returnReduceAnd(!map.constructSize() || Pstream::master());
}
if (verbose_)
{
if (!nFields)
{
Info<< " Distributing lagrangian "
<< fieldType::typeName << "s\n" << nl;
}
Info<< " " << objectName << nl;
}
++nFields;
// Read if present
CompactIOField<Field<Type>, Type> field
(
IOobject
(
objectName,
srcMesh_.time().timeName(),
cloud::prefix/cloudName,
srcMesh_,
IOobject::READ_IF_PRESENT,
IOobject::NO_WRITE,
IOobject::NO_REGISTER
),
label(0)
);
srcIOobject.resetHeader(objectName);
tgtIOobject.resetHeader(objectName);
// Read if present (ie, haveCloud means readOnProc)
Container field(srcIOobject, haveCloud);
// Distribute
map.distribute(field);
const bool writeOnProc = field.size();
// Write
const IOobject fieldIO
(
objectName,
tgtMesh_.time().timeName(),
cloud::prefix/cloudName,
tgtMesh_,
IOobject::NO_READ,
IOobject::NO_WRITE,
IOobject::NO_REGISTER
);
Container(tgtIOobject, std::move(field)).write(writeOnProc);
if (field.size())
{
CompactIOField<Field<Type>, Type>
(
fieldIO,
std::move(field)
).write();
}
else if (!reconstruct)
{
// When running with -overwrite it should also delete the old
// files. Below works but is not optimal.
const fileName fldName(fieldIO.objectPath());
Foam::rm(fldName);
}
//if (!writeOnProc && !reconstruct)
//{
// // When running with -overwrite it should also delete the old
// // files. Below works but is not optimal.
//
// Foam::rm(tgtIOobject.objectPath());
//}
}
if (nFields && verbose_) Info<< endl;
return nFields;
if (verbose_ && fieldNames.size()) Info<< endl;
return fieldNames.size();
}
@ -265,12 +250,11 @@ template<class Container>
Foam::label Foam::parLagrangianDistributor::readFields
(
const passivePositionParticleCloud& cloud,
const bool haveCloud,
const IOobjectList& objects,
const wordRes& selectedFields
)
{
const word fieldClassName(Container::typeName);
const wordList fieldNames
(
filterObjects<Container>
@ -280,40 +264,40 @@ Foam::label Foam::parLagrangianDistributor::readFields
)
);
label nFields = 0;
// Read if present
IOobject readIO
(
"none",
cloud.time().timeName(),
cloud,
IOobjectOption::LAZY_READ,
IOobjectOption::NO_WRITE,
IOobjectOption::REGISTER
);
if (verbose_ && fieldNames.size())
{
Info<< " Reading lagrangian "
<< Container::typeName << "s\n" << nl;
}
for (const word& objectName : fieldNames)
{
if (verbose_)
{
if (!nFields)
{
Info<< " Reading lagrangian "
<< Container::typeName << "s\n" << nl;
}
Info<< " " << objectName << nl;
}
++nFields;
// Read if present
Container* fieldPtr = new Container
(
IOobject
(
objectName,
cloud.time().timeName(),
cloud,
IOobject::LAZY_READ,
IOobject::NO_WRITE,
IOobject::REGISTER
),
label(0)
);
readIO.resetHeader(objectName);
// Read if present (ie, haveCloud means readOnProc)
Container* fieldPtr = new Container(readIO, haveCloud);
fieldPtr->store();
}
if (nFields && verbose_) Info<< endl;
return nFields;
if (verbose_ && fieldNames.size()) Info<< endl;
return fieldNames.size();
}
@ -324,58 +308,55 @@ Foam::label Foam::parLagrangianDistributor::distributeStoredFields
passivePositionParticleCloud& cloud
) const
{
bool reconstruct = false;
label nFields = 0;
// Parallel-consistent
UPtrList<Container> fields(cloud.sorted<Container>());
for (Container& field : cloud.sorted<Container>())
IOobject writeIO
(
"none",
tgtMesh_.time().timeName(),
cloud::prefix/cloud.name(),
tgtMesh_,
IOobjectOption::NO_READ,
IOobjectOption::NO_WRITE,
IOobjectOption::NO_REGISTER
);
//bool reconstruct = false;
if (verbose_ && fields.size())
{
if (!nFields)
{
// Performing an all-to-one (reconstruct)?
reconstruct =
returnReduceAnd(!map.constructSize() || Pstream::master());
}
Info<< " Distributing lagrangian "
<< Container::typeName << "s\n" << nl;
}
for (Container& field : fields)
{
if (verbose_)
{
if (!nFields)
{
Info<< " Distributing lagrangian "
<< Container::typeName << "s\n" << nl;
}
Info<< " " << field.name() << nl;
}
++nFields;
map.distribute(field);
const IOobject fieldIO
(
field.name(),
tgtMesh_.time().timeName(),
cloud::prefix/cloud.name(),
tgtMesh_,
IOobject::NO_READ,
IOobject::NO_WRITE,
IOobject::NO_REGISTER
);
writeIO.resetHeader(field.name());
if (field.size())
{
Container(fieldIO, std::move(field)).write();
}
else if (!reconstruct)
{
// When running with -overwrite it should also delete the old
// files. Below works but is not optimal.
const bool writeOnProc = field.size();
const fileName fldName(fieldIO.objectPath());
Foam::rm(fldName);
}
// Write
Container(writeIO, std::move(field)).write(writeOnProc);
//if (!writeOnProc && !reconstruct)
//{
// // When running with -overwrite it should also delete the old
// // files. Below works but is not optimal.
//
// Foam::rm(writeIO.objectPath());
//}
}
if (nFields && verbose_) Info<< endl;
return nFields;
if (verbose_ && fields.size()) Info<< endl;
return fields.size();
}

36
applications/utilities/parallelProcessing/redistributePar/redistributeLagrangian.H
View File

@ -6,7 +6,7 @@
\\/ M anipulation |
-------------------------------------------------------------------------------
Copyright (C) 2011-2017 OpenFOAM Foundation
Copyright (C) 2015-2022 OpenCFD Ltd.
Copyright (C) 2015-2023 OpenCFD Ltd.
-------------------------------------------------------------------------------
License
This file is part of OpenFOAM.
@ -48,6 +48,7 @@ readLagrangian
(
const fvMesh& mesh,
const wordList& cloudNames,
const boolUList& haveClouds,
const wordRes& selectedFields
)
{
@ -78,11 +79,10 @@ readLagrangian
IOobjectList cloudObjs(clouds[i], clouds[i].time().timeName());
//Pout<< "Found cloud objects:" << cloudObjs.names() << endl;
parLagrangianDistributor::readAllFields
(
clouds[i],
haveClouds[i],
cloudObjs,
selectedFields
);
@ -101,11 +101,19 @@ readLagrangian
)
{
wordList cloudNames;
boolList haveClouds;
List<wordList> fieldNames;
// Find all cloudNames on all processors
parLagrangianDistributor::findClouds(mesh, cloudNames, fieldNames);
return readLagrangian(mesh, cloudNames, selectedFields);
// Find all cloudNames on all processors
parLagrangianDistributor::findClouds
(
mesh,
cloudNames,
haveClouds,
fieldNames
);
return readLagrangian(mesh, cloudNames, haveClouds, selectedFields);
}
@ -121,9 +129,17 @@ void reconstructLagrangian
// Clouds (note: might not be present on all processors)
wordList cloudNames;
boolList haveClouds;
List<wordList> fieldNames;
// Find all cloudNames on all processors
parLagrangianDistributor::findClouds(mesh, cloudNames, fieldNames);
parLagrangianDistributor::findClouds
(
mesh,
cloudNames,
haveClouds,
fieldNames
);
if (cloudNames.empty())
{
@ -142,13 +158,16 @@ void reconstructLagrangian
baseMesh,
mesh.nCells(), // range of cell indices in clouds
distMap
//writeHandler // Which processors to write
)
);
}
const auto& distributor = *distributorPtr;
for (const word& cloudName : cloudNames)
forAll(cloudNames, cloudi)
{
const word& cloudName = cloudNames[cloudi];
Info<< "Reconstructing lagrangian fields for cloud "
<< cloudName << nl << endl;
@ -169,6 +188,7 @@ void reconstructLagrangian
(
lagrangianMapPtr(),
cloudName,
haveClouds[cloudi],
cloudObjs,
selectedFields
);

1338
applications/utilities/parallelProcessing/redistributePar/redistributePar.C
View File

File diff suppressed because it is too large Load Diff

6
src/OpenFOAM/db/Time/Time.C
View File

@ -202,7 +202,8 @@ void Foam::Time::setControls()
// alternative processorsDDD directories naming
// - do not look for compressed (is directory)
if (fileHandler().filePath(timePath(), false).empty())
const fileName timeDir(fileHandler().filePath(timePath(), false));
if (returnReduceAnd(timeDir.empty()))
{
const int oldPrecision = precision_;
int requiredPrecision = -1;
@ -225,7 +226,8 @@ void Foam::Time::setControls()
oldTime = std::move(newTime);
// Does a time directory exist with the new format?
if (!fileHandler().filePath(timePath(), false).empty())
const fileName timeDir(fileHandler().filePath(timePath(), false));
if (returnReduceOr(!timeDir.empty()))
{
requiredPrecision = precision_;
}

12
src/OpenFOAM/db/dictionary/functionEntries/codeStream/codeStream.C
View File

@ -196,7 +196,17 @@ Foam::functionEntries::codeStream::getFunction
// (flag set by regIOobject::read, baseIOdictionary constructor)
if (!masterOnly && returnReduceOr(lib == nullptr))
{
// Broadcast distributed...
// Broadcast to distributed masters
if (fileHandler().distributed())
{
fileHandler().broadcastCopy
(
UPstream::worldComm,
UPstream::master(fileHandler().comm()),
libPath,
libPath
);
}
dynamicCode::waitForFile(libPath, context.dict());
}

51
src/OpenFOAM/db/dictionary/functionEntries/includeEntry/includeEntry.C
View File

@ -6,7 +6,7 @@
\\/ M anipulation |
-------------------------------------------------------------------------------
Copyright (C) 2011-2017 OpenFOAM Foundation
Copyright (C) 2018-2021 OpenCFD Ltd.
Copyright (C) 2018-2022 OpenCFD Ltd.
-------------------------------------------------------------------------------
License
This file is part of OpenFOAM.
@ -27,12 +27,13 @@ License
\*---------------------------------------------------------------------------*/
#include "includeEntry.H"
#include "addToMemberFunctionSelectionTable.H"
#include "stringOps.H"
#include "IFstream.H"
#include "IOstreams.H"
#include "Time.H"
#include "UPstream.H"
#include "fileOperation.H"
#include "regIOobject.H"
#include "addToMemberFunctionSelectionTable.H"
// * * * * * * * * * * * * * * Static Data Members * * * * * * * * * * * * * //
@ -133,6 +134,15 @@ bool Foam::functionEntries::includeEntry::execute
Istream& is
)
{
const auto* rioPtr = isA<regIOobject>(parentDict.topDict());
const label oldComm
(
rioPtr && rioPtr->global()
? fileHandler().comm(UPstream::worldComm)
: fileHandler().comm()
);
const fileName rawName(is);
const fileName fName(resolveFile(is.name().path(), rawName, parentDict));
@ -148,20 +158,19 @@ bool Foam::functionEntries::includeEntry::execute
}
// Add watch on included file
const dictionary& top = parentDict.topDict();
if (isA<regIOobject>(top))
if (rioPtr)
{
regIOobject& rio = const_cast<regIOobject&>
(
dynamic_cast<const regIOobject&>(top)
);
rio.addWatch(fName);
const_cast<regIOobject&>(*rioPtr).addWatch(fName);
}
parentDict.read(ifs);
fileHandler().comm(oldComm);
return true;
}
fileHandler().comm(oldComm);
if (!mandatory)
{
return true; // Never fails if optional
@ -185,6 +194,15 @@ bool Foam::functionEntries::includeEntry::execute
Istream& is
)
{
const auto* rioPtr = isA<regIOobject>(parentDict.topDict());
const label oldComm
(
rioPtr && rioPtr->global()
? fileHandler().comm(UPstream::worldComm)
: fileHandler().comm()
);
const fileName rawName(is);
const fileName fName(resolveFile(is.name().path(), rawName, parentDict));
@ -200,20 +218,19 @@ bool Foam::functionEntries::includeEntry::execute
}
// Add watch on included file
const dictionary& top = parentDict.topDict();
if (isA<regIOobject>(top))
if (rioPtr)
{
regIOobject& rio = const_cast<regIOobject&>
(
dynamic_cast<const regIOobject&>(top)
);
rio.addWatch(fName);
const_cast<regIOobject&>(*rioPtr).addWatch(fName);
}
entry.read(parentDict, ifs);
fileHandler().comm(oldComm);
return true;
}
fileHandler().comm(oldComm);
if (!mandatory)
{
return true; // Never fails if optional

32
src/OpenFOAM/db/dictionary/functionEntries/includeEtcEntry/includeEtcEntry.C
View File

@ -6,7 +6,7 @@
\\/ M anipulation |
-------------------------------------------------------------------------------
Copyright (C) 2015-2017 OpenFOAM Foundation
Copyright (C) 2019-2021 OpenCFD Ltd.
Copyright (C) 2019-2022 OpenCFD Ltd.
-------------------------------------------------------------------------------
License
This file is part of OpenFOAM.
@ -27,12 +27,14 @@ License
\*---------------------------------------------------------------------------*/
#include "includeEtcEntry.H"
#include "addToMemberFunctionSelectionTable.H"
#include "etcFiles.H"
#include "stringOps.H"
#include "IFstream.H"
#include "IOstreams.H"
#include "UPstream.H"
#include "fileOperation.H"
#include "regIOobject.H"
#include "addToMemberFunctionSelectionTable.H"
// * * * * * * * * * * * * * * Static Data Members * * * * * * * * * * * * * //
@ -113,6 +115,15 @@ bool Foam::functionEntries::includeEtcEntry::execute
Istream& is
)
{
const regIOobject* rioPtr = isA<regIOobject>(parentDict.topDict());
const label oldComm
(
rioPtr && rioPtr->global()
? fileHandler().comm(UPstream::worldComm)
: fileHandler().comm()
);
const fileName rawName(is);
const fileName fName(resolveEtcFile(rawName, parentDict));
@ -127,9 +138,13 @@ bool Foam::functionEntries::includeEtcEntry::execute
Info<< fName << nl;
}
parentDict.read(ifs);
fileHandler().comm(oldComm);
return true;
}
fileHandler().comm(oldComm);
if (!mandatory)
{
return true; // Never fails if optional
@ -153,6 +168,15 @@ bool Foam::functionEntries::includeEtcEntry::execute
Istream& is
)
{
const regIOobject* rioPtr = isA<regIOobject>(parentDict.topDict());
const label oldComm
(
rioPtr && rioPtr->global()
? fileHandler().comm(UPstream::worldComm)
: fileHandler().comm()
);
const fileName rawName(is);
const fileName fName(resolveEtcFile(rawName, parentDict));
@ -167,9 +191,13 @@ bool Foam::functionEntries::includeEtcEntry::execute
Info<< fName << nl;
}
entry.read(parentDict, ifs);
fileHandler().comm(oldComm);
return true;
}
fileHandler().comm(oldComm);
if (!mandatory)
{
return true; // Never fails if optional

16
src/OpenFOAM/db/dynamicLibrary/codedBase/codedBase.C
View File

@ -254,9 +254,21 @@ void Foam::codedBase::createLibrary
UPstream::barrier(UPstream::worldComm);
}
// Broadcast distributed...
const fileName libPath = dynCode.libPath();
dynamicCode::waitForFile(dynCode.libPath(), context.dict());
// Broadcast to distributed masters
if (fileHandler().distributed())
{
fileHandler().broadcastCopy
(
UPstream::worldComm,
UPstream::master(fileHandler().comm()),
libPath,
libPath
);
}
dynamicCode::waitForFile(libPath, context.dict());
}

103
src/OpenFOAM/db/regIOobject/regIOobject.C
View File

@ -52,7 +52,6 @@ Foam::regIOobject::regIOobject(const IOobject& io, const bool isTimeObject)
IOobject(io),
registered_(false),
ownedByRegistry_(false),
watchIndices_(),
eventNo_(isTimeObject ? 0 : db().getEvent()), // No event for top-level Time
metaDataPtr_(nullptr),
isPtr_(nullptr)
@ -70,8 +69,9 @@ Foam::regIOobject::regIOobject(const regIOobject& rio)
IOobject(rio),
registered_(false),
ownedByRegistry_(false),
watchIndices_(rio.watchIndices_),
eventNo_(db().getEvent()),
watchFiles_(rio.watchFiles_),
watchIndices_(rio.watchIndices_),
metaDataPtr_(rio.metaDataPtr_.clone()),
isPtr_(nullptr)
{
@ -84,7 +84,6 @@ Foam::regIOobject::regIOobject(const regIOobject& rio, bool registerCopy)
IOobject(rio),
registered_(false),
ownedByRegistry_(false),
watchIndices_(),
eventNo_(db().getEvent()),
metaDataPtr_(rio.metaDataPtr_.clone()),
isPtr_(nullptr)
@ -111,7 +110,6 @@ Foam::regIOobject::regIOobject
IOobject(newName, rio.instance(), rio.local(), rio.db()),
registered_(false),
ownedByRegistry_(false),
watchIndices_(),
eventNo_(db().getEvent()),
metaDataPtr_(rio.metaDataPtr_.clone()),
isPtr_(nullptr)
@ -135,7 +133,6 @@ Foam::regIOobject::regIOobject
IOobject(io),
registered_(false),
ownedByRegistry_(false),
watchIndices_(),
eventNo_(db().getEvent()),
metaDataPtr_(rio.metaDataPtr_.clone()),
isPtr_(nullptr)
@ -234,6 +231,7 @@ bool Foam::regIOobject::checkOut()
fileHandler().removeWatch(watchIndices_[i]);
}
watchIndices_.clear();
watchFiles_.clear();
if (registered_)
{
@ -253,17 +251,23 @@ Foam::label Foam::regIOobject::addWatch(const fileName& f)
if
(
registered_
&& readOpt() == IOobject::MUST_READ_IF_MODIFIED
&& readOpt() == IOobjectOption::READ_MODIFIED
&& time().runTimeModifiable()
)
{
index = fileHandler().findWatch(watchIndices_, f);
//- 1. Directly add to fileHandler
//index = fileHandler().findWatch(watchIndices_, f);
//
//if (index == -1)
//{
// index = watchIndices_.size();
// watchIndices_.push_back(fileHandler().addWatch(f));
//}
if (index == -1)
{
index = watchIndices_.size();
watchIndices_.append(fileHandler().addWatch(f));
}
//- 2. Delay adding; add to list and handle in addWatch() later on
// Note: what do we return?
index = watchFiles_.size();
watchFiles_.push_back(f);
}
return index;
@ -275,7 +279,7 @@ void Foam::regIOobject::addWatch()
if
(
registered_
&& readOpt() == IOobject::MUST_READ_IF_MODIFIED
&& readOpt() == IOobjectOption::READ_MODIFIED
&& time().runTimeModifiable()
)
{
@ -307,37 +311,80 @@ void Foam::regIOobject::addWatch()
)
);
if (masterOnly && Pstream::parRun())
if (masterOnly && UPstream::parRun())
{
// Get master watched files
fileNameList watchFiles;
if (Pstream::master())
// Get all files watched on master, and broadcast at once
fileNameList filesToWatch;
if (UPstream::master())
{
watchFiles.resize(watchIndices_.size());
const bool oldParRun = UPstream::parRun(false);
filesToWatch.resize(watchIndices_.size());
forAll(watchIndices_, i)
{
watchFiles[i] = fileHandler().getFile(watchIndices_[i]);
filesToWatch[i] = fileHandler().getFile(watchIndices_[i]);
}
}
Pstream::broadcast(watchFiles);
if (!Pstream::master())
UPstream::parRun(oldParRun);
}
Pstream::broadcast(filesToWatch);
// Add master files in same order
if (!UPstream::master())
{
// unregister current ones
const bool oldParRun = UPstream::parRun(false);
// Unregister current watched indices
forAllReverse(watchIndices_, i)
{
fileHandler().removeWatch(watchIndices_[i]);
}
// Insert the ones from master, in master order
watchIndices_.clear();
forAll(watchFiles, i)
for (const auto& file : filesToWatch)
{
watchIndices_.append(fileHandler().addWatch(watchFiles[i]));
watchIndices_.push_back(fileHandler().addWatch(file));
}
}
}
watchIndices_.append(fileHandler().addWatch(f));
UPstream::parRun(oldParRun);
}
// Files that were explicitly added via addWatch(const fileName&)
// (e.g. through #include)
for (const auto& file : watchFiles_)
{
watchIndices_.push_back(fileHandler().addWatch(file));
}
// Append the local file
watchIndices_.push_back(fileHandler().addWatch(f));
}
else
{
DynamicList<fileName> filesToWatch
(
watchIndices_.size()+watchFiles_.size()+1
);
// Get existing watched files from fileHandler
for (const label index : watchIndices_)
{
filesToWatch.push_back(fileHandler().getFile(index));
}
// The files explicitly added from addWatch(const fileName&)
// (e.g. through #include)
filesToWatch.push_back(std::move(watchFiles_));
// The local file
filesToWatch.push_back(f);
// Re-do all watches
fileHandler().addWatches(*this, filesToWatch);
}
}
}

12
src/OpenFOAM/db/regIOobject/regIOobject.H
View File

@ -43,6 +43,7 @@ SourceFiles
#include "IOobject.H"
#include "OSspecific.H"
#include "DynamicList.H"
// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
@ -94,12 +95,15 @@ private:
//- Is this object owned by the registry
bool ownedByRegistry_;
//- List of modification watch indices
mutable labelList watchIndices_;
//- eventNo of last update
//- The eventNo of last update
label eventNo_;
//- List of additional files to watch
mutable DynamicList<fileName> watchFiles_;
//- List of modification watch indices
mutable DynamicList<label> watchIndices_;
//- Dictionary for any meta-data
autoPtr<dictionary> metaDataPtr_;

6
src/OpenFOAM/db/regIOobject/regIOobjectRead.C
View File

@ -58,11 +58,11 @@ bool Foam::regIOobject::readHeaderOk
{
if (masterOnly)
{
if (Pstream::master())
if (UPstream::master())
{
const bool oldParRun = Pstream::parRun(false);
const bool oldParRun = UPstream::parRun(false);
isHeaderOk = headerOk();
Pstream::parRun(oldParRun);
UPstream::parRun(oldParRun);
}
Pstream::broadcast(isHeaderOk);
}

2
src/OpenFOAM/db/regIOobject/regIOobjectWrite.C
View File

@ -105,7 +105,7 @@ bool Foam::regIOobject::writeObject
);
bool osGood = false;
if (!masterOnly || Pstream::master())
if (!masterOnly || UPstream::master())
{
osGood = fileHandler().writeObject(*this, streamOpt, writeOnProc);
}

146
src/OpenFOAM/global/argList/argList.C
View File

@ -1526,6 +1526,9 @@ void Foam::argList::parse
// also have to protect the actual dictionary parsing since
// it might trigger file access (e.g. #include, #codeStream)
const bool oldParRun = Pstream::parRun(false);
// Note: non-parallel running might update
// fileOperation::nProcs() so store & restore below
const label nOldProcs = fileHandler().nProcs();
autoPtr<ISstream> dictStream
(
@ -1584,6 +1587,7 @@ void Foam::argList::parse
}
Pstream::parRun(oldParRun); // Restore parallel state
const_cast<fileOperation&>(fileHandler()).nProcs(nOldProcs);
if (Pstream::nProcs() == 1)
{
@ -1763,16 +1767,148 @@ void Foam::argList::parse
}
// If needed, adjust fileHandler for distributed roots
if (runControl_.distributed())
if (runControl_.distributed() && fileOperation::fileHandlerPtr_)
{
if (fileOperation::fileHandlerPtr_)
fileOperation::fileHandlerPtr_->distributed(true);
const labelList& ranks = fileHandler().ioRanks();
if (runControl_.parRun() && ranks.size())
{
fileOperation::fileHandlerPtr_->distributed(true);
// Detect processor directories both on local proc and on
// (world) master proc. If the local proc doesn't have them
// but the master has it will attempt to copy them.
// Expected local directory name
const fileName procDir
(
rootPath_
/ globalCase_
/ ("processor" + Foam::name(UPstream::myProcNo()))
);
// Try and find my local directory using the fileHandler. This
// will check the local disk on the IO rank
// (since running distributed)
fileNameList pathDirs(UPstream::nProcs());
auto& pathDir = pathDirs[UPstream::myProcNo()];
pathDir = fileHandler().filePath(procDir, false);
if (returnReduceOr(pathDir.empty()))
{
// There is at least one processor that cannot find
// the processor directory. Look for it on the master.
// E.g. decomposed into 4 processors, two roots:
// processors4_0-1/
// processors4_2-3/
// So:
// - processor0 reads the same disk as processor0
// - processor2 needs the whole directory sent over
// - processor3 reads the same disk as processor2
if (UPstream::master() && bannerEnabled())
{
Info<< "I/O :"
<< " distributed - copying missing directories"
<< nl;
}
// Collect all wanted directories (or empty). Note: could
// just collect missing ones ...
Pstream::gatherList(pathDirs);
fileName masterRootPath(rootPath_);
Pstream::broadcast(masterRootPath);
List<fileNameList> rankToDirs(UPstream::nProcs());
if (UPstream::master())
{
const bool oldParRun = Pstream::parRun(false);
// Note: non-parallel running might update
// fileOperation::nProcs() so store & restore below
const label nOldProcs = fileHandler().nProcs();
label rank = 0;
for (label proci = 1; proci < pathDirs.size(); ++proci)
{
if (ranks.contains(proci))
{
rank = proci;
}
if (pathDirs[proci].empty())
{
// Synthesise corresponding name on the master
// processor
const fileName procDir
(
rootPath_
/ globalCase_
/ ("processor" + Foam::name(proci))
);
const fileName foundDir
(
fileHandler().filePath(procDir, false)
);
if
(
!foundDir.empty()
&& !rankToDirs[rank].contains(foundDir)
)
{
rankToDirs[rank].push_back(foundDir);
}
}
}
UPstream::parRun(oldParRun);
const_cast<fileOperation&>(fileHandler()).nProcs(nOldProcs);
}
Pstream::broadcast(rankToDirs);
// Copy missing directories on all the IOranks.
// Note: instead of passing 'writeOnProc' flag we could create
// communicator just between master and IOrank, but that is
// also expensive.
forAll(rankToDirs, proci)
{
// Am I the reponsible IOrank for this processor
const bool amIO = (UPstream::myProcNo() == proci);
// Construct equivalent directory on proci
for (const auto& srcDir : rankToDirs[proci])
{
const fileName tgtDir
(
rootPath_
/ srcDir.relative(masterRootPath)
);
if (amIO)
{
// I am the IO rank
Pout<< "On rank " << proci << nl
<< " copying : " << srcDir << nl
<< " to : " << tgtDir << endl;
}
fileHandler().broadcastCopy
(
UPstream::worldComm,
amIO,
tgtDir,
tgtDir
);
}
}
}
}
}
// Keep/discard sub-process host/root information for reporting:
if (Pstream::master() && runControl_.parRun())
if (UPstream::master() && runControl_.parRun())
{
if (!writeHostsSwitch)
{
@ -1785,7 +1921,7 @@ void Foam::argList::parse
}
}
if (Pstream::master() && bannerEnabled())
if (UPstream::master() && bannerEnabled())
{
Info<< "Case : " << (rootPath_/globalCase_).c_str() << nl
<< "nProcs : " << nProcs << nl;

68
src/OpenFOAM/global/fileOperations/collatedFileOperation/collatedFileOperation.C
View File

@ -113,43 +113,7 @@ void Foam::fileOperations::collatedFileOperation::printBanner
if (withRanks)
{
// Information about the ranks
stringList hosts(Pstream::nProcs());
if (Pstream::master(comm_))
{
hosts[Pstream::myProcNo()] = hostName();
}
Pstream::gatherList(hosts);
DynamicList<label> offsetMaster(Pstream::nProcs());
forAll(hosts, ranki)
{
if (!hosts[ranki].empty())
{
offsetMaster.append(ranki);
}
}
if (offsetMaster.size() > 1)
{
DetailInfo
<< "IO nodes:" << nl << '(' << nl;
offsetMaster.append(Pstream::nProcs());
for (label group = 1; group < offsetMaster.size(); ++group)
{
const label beg = offsetMaster[group-1];
const label end = offsetMaster[group];
DetailInfo
<< " (" << hosts[beg].c_str() << ' '
<< (end-beg) << ')' << nl;
}
DetailInfo
<< ')' << nl;
}
fileOperation::printRanks();
}
//- fileModificationChecking already set by base class (masterUncollated)
@ -386,8 +350,10 @@ bool Foam::fileOperations::collatedFileOperation::writeObject
if (inst.isAbsolute() || !tm.processorCase())
{
mkDir(io.path());
fileName pathName(io.objectPath());
// Note: delay mkdir to masterOFstream so it does not get created
// if not needed (e.g. when running distributed)
const fileName pathName(io.objectPath());
if (debug)
{
@ -427,10 +393,12 @@ bool Foam::fileOperations::collatedFileOperation::writeObject
else
{
// Construct the equivalent processors/ directory
fileName path(processorsPath(io, inst, processorsDir(io)));
const fileName path(processorsPath(io, inst, processorsDir(io)));
mkDir(path);
fileName pathName(path/io.name());
// Note: delay mkdir to masterOFstream so it does not get created
// if not needed (e.g. when running distributed)
const fileName pathName(path/io.name());
if (io.global() || io.globalObject())
{
@ -480,6 +448,7 @@ bool Foam::fileOperations::collatedFileOperation::writeObject
<< " appending to " << pathName << endl;
}
mkDir(path);
return appendObject(io, pathName, streamOpt);
}
else
@ -604,11 +573,16 @@ Foam::word Foam::fileOperations::collatedFileOperation::processorsDir
break;
}
}
procDir +=
+ "_"
+ Foam::name(minProc)
+ "-"
+ Foam::name(maxProc);
// Add range if not all processors
if (maxProc-minProc+1 != nProcs_)
{
procDir +=
+ "_"
+ Foam::name(minProc)
+ "-"
+ Foam::name(maxProc);
}
}
}

79
src/OpenFOAM/global/fileOperations/fileOperation/fileOperation.C
View File

@ -80,6 +80,7 @@ Foam::word Foam::fileOperation::processorsBaseDir = "processors";
//- Caching (e.g. of time directories) - enabled by default
int Foam::fileOperation::cacheLevel_(1);
int Foam::fileOperation::nProcsFilter_(-1);
// * * * * * * * * * * * * * * * Local Functions * * * * * * * * * * * * * * //
@ -179,8 +180,6 @@ static bool parseProcsNumRange
} // End anonymous namespace
#if 0
// Sorting of processor directories
#include "stringOpsSort.H"
namespace
@ -214,7 +213,6 @@ void sortProcessorDirs(Foam::UList<Foam::fileOperation::dirIndex>& dirs)
}
} // End anonymous namespace
#endif
// * * * * * * * * * * * * * Static Member Functions * * * * * * * * * * * * //
@ -431,6 +429,21 @@ Foam::fileOperation::lookupAndCacheProcessorsPath
// if directory status gets synchronised
// - distributed() : different processors have different roots
// - fileModificationChecking : (uncollated only) do IO on master only
// - nProcsFilter_ : if set to
// 0 : accept any directory (e.g. for redistributePar where we don't
// know yet number of read/write procs)
// -1 : accept only processorsDDD where DDD is nProcs(worldComm)
// >0 : accept the exact mentioned number of prcessors
// Collated : check whether/how to filter processorsXXX directory names
const label targetNProcs
(
(UPstream::parRun() && nProcsFilter_ < 0)
? UPstream::nProcs(UPstream::worldComm)
: nProcsFilter_
);
fileName path, pDir, local;
procRangeType group;
@ -528,6 +541,19 @@ Foam::fileOperation::lookupAndCacheProcessorsPath
}
else if (rNum != -1)
{
if (targetNProcs > 1 && (targetNProcs != rNum))
{
// Current directory can never contain wanted proc
//Pout<< "For fName:" << fName
// << "Ignoring directory " << dirN
// << " since parsed rNum:" << rNum
// << " targetNProcs:" << targetNProcs
// << endl;
//error::printStack(Pout);
continue;
}
// "processorsNN" or "processorsNN_start-end"
nProcs = max(nProcs, rNum);
@ -535,7 +561,7 @@ Foam::fileOperation::lookupAndCacheProcessorsPath
{
// "processorsNN"
if (proci < rNum)
if (proci < rNum || (nProcsFilter_ == 0))
{
// And it is also in range.
// Eg for "processors4": 3 is ok, 10 is not
@ -544,7 +570,7 @@ Foam::fileOperation::lookupAndCacheProcessorsPath
pathTypeIdx.second() = proci;
}
}
else if (group.contains(proci))
else if (group.contains(proci) || (nProcsFilter_ == 0))
{
// "processorsNN_start-end"
// - save the local proc offset
@ -560,6 +586,10 @@ Foam::fileOperation::lookupAndCacheProcessorsPath
}
}
// Sort processor directory names (natural order)
sortProcessorDirs(procDirs);
// Global check of empty/exists.
// 1 : empty directory
// 2 : non-empty directory
@ -571,7 +601,7 @@ Foam::fileOperation::lookupAndCacheProcessorsPath
if (debug)
{
Pout<< "fileOperation::lookupProcessorsPath " << procPath
<< " detected:" << procDirs << endl;
<< " detected:" << flatOutput(procDirs) << endl;
}
if (UPstream::parRun() && (!distributed() || syncPar))
@ -654,9 +684,6 @@ Foam::fileOperation::lookupAndCacheProcessorsPath
}
}
// Sort processor directory names (natural order)
/// sortProcessorDirs(procDirs);
if (procDirsStatus & 2u)
{
if (cacheLevel() > 0)
@ -680,7 +707,6 @@ Foam::fileOperation::lookupAndCacheProcessorsPath
Foam::refPtr<Foam::fileOperation::dirIndexList>
Foam::fileOperation::lookupProcessorsPath(const fileName& fName) const
{
// Use parallel synchronisation
return lookupAndCacheProcessorsPath(fName, true);
}
@ -844,6 +870,7 @@ Foam::fileName Foam::fileOperation::filePath
}
// Give preference to processors variant
fileName foundName;
if (proci != -1)
{
// Get all processor directories
@ -859,26 +886,40 @@ Foam::fileName Foam::fileOperation::filePath
{
Pout<< "fileOperation::filePath : " << collatedName << endl;
}
return collatedName;
foundName = collatedName;
}
}
}
if (exists(fName, checkGzip, followLink))
//if (returnReduceOr(foundName.empty())) // worldComm
if (foundName.empty())
{
// There is at least one processor that cannot find the processors
// directory. Re-do with straight supplied filename
if (exists(fName, checkGzip, followLink))
{
if (foundName.empty())
{
foundName = fName;
}
}
}
if (!foundName.empty())
{
if (debug)
{
Pout<< "fileOperation::filePath : " << fName << endl;
Pout<< "fileOperation::filePath : " << foundName << endl;
}
return fName;
}
if (debug)
else
{
Pout<< "fileOperation::filePath : Not found" << endl;
if (debug)
{
Pout<< "fileOperation::filePath : Not found" << endl;
}
}
return fileName();
return foundName;
}

22
src/OpenFOAM/global/fileOperations/fileOperation/fileOperation.H
View File

@ -150,6 +150,13 @@ private:
//- Storage of the dummy file handler (demand-driven)
static refPtr<fileOperation> dummyHandlerPtr_;
//- Filtering of processorsDDD directories (in parallel):
// - >0 : only accept processorsDDD (default)
// - 0 : accept anything (e.g. when detecting nprocs from
// directory naming)
// - -1 : use nProcs(UPstream::worldComm)
static int nProcsFilter_;
protected:
@ -452,6 +459,21 @@ public:
return old;
}
//- Return collated 'processorsDDD' filtering
static int nProcsFilter() noexcept
{
return nProcsFilter_;
}
//- Set collated 'processorsDDD' filtering (0 = off).
// \return the previous value
static int nProcsFilter(int level) noexcept
{
int old(nProcsFilter_);
nProcsFilter_ = level;
return old;
}
//- Sort directory entries according to time value,
// with "constant" appearing first (if it exists)
static instantList sortTimes

19
src/OpenFOAM/global/fileOperations/fileOperation/fileOperationNew.C
View File

@ -373,6 +373,11 @@ Foam::fileOperation::New_impl
// Allocate new handler with same type and similar IO ranks
// but with different sub-ranks (and communicator)
// Temporarily override world comm to get through the initialisation
// (e.g. fileOperation::printRanks() which uses worldComm)
const label oldWorldComm = UPstream::commWorld(commAndIORanks.first());
newHandler = fileOperation::New
(
origHandler.type(),
@ -381,9 +386,13 @@ Foam::fileOperation::New_impl
verbose
);
UPstream::commWorld(oldWorldComm);
if (newHandler)
{
newHandler->nProcs(origHandler.nProcs());
// Make sure that the output format uses the correct number of
// 'active' ranks (instead of that of the origHandler)
newHandler->nProcs(subProcs.size());
newHandler->storeComm();
}
}
@ -400,9 +409,9 @@ Foam::fileOperation::New
bool verbose
)
{
labelList subProcs = getSelectedProcs(useProc);
const labelList subProcs = getSelectedProcs(useProc);
return fileOperation::New_impl(origHandler, subProcs, verbose);
return New_impl(origHandler, subProcs, verbose);
}
@ -414,9 +423,9 @@ Foam::fileOperation::New
bool verbose
)
{
labelList subProcs = getSelectedProcs(useProc);
const labelList subProcs = getSelectedProcs(useProc);
return fileOperation::New_impl(origHandler, subProcs, verbose);
return New_impl(origHandler, subProcs, verbose);
}

2
src/OpenFOAM/global/fileOperations/fileOperation/fileOperationRanks.C
View File

@ -100,8 +100,6 @@ Foam::labelRange Foam::fileOperation::subRanks(const labelUList& mainIOranks)
}
}
Pout<< "subRanks: " << labelRange(begProc, (endProc-begProc)) << endl;
return labelRange(begProc, (endProc-begProc));
}

102
src/OpenFOAM/global/fileOperations/masterUncollatedFileOperation/masterUncollatedFileOperation.C
View File

@ -799,6 +799,23 @@ masterUncollatedFileOperation
managedComm_(getManagedComm(comm_)) // Possibly locally allocated
{
init(verbose);
if (comm_ == -1)
{
FatalErrorInFunction<< "Problem comm_:" << comm_ << exit(FatalError);
}
if (UPstream::nProcs(comm_) == -1)
{
FatalErrorInFunction<< "Problem comm_:" << comm_
<< " nProcs:" << UPstream::nProcs(comm_)
<< exit(FatalError);
}
if (UPstream::myProcNo(comm_) == -1)
{
FatalErrorInFunction<< "Problem comm_:" << comm_
<< " myProcNo:" << UPstream::myProcNo(comm_)
<< exit(FatalError);
}
}
@ -814,6 +831,23 @@ masterUncollatedFileOperation
managedComm_(-1) // Externally managed
{
init(verbose);
if (comm_ == -1)
{
FatalErrorInFunction<< "Problem comm_:" << comm_ << exit(FatalError);
}
if (UPstream::nProcs(comm_) == -1)
{
FatalErrorInFunction<< "Problem comm_:" << comm_
<< " nProcs:" << UPstream::nProcs(comm_)
<< exit(FatalError);
}
if (UPstream::myProcNo(comm_) == -1)
{
FatalErrorInFunction<< "Problem comm_:" << comm_
<< " myProcNo:" << UPstream::myProcNo(comm_)
<< exit(FatalError);
}
}
@ -1159,6 +1193,7 @@ Foam::fileName Foam::fileOperations::masterUncollatedFileOperation::filePath
{
const bool oldParRun = UPstream::parRun(false);
const int oldCache = fileOperation::cacheLevel(0);
const label oldNProcs = fileOperation::nProcs();
// All masters search locally. Note that global objects might
// fail (except on master). This gets handled later on (in PARENTOBJECT)
@ -1175,6 +1210,7 @@ Foam::fileName Foam::fileOperations::masterUncollatedFileOperation::filePath
newInstancePath
);
const_cast<masterUncollatedFileOperation&>(*this).nProcs(oldNProcs);
fileOperation::cacheLevel(oldCache);
UPstream::parRun(oldParRun);
@ -1315,6 +1351,7 @@ Foam::fileName Foam::fileOperations::masterUncollatedFileOperation::dirPath
{
const bool oldParRun = UPstream::parRun(false);
const int oldCache = fileOperation::cacheLevel(0);
const label oldNProcs = fileOperation::nProcs();
objPath = filePathInfo
(
@ -1328,6 +1365,7 @@ Foam::fileName Foam::fileOperations::masterUncollatedFileOperation::dirPath
newInstancePath
);
const_cast<masterUncollatedFileOperation&>(*this).nProcs(oldNProcs);
fileOperation::cacheLevel(oldCache);
UPstream::parRun(oldParRun);
@ -1517,11 +1555,13 @@ Foam::fileOperations::masterUncollatedFileOperation::findInstance
{
const bool oldParRun = UPstream::parRun(false);
const int oldCache = fileOperation::cacheLevel(0);
const label oldNProcs = fileOperation::nProcs();
if (exists(pDirs, io))
{
foundInstance = io.instance();
}
const_cast<masterUncollatedFileOperation&>(*this).nProcs(oldNProcs);
fileOperation::cacheLevel(oldCache);
UPstream::parRun(oldParRun);
}
@ -1556,6 +1596,7 @@ Foam::fileOperations::masterUncollatedFileOperation::findInstance
{
const bool oldParRun = UPstream::parRun(false);
const int oldCache = fileOperation::cacheLevel(0);
const label oldNProcs = fileOperation::nProcs();
label instIndex = ts.size()-1;
@ -1666,6 +1707,7 @@ Foam::fileOperations::masterUncollatedFileOperation::findInstance
}
}
const_cast<masterUncollatedFileOperation&>(*this).nProcs(oldNProcs);
fileOperation::cacheLevel(oldCache);
UPstream::parRun(oldParRun); // Restore parallel state
}
@ -1741,6 +1783,7 @@ Foam::fileOperations::masterUncollatedFileOperation::readObjects
// (through call to filePath which triggers parallel )
const bool oldParRun = UPstream::parRun(false);
const int oldCache = fileOperation::cacheLevel(0);
const label oldNProcs = fileOperation::nProcs();
//- Use non-time searching version
objectNames = fileOperation::readObjects
@ -1784,6 +1827,7 @@ Foam::fileOperations::masterUncollatedFileOperation::readObjects
}
}
const_cast<masterUncollatedFileOperation&>(*this).nProcs(oldNProcs);
fileOperation::cacheLevel(oldCache);
UPstream::parRun(oldParRun); // Restore parallel state
}
@ -2228,11 +2272,13 @@ bool Foam::fileOperations::masterUncollatedFileOperation::read
// Do master-only reading always.
const bool oldParRun = UPstream::parRun(false);
const int oldCache = fileOperation::cacheLevel(0);
const label oldNProcs = fileOperation::nProcs();
auto& is = io.readStream(typeName);
ok = io.readData(is);
io.close();
const_cast<masterUncollatedFileOperation&>(*this).nProcs(oldNProcs);
fileOperation::cacheLevel(oldCache);
UPstream::parRun(oldParRun); // Restore parallel state
}
@ -2362,9 +2408,11 @@ Foam::instantList Foam::fileOperations::masterUncollatedFileOperation::findTimes
// Do master-only reading always.
const bool oldParRun = UPstream::parRun(false);
const int oldCache = fileOperation::cacheLevel(0);
const label oldNProcs = fileOperation::nProcs();
times = fileOperation::findTimes(directory, constantName);
const_cast<masterUncollatedFileOperation&>(*this).nProcs(oldNProcs);
fileOperation::cacheLevel(oldCache);
UPstream::parRun(oldParRun); // Restore parallel state
}
@ -2766,30 +2814,46 @@ void Foam::fileOperations::masterUncollatedFileOperation::addWatches
{
const labelList& watchIndices = rio.watchIndices();
// Do on master and distribute effect to subprocs such that after
// all have consistent numbering & files
DynamicList<label> newWatchIndices;
labelHashSet removedWatches(watchIndices);
for (const fileName& f : files)
if (UPstream::master())
{
const label index = findWatch(watchIndices, f);
// Switch off comms inside findWatch/addWatch etc.
const bool oldParRun = UPstream::parRun(false);
const int oldCache = fileOperation::cacheLevel(0);
const label oldNProcs = fileOperation::nProcs();
if (index == -1)
{
newWatchIndices.append(addWatch(f));
}
else
{
// Existing watch
newWatchIndices.append(watchIndices[index]);
removedWatches.erase(index);
}
}
labelHashSet removedWatches(watchIndices);
// Remove any unused watches
for (const label index : removedWatches)
{
removeWatch(watchIndices[index]);
for (const fileName& f : files)
{
const label index = findWatch(watchIndices, f);
if (index == -1)
{
newWatchIndices.push_back(addWatch(f));
}
else
{
// Existing watch
newWatchIndices.push_back(watchIndices[index]);
removedWatches.erase(index);
}
}
// Remove any unused watches
for (const label index : removedWatches)
{
removeWatch(watchIndices[index]);
}
const_cast<masterUncollatedFileOperation&>(*this).nProcs(oldNProcs);
fileOperation::cacheLevel(oldCache);
UPstream::parRun(oldParRun);
}
Pstream::broadcast(newWatchIndices);
rio.watchIndices() = newWatchIndices;
}

7
src/OpenFOAM/global/fileOperations/uncollatedFileOperation/uncollatedFileOperation.C
View File

@ -121,12 +121,11 @@ Foam::fileName Foam::fileOperations::uncollatedFileOperation::filePathInfo
// Check if parallel "procesors" directory
if (io.time().processorCase())
{
refPtr<dirIndexList> pDirs
const refPtr<dirIndexList> pDirs
(
fileOperation::lookupAndCacheProcessorsPath
lookupProcessorsPath
(
io.objectPath(),
false // No additional parallel synchronisation
io.objectPath()
)
);

2
src/OpenFOAM/global/fileOperations/uncollatedFileOperation/uncollatedFileOperation.H
View File

@ -85,7 +85,7 @@ protected:
// \return empty fileName if not found.
virtual refPtr<dirIndexList> lookupProcessorsPath
(
const fileName&
const fileName& objectPath
) const;

34
src/finiteVolume/fvMesh/fvMeshTools/fvMeshTools.C
View File

@ -760,12 +760,19 @@ Foam::fvMeshTools::loadOrCreateMeshImpl
const label oldNumProcs = fileHandler().nProcs();
const int oldCache = fileOperation::cacheLevel(0);
const fileName facesInstance = io.time().findInstance
(
meshSubDir,
"faces",
IOobject::MUST_READ
);
patchEntries = polyBoundaryMeshEntries
(
IOobject
(
"boundary",
io.instance(),
facesInstance,
meshSubDir,
io.db(),
IOobject::MUST_READ,
@ -794,25 +801,6 @@ Foam::fvMeshTools::loadOrCreateMeshImpl
// Non-null reference when a mesh exists on given processor
haveLocalMesh = (*readHandlerPtr).good();
}
else
{
// No file handler.
// Check for presence of the "faces" file,
// but for 'decompose', only need mesh on master.
haveLocalMesh =
(
decompose
? UPstream::master()
: fileHandler().isFile
(
fileHandler().filePath
(
io.time().path()/io.instance()/meshSubDir/"faces"
)
)
);
}
// Globally consistent information about who has a mesh
boolList haveMesh
@ -820,6 +808,11 @@ Foam::fvMeshTools::loadOrCreateMeshImpl
UPstream::allGatherValues<bool>(haveLocalMesh)
);
// Make sure all have the same number of processors
label masterNProcs = fileHandler().nProcs();
Pstream::broadcast(masterNProcs);
const_cast<fileOperation&>(fileHandler()).nProcs(masterNProcs);
autoPtr<fvMesh> meshPtr;
@ -960,6 +953,7 @@ Foam::fvMeshTools::loadOrCreateMeshImpl
)
)
{
const_cast<fileOperation&>(fileHandler()).nProcs(UPstream::nProcs(oldWorldComm));
// Can use the handler communicator as is.
UPstream::commWorld(fileHandler().comm());
}

1
src/finiteVolume/fvMesh/fvMeshTools/fvMeshTools.H
View File

@ -220,6 +220,7 @@ public:
const fvMesh& procMesh,
const mapDistributePolyMesh& map,
const bool decompose,
const fileName& writeHandlerInstance,
refPtr<fileOperation>& writeHandler
);
};

51
src/finiteVolume/fvMesh/fvMeshTools/fvMeshToolsProcAddr.C
View File

@ -303,6 +303,7 @@ void Foam::fvMeshTools::writeProcAddressing
const fvMesh& mesh,
const mapDistributePolyMesh& map,
const bool decompose,
const fileName& writeHandlerInstance,
refPtr<fileOperation>& writeHandler
)
{
@ -317,7 +318,7 @@ void Foam::fvMeshTools::writeProcAddressing
// been done independently (as a registered object)
IOobject ioAddr
(
"procAddressing",
"proc-addressing",
mesh.facesInstance(),
polyMesh::meshSubDir,
mesh.thisDb(),
@ -421,23 +422,43 @@ void Foam::fvMeshTools::writeProcAddressing
);
}
auto oldHandler = fileOperation::fileHandler(writeHandler);
const bool cellOk = cellMap.write();
const bool faceOk = faceMap.write();
const bool pointOk = pointMap.write();
const bool patchOk = patchMap.write();
// Switch to using the correct
// - fileHandler
// - instance
// to write to the original mesh/time in the original format. Clunky!
// Bypass regIOobject writing to avoid taking over the current time
// as instance so instead of e.g. 'celllMap.write()' directly call
// the chosen file-handler.
writeHandler = fileOperation::fileHandler(oldHandler);
if (!cellOk || !faceOk || !pointOk || !patchOk)
const auto& tm = cellMap.time();
const IOstreamOption opt(tm.writeFormat(), tm.writeCompression());
{
WarningInFunction
<< "Failed to write some of "
<< cellMap.objectRelPath() << ", "
<< faceMap.objectRelPath() << ", "
<< pointMap.objectRelPath() << ", "
<< patchMap.objectRelPath() << endl;
auto oldHandler = fileOperation::fileHandler(writeHandler);
cellMap.instance() = writeHandlerInstance;
const bool cellOk = fileHandler().writeObject(cellMap, opt, true);
faceMap.instance() = writeHandlerInstance;
const bool faceOk = fileHandler().writeObject(faceMap, opt, true);
pointMap.instance() = writeHandlerInstance;
const bool pointOk = fileHandler().writeObject(pointMap, opt, true);
patchMap.instance() = writeHandlerInstance;
const bool patchOk = fileHandler().writeObject(patchMap, opt, true);
writeHandler = fileOperation::fileHandler(oldHandler);
if (!cellOk || !faceOk || !pointOk || !patchOk)
{
WarningInFunction
<< "Failed to write some of "
<< cellMap.objectRelPath() << ", "
<< faceMap.objectRelPath() << ", "
<< pointMap.objectRelPath() << ", "
<< patchMap.objectRelPath() << endl;
}
}
}

3
tutorials/IO/fileHandler/Allrun
View File

@ -21,11 +21,14 @@ application=$(getApplication)
#- Test writing collated format
runApplication decomposePar -fileHandler collated
#runParallel redistributePar -decompose -fileHandler collated
runParallel $application -fileHandler collated
runApplication reconstructPar -latestTime -fileHandler collated
#runParallel -s reconstruct redistributePar -reconstruct -latestTime -fileHandler collated
#- Test writing uncollated format
runApplication -s uncollated decomposePar -fileHandler uncollated -force
#runParallel -s uncollated redistributePar -decompose -fileHandler uncollated
runParallel -s uncollated $application -fileHandler uncollated

5
tutorials/IO/fileHandler/Test-distributed
View File

@ -46,7 +46,10 @@ CASE_ROOTS
#export FOAM_ABORT=true
runParallel -s decompose redistributePar -decompose -case test-distribute/machineA/testcase
runParallel -s decompose redistributePar \
-decompose \
-case test-distribute/machineA/testcase \
-fileHandler masterUncollated
runParallel checkMesh -case test-distribute/machineA/testcase

41
tutorials/IO/fileHandler_dynamicCode/0/U Normal file
View File

@ -0,0 +1,41 @@
/*--------------------------------*- C++ -*----------------------------------*\
| ========= | |
| \\ / F ield | OpenFOAM: The Open Source CFD Toolbox |
| \\ / O peration | Version: v2312 |
| \\ / A nd | Website: www.openfoam.com |
| \\/ M anipulation | |
\*---------------------------------------------------------------------------*/
FoamFile
{
version 2.0;
format ascii;
class volVectorField;
object U;
}
// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
dimensions [0 1 -1 0 0 0 0];
internalField uniform (0 0 0);
boundaryField
{
movingWall
{
type fixedValue;
value uniform (1 0 0);
}
fixedWalls
{
type noSlip;
}
frontAndBack
{
type empty;
}
}
// ************************************************************************* //

40
tutorials/IO/fileHandler_dynamicCode/0/p Normal file
View File

@ -0,0 +1,40 @@
/*--------------------------------*- C++ -*----------------------------------*\
| ========= | |
| \\ / F ield | OpenFOAM: The Open Source CFD Toolbox |
| \\ / O peration | Version: v2312 |
| \\ / A nd | Website: www.openfoam.com |
| \\/ M anipulation | |
\*---------------------------------------------------------------------------*/
FoamFile
{
version 2.0;
format ascii;
class volScalarField;
object p;
}
// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
dimensions [0 2 -2 0 0 0 0];
internalField uniform 0;
boundaryField
{
movingWall
{
type zeroGradient;
}
fixedWalls
{
type zeroGradient;
}
frontAndBack
{
type empty;
}
}
// ************************************************************************* //

20
tutorials/IO/fileHandler_dynamicCode/constant/transportProperties Normal file
View File

@ -0,0 +1,20 @@
/*--------------------------------*- C++ -*----------------------------------*\
| ========= | |
| \\ / F ield | OpenFOAM: The Open Source CFD Toolbox |
| \\ / O peration | Version: v2312 |
| \\ / A nd | Website: www.openfoam.com |
| \\/ M anipulation | |
\*---------------------------------------------------------------------------*/
FoamFile
{
version 2.0;
format ascii;
class dictionary;
object transportProperties;
}
// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
nu 0.01;
// ************************************************************************* //

72
tutorials/IO/fileHandler_dynamicCode/system/blockMeshDict Normal file
View File

@ -0,0 +1,72 @@
/*--------------------------------*- C++ -*----------------------------------*\
| ========= | |
| \\ / F ield | OpenFOAM: The Open Source CFD Toolbox |
| \\ / O peration | Version: v2312 |
| \\ / A nd | Website: www.openfoam.com |
| \\/ M anipulation | |
\*---------------------------------------------------------------------------*/
FoamFile
{
version 2.0;
format ascii;
class dictionary;
object blockMeshDict;
}
// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
scale 0.1;
vertices
(
(0 0 0)
(1 0 0)
(1 1 0)
(0 1 0)
(0 0 0.1)
(1 0 0.1)
(1 1 0.1)
(0 1 0.1)
);
blocks
(
hex (0 1 2 3 4 5 6 7) (5 5 1) simpleGrading (1 1 1)
);
edges
(
);
boundary
(
movingWall
{
type wall;
faces
(
(3 7 6 2)
);
}
fixedWalls
{
type wall;
faces
(
(0 4 7 3)
(2 6 5 1)
(1 5 4 0)
);
}
frontAndBack
{
type empty;
faces
(
(0 3 2 1)
(4 5 6 7)
);
}
);
// ************************************************************************* //

111
tutorials/IO/fileHandler_dynamicCode/system/controlDict Normal file
View File

@ -0,0 +1,111 @@
/*--------------------------------*- C++ -*----------------------------------*\
| ========= | |
| \\ / F ield | OpenFOAM: The Open Source CFD Toolbox |
| \\ / O peration | Version: v2312 |
| \\ / A nd | Website: www.openfoam.com |
| \\/ M anipulation | |
\*---------------------------------------------------------------------------*/
FoamFile
{
version 2.0;
format ascii;
class dictionary;
object controlDict;
}
// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
DebugSwitches
{
// OSspecific 1;
// POSIX 2;
// codeStream 1;
// codedBase 1;
// masterUncollated 1;
// hostCollated 1;
// collated 1;
// fileMonitor 1;
}
OptimisationSwitches
{
fileModificationSkew 1;
}
//- Set deltaT
#include "solverControls"
application icoFoam;
startFrom startTime;
startTime 0;
stopAt endTime;
//stopAt #codeStream
//{
// code
// #{
// os << "nextWrite;" << nl;
// #};
//};
endTime 0.04;
//deltaT 0.005;
writeControl timeStep;
writeInterval 20;
purgeWrite 0;
writeFormat ascii;
writePrecision 6;
writeCompression off;
timeFormat general;
timePrecision 6;
runTimeModifiable true;
functions
{
fileUpdate
{
type timeActivatedFileUpdate;
libs (utilityFunctionObjects);
writeControl timeStep;
writeInterval 1;
fileToUpdate "<system>/solverControls";
timeVsFile
(
(-1 "<system>/solverControls.0" )
( 0.02 "<system>/solverControls.5" )
);
}
// Wait a bit to make runTimeModifiable work.
// - see also fileModificationSkew
// - could also directly use system FO
sleep
{
type coded;
libs (utilityFunctionObjects);
name sleep;
writeControl timeStep;
codeExecute
#{
sleep(1);
#};
}
}
// ************************************************************************* //

27
tutorials/IO/fileHandler_dynamicCode/system/decomposeParDict Normal file
View File

@ -0,0 +1,27 @@
/*--------------------------------*- C++ -*----------------------------------*\
| ========= | |
| \\ / F ield | OpenFOAM: The Open Source CFD Toolbox |
| \\ / O peration | Version: v2312 |
| \\ / A nd | Website: www.openfoam.com |
| \\/ M anipulation | |
\*---------------------------------------------------------------------------*/
FoamFile
{
version 2.0;
format ascii;
class dictionary;
object decomposeParDict;
}
// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
numberOfSubdomains 5;
method hierarchical;
coeffs
{
n (5 1 1);
}
// ************************************************************************* //

50
tutorials/IO/fileHandler_dynamicCode/system/fvSchemes Normal file
View File

@ -0,0 +1,50 @@
/*--------------------------------*- C++ -*----------------------------------*\
| ========= | |
| \\ / F ield | OpenFOAM: The Open Source CFD Toolbox |
| \\ / O peration | Version: v2312 |
| \\ / A nd | Website: www.openfoam.com |
| \\/ M anipulation | |
\*---------------------------------------------------------------------------*/
FoamFile
{
version 2.0;
format ascii;
class dictionary;
object fvSchemes;
}
// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
ddtSchemes
{
default Euler;
}
gradSchemes
{
default Gauss linear;
grad(p) Gauss linear;
}
divSchemes
{
default none;
div(phi,U) Gauss linear;
}
laplacianSchemes
{
default Gauss linear orthogonal;
}
interpolationSchemes
{
default linear;
}
snGradSchemes
{
default orthogonal;
}
// ************************************************************************* //

51
tutorials/IO/fileHandler_dynamicCode/system/fvSolution Normal file
View File

@ -0,0 +1,51 @@
/*--------------------------------*- C++ -*----------------------------------*\
| ========= | |
| \\ / F ield | OpenFOAM: The Open Source CFD Toolbox |
| \\ / O peration | Version: v2312 |
| \\ / A nd | Website: www.openfoam.com |
| \\/ M anipulation | |
\*---------------------------------------------------------------------------*/
FoamFile
{
version 2.0;
format ascii;
class dictionary;
object fvSolution;
}
// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
solvers
{
p
{
solver PCG;
preconditioner DIC;
tolerance 1e-06;
relTol 0.05;
}
pFinal
{
$p;
relTol 0;
}
U
{
solver smoothSolver;
smoother symGaussSeidel;
tolerance 1e-05;
relTol 0;
}
}
PISO
{
nCorrectors 2;
nNonOrthogonalCorrectors 0;
pRefCell 0;
pRefValue 0;
}
// ************************************************************************* //

11
tutorials/IO/fileHandler_dynamicCode/system/solverControls Normal file
View File

@ -0,0 +1,11 @@
/*--------------------------------*- C++ -*----------------------------------*\
| ========= | |
| \\ / F ield | OpenFOAM: The Open Source CFD Toolbox |
| \\ / O peration | Version: v2312 |
| \\ / A nd | Website: www.openfoam.com |
| \\/ M anipulation | |
\*---------------------------------------------------------------------------*/
deltaT 0.001;
// ************************************************************************* //

11
tutorials/IO/fileHandler_dynamicCode/system/solverControls.0 Normal file
View File

@ -0,0 +1,11 @@
/*--------------------------------*- C++ -*----------------------------------*\
| ========= | |
| \\ / F ield | OpenFOAM: The Open Source CFD Toolbox |
| \\ / O peration | Version: v2312 |
| \\ / A nd | Website: www.openfoam.com |
| \\/ M anipulation | |
\*---------------------------------------------------------------------------*/
deltaT 0.005;
// ************************************************************************* //

11
tutorials/IO/fileHandler_dynamicCode/system/solverControls.5 Normal file
View File

@ -0,0 +1,11 @@
/*--------------------------------*- C++ -*----------------------------------*\
| ========= | |
| \\ / F ield | OpenFOAM: The Open Source CFD Toolbox |
| \\ / O peration | Version: v2312 |
| \\ / A nd | Website: www.openfoam.com |
| \\/ M anipulation | |
\*---------------------------------------------------------------------------*/
deltaT 0.001;
// ************************************************************************* //

4
tutorials/heatTransfer/buoyantBoussinesqSimpleFoam/iglooWithFridges/system/decomposeParDict
View File

@ -1,7 +1,7 @@
/*--------------------------------*- C++ -*----------------------------------*\
| ========= | |
| \\ / F ield | OpenFOAM: The Open Source CFD Toolbox |
| \\ / O peration | Version: v2306 |
| \\ / O peration | Version: v2312 |
| \\ / A nd | Website: www.openfoam.com |
| \\/ M anipulation | |
\*---------------------------------------------------------------------------*/
@ -14,8 +14,6 @@ FoamFile
}
// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
numberOfSubdomains 2;
method scotch;