ENH: provide extended fileOperation::detectProcessorPath()

- returns collated grouping as an output parameter
This commit is contained in:
Mark Olesen
2025-08-28 18:26:19 +02:00
parent bd57627955
commit 4c13fd8658
8 changed files with 87 additions and 69 deletions

View File

@ -1,3 +1,3 @@
Test-fileOperation1.C Test-fileOperation1.cxx
EXE = $(FOAM_USER_APPBIN)/Test-fileOperation1 EXE = $(FOAM_USER_APPBIN)/Test-fileOperation1

View File

@ -42,7 +42,8 @@ Description
using namespace Foam; using namespace Foam;
word toString(const fileOperation::procRangeType& group) template<class IntType>
word toString(const IntRange<IntType>& group)
{ {
if (group.empty()) if (group.empty())
{ {

View File

@ -6,7 +6,7 @@
\\/ M anipulation | \\/ M anipulation |
------------------------------------------------------------------------------- -------------------------------------------------------------------------------
Copyright (C) 2012-2017 OpenFOAM Foundation Copyright (C) 2012-2017 OpenFOAM Foundation
Copyright (C) 2015-2024 OpenCFD Ltd. Copyright (C) 2015-2025 OpenCFD Ltd.
------------------------------------------------------------------------------- -------------------------------------------------------------------------------
License License
This file is part of OpenFOAM. This file is part of OpenFOAM.
@ -78,11 +78,9 @@ bool Foam::checkFileExistence(const fileName& fName)
for (const fileName& dirN : dirEntries) for (const fileName& dirN : dirEntries)
{ {
// Analyse directory name // Analyse directory name
fileName rp, rd, rl; label rNum(-1);
label rNum;
const label readProci = const label readProci =
fileOperation::splitProcessorPath fileOperation::detectProcessorPath(dirN, group, &rNum);
(dirN, rp, rd, rl, group, rNum);
if (proci == readProci) if (proci == readProci)
{ {
@ -211,18 +209,8 @@ Foam::boolList Foam::haveMeshFile
// Collect local block number // Collect local block number
label myBlockNumber = -1; label myBlockNumber = -1;
{ {
fileName path, pDir, local;
procRangeType group; procRangeType group;
label numProcs; label proci = fileOperation::detectProcessorPath(fName, group);
label proci = fileOperation::splitProcessorPath
(
fName,
path,
pDir,
local,
group,
numProcs
);
if (proci == -1 && group.empty()) if (proci == -1 && group.empty())
{ {

View File

@ -6,7 +6,7 @@
\\/ M anipulation | \\/ M anipulation |
------------------------------------------------------------------------------- -------------------------------------------------------------------------------
Copyright (C) 2017-2018 OpenFOAM Foundation Copyright (C) 2017-2018 OpenFOAM Foundation
Copyright (C) 2020-2023 OpenCFD Ltd. Copyright (C) 2020-2025 OpenCFD Ltd.
------------------------------------------------------------------------------- -------------------------------------------------------------------------------
License License
This file is part of OpenFOAM. This file is part of OpenFOAM.
@ -154,7 +154,7 @@ bool Foam::fileOperations::collatedFileOperation::appendObject
{ {
// Append to processorsNN/ file // Append to processorsNN/ file
const label proci = detectProcessorPath(io.objectPath()); const label proci = fileOperation::detectProcessorPath(io.objectPath());
if (debug) if (debug)
{ {
@ -544,7 +544,7 @@ Foam::word Foam::fileOperations::collatedFileOperation::processorsDir
{ {
if (UPstream::parRun()) if (UPstream::parRun())
{ {
const List<int>& procs(UPstream::procID(comm_)); const auto& procs = UPstream::procID(comm_);
word procDir(processorsBaseDir+Foam::name(nProcs_)); word procDir(processorsBaseDir+Foam::name(nProcs_));
@ -565,7 +565,7 @@ Foam::word Foam::fileOperations::collatedFileOperation::processorsDir
if (ioRanks_.size()) if (ioRanks_.size())
{ {
// Detect current processor number // Detect current processor number
label proci = detectProcessorPath(fName); label proci = fileOperation::detectProcessorPath(fName);
if (proci != -1) if (proci != -1)
{ {

View File

@ -6,7 +6,7 @@
\\/ M anipulation | \\/ M anipulation |
------------------------------------------------------------------------------- -------------------------------------------------------------------------------
Copyright (C) 2017-2018 OpenFOAM Foundation Copyright (C) 2017-2018 OpenFOAM Foundation
Copyright (C) 2019-2024 OpenCFD Ltd. Copyright (C) 2019-2025 OpenCFD Ltd.
------------------------------------------------------------------------------- -------------------------------------------------------------------------------
License License
This file is part of OpenFOAM. This file is part of OpenFOAM.
@ -524,12 +524,11 @@ Foam::fileOperation::lookupAndCacheProcessorsPath
for (const fileName& dirN : dirEntries) for (const fileName& dirN : dirEntries)
{ {
// Analyse directory name // Analyse directory name
fileName rp, rd, rl; label rNum(-1);
label rNum;
const label readProci = const label readProci =
splitProcessorPath(dirN, rp, rd, rl, group, rNum); fileOperation::detectProcessorPath(dirN, group, &rNum);
nProcs = max(nProcs, readProci+1); nProcs = Foam::max(nProcs, readProci+1);
Tuple2<pathType, int> pathTypeIdx(pathType::NOTFOUND, 0); Tuple2<pathType, int> pathTypeIdx(pathType::NOTFOUND, 0);
@ -554,7 +553,7 @@ Foam::fileOperation::lookupAndCacheProcessorsPath
} }
// "processorsNN" or "processorsNN_start-end" // "processorsNN" or "processorsNN_start-end"
nProcs = max(nProcs, rNum); nProcs = Foam::max(nProcs, rNum);
if (group.empty()) if (group.empty())
{ {
@ -581,7 +580,7 @@ Foam::fileOperation::lookupAndCacheProcessorsPath
if (pathTypeIdx.first() != pathType::NOTFOUND) if (pathTypeIdx.first() != pathType::NOTFOUND)
{ {
procDirs.append(dirIndex(dirN, pathTypeIdx)); procDirs.emplace_back(dirN, pathTypeIdx);
} }
} }
@ -621,7 +620,7 @@ Foam::fileOperation::lookupAndCacheProcessorsPath
int flavour(pathType::PROCUNCOLLATED); int flavour(pathType::PROCUNCOLLATED);
for (const dirIndex& pDir : procDirs) for (const dirIndex& pDir : procDirs)
{ {
flavour = max(flavour, int(pDir.second().first())); flavour = Foam::max(flavour, int(pDir.second().first()));
} }
reduce(nProcs, maxOp<label>()); // worldComm reduce(nProcs, maxOp<label>()); // worldComm
@ -640,13 +639,10 @@ Foam::fileOperation::lookupAndCacheProcessorsPath
{ {
pathTypeIdx.second() = proci; pathTypeIdx.second() = proci;
procDirs.append procDirs.emplace_back
(
dirIndex
( (
processorsBaseDir + Foam::name(nProcs), processorsBaseDir + Foam::name(nProcs),
pathTypeIdx pathTypeIdx
)
); );
} }
else else
@ -655,13 +651,10 @@ Foam::fileOperation::lookupAndCacheProcessorsPath
// - poor fallback for pathType::PROCOBJECT // - poor fallback for pathType::PROCOBJECT
// - out-of-range pathType::PROCBASEOBJECT // - out-of-range pathType::PROCBASEOBJECT
procDirs.append procDirs.emplace_back
(
dirIndex
( (
"processor" + Foam::name(proci), "processor" + Foam::name(proci),
pathTypeIdx pathTypeIdx
)
); );
} }
@ -1342,12 +1335,12 @@ Foam::label Foam::fileOperation::nProcs
const label readProci = const label readProci =
splitProcessorPath(dirN, rp, rd, rl, group, rNum); splitProcessorPath(dirN, rp, rd, rl, group, rNum);
maxProc = max(maxProc, readProci); maxProc = Foam::max(maxProc, readProci);
if (rNum > 0) // processorsDDD where DDD>0 if (rNum > 0) // processorsDDD where DDD>0
{ {
// Use processors number // Use processors number
maxProc = max(maxProc, rNum-1); maxProc = Foam::max(maxProc, rNum-1);
// Mark in cache. TBD separate handling for groups? // Mark in cache. TBD separate handling for groups?
foundDirs.set(labelRange(0, rNum)); foundDirs.set(labelRange(0, rNum));
@ -1632,12 +1625,39 @@ Foam::label Foam::fileOperation::splitProcessorPath
} }
Foam::label Foam::fileOperation::detectProcessorPath
(
const fileName& objPath,
procRangeType& group,
label* numProcs
)
{
fileName path, procDir, local;
label nProcs;
label proci = fileOperation::splitProcessorPath
(
objPath,
path,
procDir,
local,
group,
nProcs
);
if (numProcs)
{
*numProcs = nProcs;
}
return proci;
}
Foam::label Foam::fileOperation::detectProcessorPath(const fileName& fName) Foam::label Foam::fileOperation::detectProcessorPath(const fileName& fName)
{ {
fileName path, pDir, local;
procRangeType group; procRangeType group;
label nProcs; return fileOperation::detectProcessorPath(fName, group);
return splitProcessorPath(fName, path, pDir, local, group, nProcs);
} }

View File

@ -6,7 +6,7 @@
\\/ M anipulation | \\/ M anipulation |
------------------------------------------------------------------------------- -------------------------------------------------------------------------------
Copyright (C) 2017 OpenFOAM Foundation Copyright (C) 2017 OpenFOAM Foundation
Copyright (C) 2020-2024 OpenCFD Ltd. Copyright (C) 2020-2025 OpenCFD Ltd.
------------------------------------------------------------------------------- -------------------------------------------------------------------------------
License License
This file is part of OpenFOAM. This file is part of OpenFOAM.
@ -138,9 +138,8 @@ public:
typedef Tuple2<fileName, Tuple2<pathType, int>> dirIndex; typedef Tuple2<fileName, Tuple2<pathType, int>> dirIndex;
typedef List<dirIndex> dirIndexList; typedef List<dirIndex> dirIndexList;
//- For addressing a range of processors, //- For addressing a range of processors (an int range)
//- identical to UPstream::rangeType typedef UPstream::rangeType procRangeType;
typedef IntRange<int> procRangeType;
private: private:
@ -933,16 +932,32 @@ public:
static label splitProcessorPath static label splitProcessorPath
( (
const fileName& objectPath, const fileName& objectPath,
//! [out] the leading part of the path
fileName& path, fileName& path,
//! [out] the processor directory name
fileName& procDir, fileName& procDir,
//! [out] the local path
fileName& local, fileName& local,
//! [out] any detected collated grouping
procRangeType& group, procRangeType& group,
//! [out] detected number of processors, -1 if none detected
label& nProcs label& nProcs
); );
//- Detect processor number from '/aa/bb/processorDDD/cc' //- Detect processor number from '/aa/bb/processorDDD/cc'
static label detectProcessorPath(const fileName& objPath); static label detectProcessorPath(const fileName& objPath);
//- Detect processor number from 'path/processorDDD/abc'
//- or 'path/processorsNN/abc', 'path/processorsNN_0-10/abc'
static label detectProcessorPath
(
const fileName& objPath,
//! [out] any detected collated grouping
procRangeType& group,
//! [out] detected number of collated processors, -1 if none
label* numProcs = nullptr
);
// Rank selection/sub-selection // Rank selection/sub-selection

View File

@ -1984,17 +1984,15 @@ Foam::fileOperations::masterUncollatedFileOperation::readStream
// Note: this should really be part of filePath() which should return // Note: this should really be part of filePath() which should return
// both file and index in file. // both file and index in file.
fileName path, procDir, local;
procRangeType group; procRangeType group;
label nProcs; fileOperation::detectProcessorPath(fName, group);
splitProcessorPath(fName, path, procDir, local, group, nProcs);
if (!UPstream::parRun()) if (!UPstream::parRun())
{ {
// Analyse the objectpath to find out the processor we're trying // Analyse the objectpath to find out the processor we're trying
// to access // to access
label proci = detectProcessorPath(io.objectPath()); label proci = fileOperation::detectProcessorPath(io.objectPath());
if (proci == -1) if (proci == -1)
{ {
@ -2009,7 +2007,7 @@ Foam::fileOperations::masterUncollatedFileOperation::readStream
// The local rank (offset) // The local rank (offset)
if (!group.empty()) if (!group.empty())
{ {
proci = proci - group.start(); proci -= group.start();
} }
if (debug) if (debug)
@ -2053,7 +2051,7 @@ Foam::fileOperations::masterUncollatedFileOperation::readStream
// Get size of file to determine communications type // Get size of file to determine communications type
bool bigSize = false; bool bigSize = false;
if (Pstream::master(UPstream::worldComm)) if (UPstream::master(UPstream::worldComm))
{ {
// TBD: handle multiple masters? // TBD: handle multiple masters?
bigSize = bigSize =
@ -2064,7 +2062,7 @@ Foam::fileOperations::masterUncollatedFileOperation::readStream
} }
// Reduce (not broadcast) // Reduce (not broadcast)
// - if we have multiple master files (FUTURE) // - if we have multiple master files (FUTURE)
Pstream::reduceOr(bigSize, UPstream::worldComm); UPstream::reduceOr(bigSize, UPstream::worldComm);
const UPstream::commsTypes myCommsType const UPstream::commsTypes myCommsType
( (
@ -2610,12 +2608,10 @@ void Foam::fileOperations::masterUncollatedFileOperation::sync()
if (Pstream::parRun() && !Pstream::master(UPstream::worldComm)) if (Pstream::parRun() && !Pstream::master(UPstream::worldComm))
{ {
// Replace processor0 ending with processorDDD // Replace processor0 ending with processorDDD
fileName path; fileName path, pDir, local;
fileName pDir;
fileName local;
procRangeType group; procRangeType group;
label numProcs; label numProcs;
const label proci = splitProcessorPath const label proci = fileOperation::splitProcessorPath
( (
dir, dir,
path, path,

View File

@ -6,7 +6,7 @@
\\/ M anipulation | \\/ M anipulation |
------------------------------------------------------------------------------- -------------------------------------------------------------------------------
Copyright (C) 2017 OpenFOAM Foundation Copyright (C) 2017 OpenFOAM Foundation
Copyright (C) 2020-2024 OpenCFD Ltd. Copyright (C) 2020-2025 OpenCFD Ltd.
------------------------------------------------------------------------------- -------------------------------------------------------------------------------
License License
This file is part of OpenFOAM. This file is part of OpenFOAM.
@ -661,7 +661,7 @@ Foam::fileOperations::uncollatedFileOperation::readStream
{ {
// Analyse the objectpath to find out the processor we're trying // Analyse the objectpath to find out the processor we're trying
// to access // to access
label proci = detectProcessorPath(io.objectPath()); label proci = fileOperation::detectProcessorPath(io.objectPath());
if (proci == -1) if (proci == -1)
{ {
@ -675,15 +675,13 @@ Foam::fileOperations::uncollatedFileOperation::readStream
// Analyse the fileName for any processor subset. Note: this // Analyse the fileName for any processor subset. Note: this
// should really be part of filePath() which should return // should really be part of filePath() which should return
// both file and index in file. // both file and index in file.
fileName path, procDir, local;
procRangeType group; procRangeType group;
label nProcs; fileOperation::detectProcessorPath(fName, group);
splitProcessorPath(fName, path, procDir, local, group, nProcs);
// The local rank (offset) // The local rank (offset)
if (!group.empty()) if (!group.empty())
{ {
proci = proci - group.start(); proci -= group.start();
} }
// Read data and return as stream // Read data and return as stream