mirror of
https://develop.openfoam.com/Development/openfoam.git
synced 2025-11-28 03:28:01 +00:00
ENH: robuster fileOperations splitProcessorPath
- robuster matching behaviour when encountering paths that themselves
contain the word "processor" in them. For example,
"/path/processor0generation2/case1/processor10/system"
will now correctly match on processor10 instead of failing.
- use procRangeType for encapsulating the processor ranges
- provision for information of distributed vs non-distributed roots.
The information is currently available from the initial setup, but
can useful to access directly within fileOperation.
STYLE: modernize list iteration
This commit is contained in:
Mark Olesen
3
applications/test/fileOperation1/Make/files
Normal file
3
applications/test/fileOperation1/Make/files
Normal file
@ -0,0 +1,3 @@
|
||||
Test-fileOperation1.C
|
||||
|
||||
EXE = $(FOAM_USER_APPBIN)/Test-fileOperation1
|
||||
2
applications/test/fileOperation1/Make/options
Normal file
2
applications/test/fileOperation1/Make/options
Normal file
@ -0,0 +1,2 @@
|
||||
/* EXE_INC = */
|
||||
/* EXE_LIBS = */
|
||||
125
applications/test/fileOperation1/Test-fileOperation1.C
Normal file
125
applications/test/fileOperation1/Test-fileOperation1.C
Normal file
@ -0,0 +1,125 @@
|
||||
/*---------------------------------------------------------------------------*\
|
||||
========= |
|
||||
\\ / F ield | OpenFOAM: The Open Source CFD Toolbox
|
||||
\\ / O peration |
|
||||
\\ / A nd | www.openfoam.com
|
||||
\\/ M anipulation |
|
||||
-------------------------------------------------------------------------------
|
||||
Copyright (C) 2020 OpenCFD Ltd.
|
||||
-------------------------------------------------------------------------------
|
||||
License
|
||||
This file is part of OpenFOAM.
|
||||
|
||||
OpenFOAM is free software: you can redistribute it and/or modify it
|
||||
under the terms of the GNU General Public License as published by
|
||||
the Free Software Foundation, either version 3 of the License, or
|
||||
(at your option) any later version.
|
||||
|
||||
OpenFOAM is distributed in the hope that it will be useful, but WITHOUT
|
||||
ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
|
||||
FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
|
||||
for more details.
|
||||
|
||||
You should have received a copy of the GNU General Public License
|
||||
along with OpenFOAM. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
Application
|
||||
Test-fileOperation1
|
||||
|
||||
Description
|
||||
Test string parsing and other bits for fileOperation
|
||||
|
||||
\*---------------------------------------------------------------------------*/
|
||||
|
||||
#include "argList.H"
|
||||
#include "fileName.H"
|
||||
#include "fileOperation.H"
|
||||
#include "SubList.H"
|
||||
#include "IOobject.H"
|
||||
#include "IOstreams.H"
|
||||
#include "OSspecific.H"
|
||||
|
||||
|
||||
using namespace Foam;
|
||||
|
||||
word toString(const fileOperation::procRangeType& group)
|
||||
{
|
||||
if (group.empty())
|
||||
{
|
||||
return word::null;
|
||||
}
|
||||
return Foam::name(group.first()) + "-" + Foam::name(group.last());
|
||||
}
|
||||
|
||||
|
||||
void testSplitPath(const fileName& pathName)
|
||||
{
|
||||
fileName path, procDir, local;
|
||||
fileOperation::procRangeType group;
|
||||
label nProcs;
|
||||
|
||||
const label proci =
|
||||
fileOperation::splitProcessorPath
|
||||
(
|
||||
pathName,
|
||||
path,
|
||||
procDir,
|
||||
local,
|
||||
group,
|
||||
nProcs
|
||||
);
|
||||
|
||||
|
||||
Info<< nl
|
||||
<< "Input = " << pathName << nl
|
||||
<< " path = " << path << nl
|
||||
<< " proc = " << procDir << nl
|
||||
<< " local = " << local << nl
|
||||
<< " group = " << group << " = " << toString(group) << nl
|
||||
<< " proci = " << proci << nl
|
||||
<< " nProcs = " << nProcs << nl;
|
||||
}
|
||||
|
||||
|
||||
void testSplitPaths(std::initializer_list<const char* const> dirNames)
|
||||
{
|
||||
for (const auto& dirName : dirNames)
|
||||
{
|
||||
testSplitPath(fileName(dirName));
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
|
||||
// Main program:
|
||||
|
||||
int main(int argc, char *argv[])
|
||||
{
|
||||
argList::addArgument("fileName .. fileNameN");
|
||||
argList::addOption("istream", "file", "test Istream values");
|
||||
|
||||
|
||||
testSplitPaths
|
||||
({
|
||||
"foo/bar",
|
||||
"foo/processor5/system",
|
||||
"foo/processors100_0-5/constant",
|
||||
"foo/processors20_12-16/constant",
|
||||
"/new-processor-gen/case1/processors20",
|
||||
"/new-processor-gen/case1/processors100_0-5/constant",
|
||||
"/new-processor-gen/case1/processors/input",
|
||||
"devel/processor/ideas/processor0/system",
|
||||
|
||||
"/path/processor0Generation1/case1/processor10/input",
|
||||
|
||||
"path/processors100_ab-cd/constant",
|
||||
"path/processors100_a11-d00/constant",
|
||||
});
|
||||
|
||||
|
||||
Info<< "\nEnd\n" << endl;
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
||||
// ************************************************************************* //
|
||||
@ -66,19 +66,14 @@ using namespace Foam;
|
||||
// Many ways to name processor directories
|
||||
//
|
||||
// Uncollated | "processor0", "processor1" ...
|
||||
// Collated (old) | "processors"
|
||||
// Collated (new) | "processors<N>"
|
||||
// Collated | "processors<N>"
|
||||
// Host collated | "processors<N>_<low>-<high>"
|
||||
|
||||
const regExp matcher("processors?[0-9]+(_[0-9]+-[0-9]+)?");
|
||||
|
||||
bool isProcessorDir(const string& dir)
|
||||
{
|
||||
return
|
||||
(
|
||||
dir.starts_with("processor")
|
||||
&& (dir == "processors" || matcher.match(dir))
|
||||
);
|
||||
return (dir.starts_with("processor") && matcher.match(dir));
|
||||
}
|
||||
|
||||
|
||||
|
||||
@ -71,19 +71,14 @@ using namespace Foam;
|
||||
// Many ways to name processor directories
|
||||
//
|
||||
// Uncollated | "processor0", "processor1" ...
|
||||
// Collated (old) | "processors"
|
||||
// Collated (new) | "processors<N>"
|
||||
// Collated | "processors<N>"
|
||||
// Host collated | "processors<N>_<low>-<high>"
|
||||
|
||||
const regExp matcher("processors?[0-9]+(_[0-9]+-[0-9]+)?");
|
||||
|
||||
bool isProcessorDir(const string& dir)
|
||||
{
|
||||
return
|
||||
(
|
||||
dir.starts_with("processor")
|
||||
&& (dir == "processors" || matcher.match(dir))
|
||||
);
|
||||
return (dir.starts_with("processor") && matcher.match(dir));
|
||||
}
|
||||
|
||||
|
||||
@ -384,13 +379,7 @@ int main(int argc, char *argv[])
|
||||
{
|
||||
if (leadProcIdx < 0)
|
||||
{
|
||||
// Collated (old)
|
||||
leadProcIdx = procDirs.find("processors");
|
||||
}
|
||||
|
||||
if (leadProcIdx < 0)
|
||||
{
|
||||
// Collated (new)
|
||||
// Collated
|
||||
leadProcIdx = procDirs.find("processors" + Foam::name(nProcs));
|
||||
}
|
||||
|
||||
|
||||
@ -519,29 +519,26 @@ int main(int argc, char *argv[])
|
||||
{
|
||||
const fileName& d = dirs[diri];
|
||||
|
||||
// Starts with 'processors'
|
||||
if (d.find("processors") == 0)
|
||||
label proci = -1;
|
||||
|
||||
if
|
||||
(
|
||||
d.starts_with("processor")
|
||||
&&
|
||||
(
|
||||
// Collated is "processors"
|
||||
d[9] == 's'
|
||||
|
||||
// Uncollated has integer(s) after 'processor'
|
||||
|| Foam::read(d.substr(9), proci)
|
||||
)
|
||||
)
|
||||
{
|
||||
if (fileHandler().exists(d))
|
||||
{
|
||||
fileHandler().rmDir(d);
|
||||
}
|
||||
}
|
||||
|
||||
// Starts with 'processor'
|
||||
if (d.find("processor") == 0)
|
||||
{
|
||||
// Check that integer after processor
|
||||
fileName num(d.substr(9));
|
||||
label proci = -1;
|
||||
if (Foam::read(num.c_str(), proci))
|
||||
{
|
||||
if (fileHandler().exists(d))
|
||||
{
|
||||
fileHandler().rmDir(d);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
procDirsProblem = false;
|
||||
|
||||
@ -5,7 +5,7 @@
|
||||
\\ / A nd | www.openfoam.com
|
||||
\\/ M anipulation |
|
||||
-------------------------------------------------------------------------------
|
||||
Copyright (C) 2017 OpenCFD Ltd.
|
||||
Copyright (C) 2017-2020 OpenCFD Ltd.
|
||||
-------------------------------------------------------------------------------
|
||||
License
|
||||
This file is part of OpenFOAM.
|
||||
@ -86,15 +86,7 @@ int main(int argc, char *argv[])
|
||||
#include "createTime.H"
|
||||
|
||||
// Determine the processor count
|
||||
#ifdef fileOperation_H
|
||||
const label nProcs = fileHandler().nProcs(args.path());
|
||||
#else
|
||||
label nProcs = 0;
|
||||
while (isDir(args.path()/("processor" + Foam::name(nProcs))))
|
||||
{
|
||||
++nProcs;
|
||||
}
|
||||
#endif
|
||||
|
||||
// Create the processor databases
|
||||
PtrList<Time> databases(nProcs);
|
||||
|
||||
@ -151,9 +151,9 @@ bool Foam::OFstreamCollator::writeFile
|
||||
if (UPstream::master(comm))
|
||||
{
|
||||
off_t sum = 0;
|
||||
forAll(recvSizes, i)
|
||||
for (const label recv : recvSizes)
|
||||
{
|
||||
sum += recvSizes[i];
|
||||
sum += recv;
|
||||
}
|
||||
// Use ostringstream to display long int (until writing these is
|
||||
// supported)
|
||||
|
||||
@ -142,24 +142,18 @@ bool Foam::fileOperations::collatedFileOperation::appendObject
|
||||
|
||||
const bool isMaster = isMasterRank(proci);
|
||||
|
||||
// Determine the local rank if the pathName is a per-rank one
|
||||
// Determine local rank (offset) if the pathName is a per-rank one
|
||||
label localProci = proci;
|
||||
{
|
||||
fileName path, procDir, local;
|
||||
label groupStart, groupSize, nProcs;
|
||||
splitProcessorPath
|
||||
(
|
||||
pathName,
|
||||
path,
|
||||
procDir,
|
||||
local,
|
||||
groupStart,
|
||||
groupSize,
|
||||
nProcs
|
||||
);
|
||||
if (groupSize > 0 && groupStart != -1)
|
||||
procRangeType group;
|
||||
label nProcs;
|
||||
splitProcessorPath(pathName, path, procDir, local, group, nProcs);
|
||||
|
||||
// The local rank (offset)
|
||||
if (!group.empty())
|
||||
{
|
||||
localProci = proci-groupStart;
|
||||
localProci = proci - group.start();
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@ -36,6 +36,8 @@ License
|
||||
#include "polyMesh.H"
|
||||
#include "registerSwitch.H"
|
||||
#include "Time.H"
|
||||
#include <cerrno>
|
||||
#include <cinttypes>
|
||||
|
||||
/* * * * * * * * * * * * * * * Static Member Data * * * * * * * * * * * * * */
|
||||
|
||||
@ -56,11 +58,6 @@ namespace Foam
|
||||
);
|
||||
}
|
||||
|
||||
|
||||
Foam::autoPtr<Foam::fileOperation> Foam::fileOperation::fileHandlerPtr_;
|
||||
|
||||
Foam::word Foam::fileOperation::processorsBaseDir = "processors";
|
||||
|
||||
const Foam::Enum<Foam::fileOperation::pathType>
|
||||
Foam::fileOperation::pathTypeNames_
|
||||
({
|
||||
@ -79,6 +76,109 @@ Foam::fileOperation::pathTypeNames_
|
||||
});
|
||||
|
||||
|
||||
Foam::word Foam::fileOperation::processorsBaseDir = "processors";
|
||||
|
||||
Foam::autoPtr<Foam::fileOperation> Foam::fileOperation::fileHandlerPtr_;
|
||||
|
||||
|
||||
// * * * * * * * * * * * * * * * Local Functions * * * * * * * * * * * * * * //
|
||||
|
||||
namespace
|
||||
{
|
||||
|
||||
// Need to parse the numbers
|
||||
// from "processors(\d+)" and
|
||||
// from "processors(\d+)_(\d+)-(\d+)"
|
||||
//
|
||||
// Receive the string matching "^(\d+)(?:_(\d+)-(\d+))?/?$"
|
||||
//
|
||||
// \1 = numProcs
|
||||
// \2 = firstProc
|
||||
// \3 = lastProc
|
||||
//
|
||||
// Return true on success and set parameters numProcs and group (size,start)
|
||||
//
|
||||
// Use low-level C-string to integer parsing to drive the sequence.
|
||||
//
|
||||
// For simplicity, also skip INT_MAX checks everywhere but check for
|
||||
// - (errno) for success
|
||||
// - (nptr == endptr) for leading junk
|
||||
// - (*endptr != endChar) for trailing junk
|
||||
// - skip INT_MAX checks as being too pessimistic
|
||||
|
||||
static bool parseProcsNumRange
|
||||
(
|
||||
const std::string str,
|
||||
int& numProcs,
|
||||
Foam::fileOperation::procRangeType& group
|
||||
)
|
||||
{
|
||||
const char * nptr = str.c_str();
|
||||
char *endptr = nullptr;
|
||||
|
||||
// 1. numProcs
|
||||
errno = 0;
|
||||
intmax_t parsed = std::strtoimax(nptr, &endptr, 10);
|
||||
if (errno || nptr == endptr) return false; // bad parse
|
||||
|
||||
const int nProcs = int(parsed);
|
||||
|
||||
// End of string? Then no range and we are done.
|
||||
if (*endptr == '\0')
|
||||
{
|
||||
numProcs = nProcs;
|
||||
return true;
|
||||
}
|
||||
|
||||
// Parse point at start of range ('_' character)?
|
||||
if (*endptr != '_') return false;
|
||||
nptr = ++endptr;
|
||||
|
||||
|
||||
// 2. firstProc
|
||||
errno = 0;
|
||||
parsed = std::strtoimax(nptr, &endptr, 10);
|
||||
if (errno || nptr == endptr) return false; // bad parse
|
||||
|
||||
const int firstProc = int(parsed);
|
||||
|
||||
// Parse point at range separator ('-' character)?
|
||||
if (*endptr != '-') return false;
|
||||
nptr = ++endptr;
|
||||
|
||||
|
||||
// 3. lastProc
|
||||
errno = 0;
|
||||
parsed = std::strtoimax(nptr, &endptr, 10);
|
||||
if (errno || nptr == endptr) return false; // bad parse
|
||||
|
||||
const int lastProc = int(parsed);
|
||||
|
||||
|
||||
if
|
||||
(
|
||||
// Parse point at end of string
|
||||
(*endptr == '\0')
|
||||
|
||||
// Input plausibility
|
||||
// Accept nProcs == 0 in case that becomes useful in the future
|
||||
&& (nProcs >= 0 && firstProc >= 0 && firstProc <= lastProc)
|
||||
)
|
||||
{
|
||||
numProcs = nProcs;
|
||||
|
||||
// Convert first/last to start/size
|
||||
group.reset(firstProc, lastProc-firstProc+1);
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
} // End anonymous namespace
|
||||
|
||||
|
||||
// * * * * * * * * * * * * * Private Member Functions * * * * * * * * * * * //
|
||||
|
||||
Foam::fileMonitor& Foam::fileOperation::monitor() const
|
||||
@ -100,17 +200,17 @@ Foam::fileMonitor& Foam::fileOperation::monitor() const
|
||||
|
||||
Foam::instantList Foam::fileOperation::sortTimes
|
||||
(
|
||||
const fileNameList& dirEntries,
|
||||
const fileNameList& dirNames,
|
||||
const word& constantName
|
||||
)
|
||||
{
|
||||
// Initialise instant list
|
||||
instantList times(dirEntries.size() + 1);
|
||||
instantList times(dirNames.size() + 1);
|
||||
label nTimes = 0;
|
||||
|
||||
// Check for "constant"
|
||||
bool haveConstant = false;
|
||||
for (const fileName& dirName : dirEntries)
|
||||
for (const fileName& dirName : dirNames)
|
||||
{
|
||||
if (dirName == constantName)
|
||||
{
|
||||
@ -123,7 +223,7 @@ Foam::instantList Foam::fileOperation::sortTimes
|
||||
}
|
||||
|
||||
// Read and parse all the entries in the directory
|
||||
for (const fileName& dirName : dirEntries)
|
||||
for (const fileName& dirName : dirNames)
|
||||
{
|
||||
scalar timeValue;
|
||||
if (readScalar(dirName, timeValue))
|
||||
@ -244,14 +344,11 @@ Foam::fileOperation::lookupAndCacheProcessorsPath
|
||||
// find the corresponding actual processor directory (e.g. 'processors4')
|
||||
// and index (2)
|
||||
|
||||
fileName path;
|
||||
fileName pDir;
|
||||
fileName local;
|
||||
label gStart;
|
||||
label gSz;
|
||||
fileName path, pDir, local;
|
||||
procRangeType group;
|
||||
label numProcs;
|
||||
label proci =
|
||||
splitProcessorPath(fName, path, pDir, local, gStart, gSz, numProcs);
|
||||
const label proci =
|
||||
splitProcessorPath(fName, path, pDir, local, group, numProcs);
|
||||
|
||||
if (proci != -1)
|
||||
{
|
||||
@ -278,15 +375,14 @@ Foam::fileOperation::lookupAndCacheProcessorsPath
|
||||
// - directory+offset containing data for proci
|
||||
label maxProc = -1;
|
||||
|
||||
forAll(dirNames, i)
|
||||
for (const fileName& dirN : dirNames)
|
||||
{
|
||||
const fileName& dirN = dirNames[i];
|
||||
|
||||
// Analyse directory name
|
||||
fileName rp, rd, rl;
|
||||
label rStart, rSize, rNum;
|
||||
label readProci =
|
||||
splitProcessorPath(dirN, rp, rd, rl, rStart, rSize, rNum);
|
||||
label rNum;
|
||||
const label readProci =
|
||||
splitProcessorPath(dirN, rp, rd, rl, group, rNum);
|
||||
|
||||
maxProc = max(maxProc, readProci);
|
||||
|
||||
if (proci == readProci)
|
||||
@ -301,16 +397,17 @@ Foam::fileOperation::lookupAndCacheProcessorsPath
|
||||
)
|
||||
);
|
||||
}
|
||||
else if (proci >= rStart && proci < rStart+rSize)
|
||||
else if (group.found(proci))
|
||||
{
|
||||
// "processorsDDD_start-end"
|
||||
// Found the file that contains the data for proci
|
||||
const label localProci = proci - group.start();
|
||||
procDirs.append
|
||||
(
|
||||
dirIndex
|
||||
(
|
||||
dirN,
|
||||
Tuple2<pathType, label>(PROCOBJECT, proci-rStart)
|
||||
Tuple2<pathType, label>(PROCOBJECT, localProci)
|
||||
)
|
||||
);
|
||||
}
|
||||
@ -319,7 +416,7 @@ Foam::fileOperation::lookupAndCacheProcessorsPath
|
||||
// Direct detection of processorsDDD
|
||||
maxProc = rNum-1;
|
||||
|
||||
if (rStart == -1)
|
||||
if (group.empty())
|
||||
{
|
||||
// "processorsDDD"
|
||||
procDirs.append
|
||||
@ -419,13 +516,19 @@ bool Foam::fileOperation::exists(IOobject& io) const
|
||||
|
||||
// * * * * * * * * * * * * * * * * Constructors * * * * * * * * * * * * * * //
|
||||
|
||||
Foam::fileOperation::fileOperation(label comm)
|
||||
Foam::fileOperation::fileOperation
|
||||
(
|
||||
const label comm,
|
||||
const bool distributedRoots
|
||||
)
|
||||
:
|
||||
comm_(comm)
|
||||
comm_(comm),
|
||||
distributed_(distributedRoots)
|
||||
{}
|
||||
|
||||
|
||||
Foam::autoPtr<Foam::fileOperation> Foam::fileOperation::New
|
||||
Foam::autoPtr<Foam::fileOperation>
|
||||
Foam::fileOperation::New
|
||||
(
|
||||
const word& handlerType,
|
||||
bool verbose
|
||||
@ -471,7 +574,7 @@ bool Foam::fileOperation::writeObject
|
||||
{
|
||||
if (valid)
|
||||
{
|
||||
fileName pathName(io.objectPath());
|
||||
const fileName pathName(io.objectPath());
|
||||
|
||||
mkDir(pathName.path());
|
||||
|
||||
@ -482,7 +585,7 @@ bool Foam::fileOperation::writeObject
|
||||
return false;
|
||||
}
|
||||
|
||||
OSstream& os = osPtr();
|
||||
OSstream& os = *osPtr;
|
||||
|
||||
// If any of these fail, return (leave error handling to Ostream class)
|
||||
if (!os.good())
|
||||
@ -514,14 +617,11 @@ Foam::fileName Foam::fileOperation::filePath(const fileName& fName) const
|
||||
Pout<< "fileOperation::filePath :" << " fName:" << fName << endl;
|
||||
}
|
||||
|
||||
fileName path;
|
||||
fileName pDir;
|
||||
fileName local;
|
||||
label gStart;
|
||||
label gSz;
|
||||
fileName path, pDir, local;
|
||||
procRangeType group;
|
||||
label numProcs;
|
||||
label proci =
|
||||
splitProcessorPath(fName, path, pDir, local, gStart, gSz, numProcs);
|
||||
splitProcessorPath(fName, path, pDir, local, group, numProcs);
|
||||
|
||||
if (numProcs != -1)
|
||||
{
|
||||
@ -533,9 +633,9 @@ Foam::fileName Foam::fileOperation::filePath(const fileName& fName) const
|
||||
{
|
||||
// Get all processor directories
|
||||
refPtr<dirIndexList> procDirs(lookupProcessorsPath(fName));
|
||||
forAll(procDirs(), i)
|
||||
for (const dirIndex& dirIdx : procDirs())
|
||||
{
|
||||
const fileName& procDir = procDirs()[i].first();
|
||||
const fileName& procDir = dirIdx.first();
|
||||
|
||||
fileName collatedName(path/procDir/local);
|
||||
if (exists(collatedName))
|
||||
@ -690,9 +790,9 @@ Foam::instantList Foam::fileOperation::findTimes
|
||||
|
||||
// Get all processor directories
|
||||
refPtr<dirIndexList> procDirs(lookupProcessorsPath(directory));
|
||||
forAll(procDirs(), i)
|
||||
for (const dirIndex& dirIdx : procDirs())
|
||||
{
|
||||
const fileName& procDir = procDirs()[i].first();
|
||||
const fileName& procDir = dirIdx.first();
|
||||
fileName collDir(processorsPath(directory, procDir));
|
||||
if (!collDir.empty() && collDir != directory)
|
||||
{
|
||||
@ -925,21 +1025,20 @@ Foam::label Foam::fileOperation::nProcs
|
||||
|
||||
// Detect any processorsDDD or processorDDD
|
||||
label maxProc = -1;
|
||||
forAll(dirNames, i)
|
||||
for (const fileName& dirN : dirNames)
|
||||
{
|
||||
const fileName& dirN = dirNames[i];
|
||||
fileName rp, rd, rl;
|
||||
procRangeType group;
|
||||
label rNum;
|
||||
|
||||
fileName path, pDir, local;
|
||||
label start, size, n;
|
||||
maxProc = max
|
||||
(
|
||||
maxProc,
|
||||
splitProcessorPath(dirN, path, pDir, local, start, size, n)
|
||||
);
|
||||
if (n != -1)
|
||||
const label readProci =
|
||||
splitProcessorPath(dirN, rp, rd, rl, group, rNum);
|
||||
|
||||
maxProc = max(maxProc, readProci);
|
||||
if (rNum != -1)
|
||||
{
|
||||
// Direct detection of processorsDDD
|
||||
maxProc = n-1;
|
||||
maxProc = rNum-1;
|
||||
break;
|
||||
}
|
||||
}
|
||||
@ -1018,12 +1117,13 @@ Foam::fileName Foam::fileOperation::processorsPath
|
||||
) const
|
||||
{
|
||||
// Check if directory is processorDDD
|
||||
word caseName(dir.name());
|
||||
|
||||
std::string::size_type pos = caseName.find("processor");
|
||||
if (pos == 0)
|
||||
const word caseName(dir.name());
|
||||
if (caseName.starts_with("processor"))
|
||||
{
|
||||
if (caseName.size() <= 9 || caseName[9] == 's')
|
||||
// Reject both '^processor$' and '^processors.*$'
|
||||
|
||||
if (!std::isdigit(caseName[9]))
|
||||
{
|
||||
WarningInFunction << "Directory " << dir
|
||||
<< " does not end in old-style processorDDD" << endl;
|
||||
@ -1038,131 +1138,172 @@ Foam::fileName Foam::fileOperation::processorsPath
|
||||
|
||||
Foam::label Foam::fileOperation::splitProcessorPath
|
||||
(
|
||||
const fileName& objectPath,
|
||||
const fileName& objPath,
|
||||
fileName& path,
|
||||
fileName& procDir,
|
||||
fileName& local,
|
||||
|
||||
label& groupStart,
|
||||
label& groupSize,
|
||||
|
||||
procRangeType& group,
|
||||
label& nProcs
|
||||
)
|
||||
{
|
||||
// Return value
|
||||
label returnProci = -1;
|
||||
|
||||
// Clear out the return parameters
|
||||
|
||||
path.clear();
|
||||
procDir.clear();
|
||||
local.clear();
|
||||
group.clear();
|
||||
|
||||
// Potentially detected start of number of processors in local group
|
||||
groupStart = -1;
|
||||
groupSize = 0;
|
||||
|
||||
// Potentially detected number of processors
|
||||
// Invalidate detected number of processors
|
||||
nProcs = -1;
|
||||
|
||||
// Search for processor at start of line or /processor
|
||||
std::string::size_type pos = objectPath.find("processor");
|
||||
if (pos == string::npos)
|
||||
// The local processor group is read as first/last, but stored as
|
||||
// start/size. Empty with start=0, size=0 if no range is detected
|
||||
|
||||
|
||||
// Start of 'processor..' directory name (the procDir)
|
||||
size_t pos = 0;
|
||||
|
||||
// The slash starting the trailing (local) directory
|
||||
size_t slashLocal = string::npos;
|
||||
|
||||
|
||||
// Search for processor at start of string or after /processor
|
||||
//
|
||||
// 'processor(\d+)'
|
||||
// 'processors(\d+)'
|
||||
// 'processors(\d+)_(\d+)-(\d+)'
|
||||
|
||||
for
|
||||
(
|
||||
/*nil*/;
|
||||
(pos = objPath.find("processor", pos)) != string::npos;
|
||||
pos += 9
|
||||
)
|
||||
{
|
||||
return -1;
|
||||
}
|
||||
|
||||
// "processorDDD"
|
||||
// "processorsNNN"
|
||||
// "processorsNNN_AA-BB"
|
||||
|
||||
|
||||
if (pos > 0 && objectPath[pos-1] != '/')
|
||||
{
|
||||
// Directory not starting with "processor" e.g. "somenamewithprocessor"
|
||||
return -1;
|
||||
}
|
||||
|
||||
procDir = objectPath;
|
||||
|
||||
// Strip leading directory
|
||||
if (pos > 0)
|
||||
{
|
||||
path = objectPath.substr(0, pos-1);
|
||||
procDir = objectPath.substr(pos);
|
||||
}
|
||||
|
||||
// Strip trailing local directory
|
||||
pos = procDir.find('/');
|
||||
if (pos != string::npos)
|
||||
{
|
||||
local = procDir.substr(pos+1);
|
||||
procDir = procDir.substr(0, pos);
|
||||
}
|
||||
|
||||
// Now procDir is e.g.
|
||||
// - processor0
|
||||
// - processors0
|
||||
// - processorBananas
|
||||
|
||||
// Look for number after "processor"
|
||||
|
||||
fileName f(procDir.substr(9));
|
||||
|
||||
if (f.size() && f[0] == 's')
|
||||
{
|
||||
// "processsorsNNN"
|
||||
|
||||
f = f.substr(1);
|
||||
|
||||
// Detect "processorsNNN_AA-BB"
|
||||
if (pos > 0 && objPath[pos-1] != '/')
|
||||
{
|
||||
std::string::size_type fromStart = f.find("_");
|
||||
std::string::size_type toStart = f.find("-");
|
||||
if (fromStart != string::npos && toStart != string::npos)
|
||||
{
|
||||
string nProcsName(f.substr(0, fromStart));
|
||||
string fromName(f.substr(fromStart+1, toStart-(fromStart+1)));
|
||||
string toName(f.substr(toStart+1));
|
||||
// Not start of string or after /processor
|
||||
continue;
|
||||
}
|
||||
|
||||
label groupEnd = -1;
|
||||
if
|
||||
// The parse point. One past 'processor'
|
||||
size_t firstp = pos + 9;
|
||||
|
||||
// normal: 'processor(\d+)'
|
||||
// plural: 'processors(\d+)'
|
||||
|
||||
const bool plural = (objPath[firstp] == 's');
|
||||
|
||||
if (plural)
|
||||
{
|
||||
++firstp; // Skip over the 's'
|
||||
}
|
||||
else if (!std::isdigit(objPath[firstp]))
|
||||
{
|
||||
// Non-plural version (uncollated) requires digits only
|
||||
continue;
|
||||
}
|
||||
|
||||
// The next slash indicates there is a local directory
|
||||
slashLocal = objPath.find('/', firstp);
|
||||
|
||||
// The last parse point is the slash, or end of string
|
||||
const size_t lastp =
|
||||
(slashLocal == string::npos ? objPath.length() : slashLocal);
|
||||
|
||||
if (!std::isdigit(objPath[lastp-1]))
|
||||
{
|
||||
// Must end in a digit!
|
||||
// This traps entries that are too short or look quite wrong
|
||||
// and avoid a string to int conversion that will fail anyhow
|
||||
continue;
|
||||
}
|
||||
|
||||
|
||||
// Match: '^processors(\d+)$' -> nProcs
|
||||
|
||||
// Match: '^processors(\d+)_(\d+)-(\d+)$'
|
||||
// \1 = nProcs
|
||||
// \2 = beg processor group
|
||||
// \3 = end processor group (inclusive)
|
||||
|
||||
if (plural)
|
||||
{
|
||||
int nProcsRead = 0;
|
||||
|
||||
if
|
||||
(
|
||||
parseProcsNumRange
|
||||
(
|
||||
Foam::read(fromName.c_str(), groupStart)
|
||||
&& Foam::read(toName.c_str(), groupEnd)
|
||||
&& Foam::read(nProcsName.c_str(), nProcs)
|
||||
objPath.substr(firstp, lastp-firstp),
|
||||
nProcsRead,
|
||||
group
|
||||
)
|
||||
{
|
||||
groupSize = groupEnd-groupStart+1;
|
||||
return -1;
|
||||
}
|
||||
)
|
||||
{
|
||||
// Total number of processors
|
||||
nProcs = nProcsRead;
|
||||
|
||||
// We are done!
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
// Detect "processorsN"
|
||||
label n;
|
||||
if (Foam::read(f.c_str(), n))
|
||||
// Single
|
||||
// Match: '^processor(\d+)$' -> proci
|
||||
|
||||
label proci = 0;
|
||||
if
|
||||
(
|
||||
Foam::read(objPath.substr(firstp, lastp-firstp), proci)
|
||||
&& (proci >= 0)
|
||||
)
|
||||
{
|
||||
nProcs = n;
|
||||
// Capture value of an individual processor
|
||||
returnProci = proci;
|
||||
|
||||
// We are done!
|
||||
break;
|
||||
}
|
||||
return -1;
|
||||
|
||||
}
|
||||
else
|
||||
|
||||
if (pos != string::npos)
|
||||
{
|
||||
// Detect "processorN"
|
||||
label proci;
|
||||
if (Foam::read(f.c_str(), proci))
|
||||
// The split succeeded, extract the components.
|
||||
|
||||
// The leading directory
|
||||
if (pos > 0)
|
||||
{
|
||||
return proci;
|
||||
path = objPath.substr(0, pos-1);
|
||||
}
|
||||
|
||||
// The slash starting the trailing (local) directory
|
||||
if (slashLocal != string::npos)
|
||||
{
|
||||
procDir = objPath.substr(pos, slashLocal-pos);
|
||||
local = objPath.substr(slashLocal+1);
|
||||
}
|
||||
else
|
||||
{
|
||||
return -1;
|
||||
procDir = objPath.substr(pos);
|
||||
}
|
||||
}
|
||||
|
||||
return returnProci;
|
||||
}
|
||||
|
||||
|
||||
Foam::label Foam::fileOperation::detectProcessorPath(const fileName& fName)
|
||||
{
|
||||
fileName path, pDir, local;
|
||||
label start, size, nProcs;
|
||||
return splitProcessorPath(fName, path, pDir, local, start, size, nProcs);
|
||||
procRangeType group;
|
||||
label nProcs;
|
||||
return splitProcessorPath(fName, path, pDir, local, group, nProcs);
|
||||
}
|
||||
|
||||
|
||||
|
||||
@ -99,26 +99,36 @@ public:
|
||||
};
|
||||
static const Enum<pathType> pathTypeNames_;
|
||||
|
||||
|
||||
//- A dirIndex adds the path type and local offset to a fileName
|
||||
typedef Tuple2<fileName, Tuple2<pathType, label>> dirIndex;
|
||||
typedef List<dirIndex> dirIndexList;
|
||||
|
||||
//- For addressing a range of processors,
|
||||
//- identical to UPstream::rangeType
|
||||
typedef IntRange<int> procRangeType;
|
||||
|
||||
|
||||
protected:
|
||||
|
||||
// Protected data
|
||||
// Protected Data
|
||||
|
||||
//- Communicator to use
|
||||
const label comm_;
|
||||
|
||||
//- Distributed roots (parallel run)
|
||||
bool distributed_;
|
||||
|
||||
//- Detected processors directories
|
||||
mutable HashTable<dirIndexList> procsDirs_;
|
||||
|
||||
//- file-change monitor for all registered files
|
||||
//- File-change monitor for all registered files
|
||||
mutable autoPtr<fileMonitor> monitorPtr_;
|
||||
|
||||
|
||||
// Protected Member Functions
|
||||
|
||||
//- Get or create fileMonitor singleton
|
||||
fileMonitor& monitor() const;
|
||||
|
||||
//- Sort directory entries according to time value
|
||||
@ -148,7 +158,7 @@ protected:
|
||||
// \return empty fileName if not found.
|
||||
virtual refPtr<dirIndexList> lookupProcessorsPath
|
||||
(
|
||||
const fileName&
|
||||
const fileName& objectPath
|
||||
) const;
|
||||
|
||||
//- Does ioobject exist. Is either a directory (empty name()) or
|
||||
@ -158,16 +168,16 @@ protected:
|
||||
|
||||
public:
|
||||
|
||||
// Static data
|
||||
// Static Data
|
||||
|
||||
//- Return the processors directory name (usually "processors")
|
||||
static word processorsBaseDir;
|
||||
|
||||
//- Default fileHandler
|
||||
//- Name of the default fileHandler
|
||||
static word defaultFileHandler;
|
||||
|
||||
|
||||
// Public data types
|
||||
// Public Data Types
|
||||
|
||||
//- Runtime type information
|
||||
TypeName("fileOperation");
|
||||
@ -179,8 +189,12 @@ public:
|
||||
|
||||
// Constructors
|
||||
|
||||
//- Construct from communicator
|
||||
explicit fileOperation(const label comm);
|
||||
//- Construct from communicator, optionally with distributed roots
|
||||
explicit fileOperation
|
||||
(
|
||||
const label comm,
|
||||
const bool distributedRoots = false
|
||||
);
|
||||
|
||||
|
||||
// Declare run-time constructor selection table
|
||||
@ -213,6 +227,21 @@ public:
|
||||
|
||||
// Member Functions
|
||||
|
||||
//- Distributed roots (parallel run)
|
||||
bool distributed() const noexcept
|
||||
{
|
||||
return distributed_;
|
||||
}
|
||||
|
||||
//- Set distributed roots on/off, return old value
|
||||
bool distributed(bool on) noexcept
|
||||
{
|
||||
bool old(distributed_);
|
||||
distributed_ = on;
|
||||
return old;
|
||||
}
|
||||
|
||||
|
||||
// OSSpecific equivalents
|
||||
|
||||
//- Make directory
|
||||
@ -538,7 +567,8 @@ public:
|
||||
//- Operating on fileName: replace processorXXX with procDir
|
||||
fileName processorsPath(const fileName&, const word& procDir) const;
|
||||
|
||||
//- Split fileName into part before 'processor' and part after.
|
||||
//- Split objectPath into part before 'processor' and part after.
|
||||
//
|
||||
// Returns -1 or processor number and optionally number
|
||||
// of processors. Use with care.
|
||||
// - path/"processor"+Foam::name(proci)/local reconstructs input
|
||||
@ -546,17 +576,16 @@ public:
|
||||
// collated processors equivalence
|
||||
static label splitProcessorPath
|
||||
(
|
||||
const fileName&,
|
||||
const fileName& objectPath,
|
||||
fileName& path,
|
||||
fileName& procDir,
|
||||
fileName& local,
|
||||
label& groupStart,
|
||||
label& groupSize,
|
||||
procRangeType& group,
|
||||
label& nProcs
|
||||
);
|
||||
|
||||
//- Detect processor number from '/aa/bb/processorDDD/cc'
|
||||
static label detectProcessorPath(const fileName&);
|
||||
static label detectProcessorPath(const fileName& objPath);
|
||||
};
|
||||
|
||||
|
||||
|
||||
@ -200,15 +200,16 @@ Foam::fileOperations::masterUncollatedFileOperation::filePathInfo
|
||||
if (io.time().processorCase())
|
||||
{
|
||||
refPtr<dirIndexList> pDirs(lookupProcessorsPath(io.objectPath()));
|
||||
forAll(pDirs(), i)
|
||||
|
||||
for (const dirIndex& dirIdx : pDirs())
|
||||
{
|
||||
const fileName& pDir = pDirs()[i].first();
|
||||
const fileName& pDir = dirIdx.first();
|
||||
fileName objPath =
|
||||
processorsPath(io, io.instance(), pDir)
|
||||
/io.name();
|
||||
if (objPath != writePath && isFileOrDir(isFile, objPath))
|
||||
{
|
||||
searchType = pDirs()[i].second().first();
|
||||
searchType = dirIdx.second().first();
|
||||
procsDir = pDir;
|
||||
return objPath;
|
||||
}
|
||||
@ -272,9 +273,10 @@ Foam::fileOperations::masterUncollatedFileOperation::filePathInfo
|
||||
(
|
||||
lookupProcessorsPath(io.objectPath())
|
||||
);
|
||||
forAll(pDirs(), i)
|
||||
|
||||
for (const dirIndex& dirIdx : pDirs())
|
||||
{
|
||||
const fileName& pDir = pDirs()[i].first();
|
||||
const fileName& pDir = dirIdx.first();
|
||||
|
||||
fileName fName
|
||||
(
|
||||
@ -283,7 +285,7 @@ Foam::fileOperations::masterUncollatedFileOperation::filePathInfo
|
||||
);
|
||||
if (isFileOrDir(isFile, fName))
|
||||
{
|
||||
switch (pDirs()[i].second().first())
|
||||
switch (dirIdx.second().first())
|
||||
{
|
||||
case fileOperation::PROCUNCOLLATED:
|
||||
{
|
||||
@ -1435,9 +1437,9 @@ bool Foam::fileOperations::masterUncollatedFileOperation::exists
|
||||
// 2. Check processors/
|
||||
if (io.time().processorCase())
|
||||
{
|
||||
forAll(pDirs, i)
|
||||
for (const dirIndex& dirIdx : pDirs)
|
||||
{
|
||||
const fileName& pDir = pDirs[i].first();
|
||||
const fileName& pDir = dirIdx.first();
|
||||
fileName procPath =
|
||||
processorsPath(io, io.instance(), pDir)
|
||||
/io.name();
|
||||
@ -1953,18 +1955,13 @@ Foam::fileOperations::masterUncollatedFileOperation::readStream
|
||||
|
||||
|
||||
// Analyse the file path (on (co)master) to see the processors type
|
||||
// Note: this should really be part of filePath() which should return
|
||||
// both file and index in file.
|
||||
|
||||
fileName path, procDir, local;
|
||||
label groupStart, groupSize, nProcs;
|
||||
splitProcessorPath
|
||||
(
|
||||
fName,
|
||||
path,
|
||||
procDir,
|
||||
local,
|
||||
groupStart,
|
||||
groupSize,
|
||||
nProcs
|
||||
);
|
||||
procRangeType group;
|
||||
label nProcs;
|
||||
splitProcessorPath(fName, path, procDir, local, group, nProcs);
|
||||
|
||||
|
||||
if (!Pstream::parRun())
|
||||
@ -1981,12 +1978,10 @@ Foam::fileOperations::masterUncollatedFileOperation::readStream
|
||||
<< exit(FatalIOError);
|
||||
}
|
||||
|
||||
// Analyse the fileName for any processor subset. Note: this
|
||||
// should really be part of filePath() which should return
|
||||
// both file and index in file.
|
||||
if (groupStart != -1 && groupSize > 0)
|
||||
// The local rank (offset)
|
||||
if (!group.empty())
|
||||
{
|
||||
proci = proci-groupStart;
|
||||
proci = proci - group.start();
|
||||
}
|
||||
|
||||
if (debug)
|
||||
@ -2009,7 +2004,7 @@ Foam::fileOperations::masterUncollatedFileOperation::readStream
|
||||
// Are we reading from single-master file ('processors256') or
|
||||
// from multi-master files ('processors256_0-9')
|
||||
label readComm = -1;
|
||||
if (groupStart != -1 && groupSize > 0)
|
||||
if (!group.empty())
|
||||
{
|
||||
readComm = comm_;
|
||||
if (UPstream::master(comm_) && !isPtr && !fName.empty())
|
||||
|
||||
@ -119,12 +119,13 @@ Foam::fileName Foam::fileOperations::uncollatedFileOperation::filePathInfo
|
||||
fileOperation::lookupAndCacheProcessorsPath
|
||||
(
|
||||
io.objectPath(),
|
||||
false
|
||||
false // No additional parallel synchronisation
|
||||
)
|
||||
);
|
||||
forAll(pDirs(), i)
|
||||
|
||||
for (const dirIndex& dirIdx : pDirs())
|
||||
{
|
||||
const fileName& pDir = pDirs()[i].first();
|
||||
const fileName& pDir = dirIdx.first();
|
||||
fileName objPath =
|
||||
processorsPath(io, io.instance(), pDir)
|
||||
/io.name();
|
||||
@ -172,8 +173,8 @@ Foam::fileOperations::uncollatedFileOperation::lookupProcessorsPath
|
||||
const fileName& fName
|
||||
) const
|
||||
{
|
||||
// Do not use parallel synchronisation
|
||||
return lookupAndCacheProcessorsPath(fName, false);
|
||||
// No additional parallel synchronisation
|
||||
return fileOperation::lookupAndCacheProcessorsPath(fName, false);
|
||||
}
|
||||
|
||||
|
||||
@ -592,20 +593,14 @@ Foam::fileOperations::uncollatedFileOperation::readStream
|
||||
// should really be part of filePath() which should return
|
||||
// both file and index in file.
|
||||
fileName path, procDir, local;
|
||||
label groupStart, groupSize, nProcs;
|
||||
splitProcessorPath
|
||||
(
|
||||
fName,
|
||||
path,
|
||||
procDir,
|
||||
local,
|
||||
groupStart,
|
||||
groupSize,
|
||||
nProcs
|
||||
);
|
||||
if (groupStart != -1 && groupSize > 0)
|
||||
procRangeType group;
|
||||
label nProcs;
|
||||
splitProcessorPath(fName, path, procDir, local, group, nProcs);
|
||||
|
||||
// The local rank (offset)
|
||||
if (!group.empty())
|
||||
{
|
||||
proci = proci-groupStart;
|
||||
proci = proci - group.start();
|
||||
}
|
||||
|
||||
// Read data and return as stream
|
||||
|
||||
@ -69,7 +69,7 @@ protected:
|
||||
) const;
|
||||
|
||||
//- Lookup name of processorsDDD using cache.
|
||||
// \not Do not use any parallel synchronisation
|
||||
// \note Do not use any parallel synchronisation
|
||||
// \return empty fileName if not found.
|
||||
virtual refPtr<dirIndexList> lookupProcessorsPath
|
||||
(
|
||||
@ -85,7 +85,7 @@ public:
|
||||
|
||||
// Constructors
|
||||
|
||||
//- Construct null
|
||||
//- Default construct
|
||||
uncollatedFileOperation(bool verbose);
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user