From 201f117f5f02d7e77c9d6d6a250a3bdbf3f33bb3 Mon Sep 17 00:00:00 2001 From: Mark Olesen Date: Wed, 17 Mar 2021 16:40:42 +0100 Subject: [PATCH] BUG: prevent blocking when processor directories are missing (fixes #2027) - ensure that the processor directory cache is consistently available by providing appropriate synthesized names. If the expected files really do not exist, it will still fail at a later point (but not in the caching). This addresses cases that have been manually distributed (different content on different nodes, but the same root path), while still maintaining the readDir on master only for massively parallel cases (#1946). It also addresses regular cases with gaps in the processor directories. --- .../fileOperation/fileOperation.C | 216 +++++++++++++----- 1 file changed, 163 insertions(+), 53 deletions(-) diff --git a/src/OpenFOAM/global/fileOperations/fileOperation/fileOperation.C b/src/OpenFOAM/global/fileOperations/fileOperation/fileOperation.C index eaac9a90ff..54803fdc85 100644 --- a/src/OpenFOAM/global/fileOperations/fileOperation/fileOperation.C +++ b/src/OpenFOAM/global/fileOperations/fileOperation/fileOperation.C @@ -180,6 +180,43 @@ static bool parseProcsNumRange } // End anonymous namespace +#if 0 + +// Sorting of processor directories +#include "stringOpsSort.H" +namespace +{ + +// Sort processor directory names (natural order) +// - not strictly necessary +void sortProcessorDirs(Foam::UList& dirs) +{ + if (dirs.size() > 1) + { + std::stable_sort + ( + dirs.begin(), + dirs.end(), + [] + ( + const Foam::fileOperation::dirIndex& a, + const Foam::fileOperation::dirIndex& b + ) -> bool + { + return + Foam::stringOps::natural_sort::compare + ( + a.first(), + b.first() + ) < 0; + } + ); + } +} + +} // End anonymous namespace +#endif + // * * * * * * * * * * * * * Static Member Functions * * * * * * * * * * * * // Foam::labelList Foam::fileOperation::ioRanks() @@ -407,10 +444,10 @@ Foam::fileOperation::lookupAndCacheProcessorsPath if (readDirMasterOnly) { - // Non-distributed. + // Parallel and non-distributed // Read on master only and send to subProcs - if (Pstream::master()) + if (Pstream::master(comm_)) { dirEntries = Foam::readDir(path, fileName::Type::DIRECTORY); @@ -435,12 +472,11 @@ Foam::fileOperation::lookupAndCacheProcessorsPath dirEntries = readDir(path, fileName::Type::DIRECTORY); } - - // Extract info from processorsDDD or processorDDD: + // Extract info from processorN or processorsNN // - highest processor number // - directory+offset containing data for proci - label maxProc = -1; + label nProcs = 0; for (const fileName& dirN : dirEntries) { // Analyse directory name @@ -449,77 +485,151 @@ Foam::fileOperation::lookupAndCacheProcessorsPath const label readProci = splitProcessorPath(dirN, rp, rd, rl, group, rNum); - maxProc = max(maxProc, readProci); + nProcs = max(nProcs, readProci+1); + + Tuple2 pathTypeIdx(pathType::NOTFOUND, 0); if (proci == readProci) { - // Found "processorDDD". No need for index. - procDirs.append - ( - dirIndex - ( - dirN, - Tuple2(PROCUNCOLLATED, -1) - ) - ); + // Found "processorN" + pathTypeIdx.first() = pathType::PROCUNCOLLATED; } - else if (group.found(proci)) + else if (rNum != -1) { - // "processorsDDD_start-end" - // Found the file that contains the data for proci - const label localProci = proci - group.start(); - procDirs.append - ( - dirIndex - ( - dirN, - Tuple2(PROCOBJECT, localProci) - ) - ); - } - if (rNum != -1) - { - // Direct detection of processorsDDD - maxProc = rNum-1; + // "processorsNN" or "processorsNN_start-end" + nProcs = max(nProcs, rNum); if (group.empty()) { - // "processorsDDD" - procDirs.append + // "processorsNN" + + if (proci < rNum) + { + // And it is also in range. + // Eg for "processors4": 3 is ok, 10 is not + + pathTypeIdx.first() = pathType::PROCBASEOBJECT; + pathTypeIdx.second() = proci; + } + } + else if (group.found(proci)) + { + // "processorsNN_start-end" + // - save the local proc offset + + pathTypeIdx.first() = pathType::PROCOBJECT; + pathTypeIdx.second() = (proci - group.start()); + } + } + + if (pathTypeIdx.first() != pathType::NOTFOUND) + { + procDirs.append(dirIndex(dirN, pathTypeIdx)); + } + } + + // Global check of empty/exists. + // 1 : empty directory + // 2 : non-empty directory + // 3 : mixed empty/non-empty directory (after reduce) + // Combines andOp() and orOp() in single operation + + unsigned procDirsStatus = (procDirs.empty() ? 1u : 2u); + + if (debug) + { + Pout<< "fileOperation::lookupProcessorsPath " << procPath + << " detected:" << procDirs << endl; + } + + if (Pstream::parRun()) + { + reduce(procDirsStatus, bitOrOp()); // worldComm + + if (procDirsStatus == 3u) + { + // Mixed empty/exists for procDirs. + // Synthesize missing directory name (consistency in cache + // existence). + // Cannot reliably synthesize RANK-COLLATED, only COLLATED or + // UNCOLLATED. + // + // RANK-COLLATED should have been read from its corresponding + // master anyhow + + int flavour(pathType::PROCUNCOLLATED); + for (const dirIndex& pDir : procDirs) + { + flavour = max(flavour, int(pDir.second().first())); + } + + reduce(nProcs, maxOp