ENH: more flexible handling of FOAM_IORANKS

- accept plain lists (space or comma separated) as well as the
  traditional OpenFOAM lists. This simplifies argument handling
  with job scripts.

  For example,
     simpleFoam -ioRanks 0,4,8 ...
  vs
     simpleFoam -ioRanks '(0 4 8)'  ...

  It is also possible to select the IO ranks on a per-host basis:

     simpleFoam -ioRanks host  ...

- expose rank/subrank handling as static fileOperation methods

applications/test/fileHandler-ranks1/Make/files

@@ -0,0 +1,3 @@
+Test-fileHandler-ranks1.C
+
+EXE = $(FOAM_USER_APPBIN)/Test-fileHandler-ranks1

applications/test/fileHandler-ranks1/Make/options

@@ -0,0 +1,2 @@
+/* EXE_INC = */
+/* EXE_LIBS = */

applications/test/fileHandler-ranks1/README

@@ -0,0 +1,20 @@
+Test that -ioRanks selections perform as expected and that sub-rank
+selection is also doing the correct thing.
+
+For example,
+
+mpirun -np 10 --oversubscribe Test-fileHandler-ranks1 -parallel -ioRanks host
+mpirun -np 10 --oversubscribe Test-fileHandler-ranks1 -parallel -ioRanks '(0 3 14)'
+mpirun -np 10 --oversubscribe Test-fileHandler-ranks1 -parallel -ioRanks 14,15,2,0,10,15,0,0,0,
+mpirun -np 10 --oversubscribe Test-fileHandler-ranks1 -parallel -ioRanks '0,3,10'
+mpirun -np 10 --oversubscribe Test-fileHandler-ranks1 -parallel -ioRanks '0,3,11'
+mpirun -np 10 --oversubscribe Test-fileHandler-ranks1 -parallel -ioRanks '0,3,14'
+mpirun -np 10 --oversubscribe Test-fileHandler-ranks1 -parallel -ioRanks '0,3,15'
+mpirun -np 10 --oversubscribe Test-fileHandler-ranks1 -parallel -ioRanks '0,3,12,115'
+mpirun -np 10 --oversubscribe Test-fileHandler-ranks1 -parallel -ioRanks '0,3,10,12,115'
+mpirun -np 10 --oversubscribe Test-fileHandler-ranks1 -parallel -ioRanks host -pick '1,2,3,4'
+mpirun -np 10 --oversubscribe Test-fileHandler-ranks1 -parallel -pick '2,4,6,8'
+mpirun -np 10 --oversubscribe Test-fileHandler-ranks1 -parallel -ioRanks '0,3' -pick '2,4,6,8'
+mpirun -np 10 --oversubscribe Test-fileHandler-ranks1 -parallel -ioRanks '0,4' -pick '2,4,6,8'
+mpirun -np 10 --oversubscribe Test-fileHandler-ranks1 -parallel -ioRanks '0,6' -pick '2,4,6,8'
+mpirun -np 10 --oversubscribe Test-fileHandler-ranks1 -parallel -ioRanks '0,6' -pick '2,4,6,8'

applications/test/fileHandler-ranks1/Test-fileHandler-ranks1.C

@@ -0,0 +1,249 @@
+/*---------------------------------------------------------------------------*\
+  =========                 |
+  \\      /  F ield         | OpenFOAM: The Open Source CFD Toolbox
+   \\    /   O peration     |
+    \\  /    A nd           | www.openfoam.com
+     \\/     M anipulation  |
+-------------------------------------------------------------------------------
+    Copyright (C) 2023 OpenCFD Ltd.
+-------------------------------------------------------------------------------
+License
+    This file is part of OpenFOAM.
+
+    OpenFOAM is free software: you can redistribute it and/or modify it
+    under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    OpenFOAM is distributed in the hope that it will be useful, but WITHOUT
+    ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+    FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+    for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with OpenFOAM.  If not, see <http://www.gnu.org/licenses/>.
+
+Application
+    Test-fileHandler-ranks1
+
+Description
+    Test IO ranks and ranks selection
+
+\*---------------------------------------------------------------------------*/
+
+#include "argList.H"
+#include "fileName.H"
+#include "fileOperation.H"
+#include "IOstreams.H"
+#include "ITstream.H"
+#include "OSspecific.H"
+#include "Pstream.H"
+#include "SHA1.H"
+#include "stringOps.H"
+
+using namespace Foam;
+
+// Parse space, comma, semicolon separated list of integers, floats etc...
+template<class PrimitiveType>
+static List<PrimitiveType> splitStringToList(const std::string& str)
+{
+    const SubStrings<std::string> items = stringOps::splitAny(str, " ,;");
+
+    DynamicList<PrimitiveType> values(items.size());
+
+    for (const auto& item : items)
+    {
+        const std::string s(item.str());
+
+        PrimitiveType val;
+
+        if (Foam::read(s, val))
+        {
+            values.push_back(val);
+        }
+        else
+        {
+            // Report errors? Could get noisy...
+        }
+    }
+
+    return List<PrimitiveType>(std::move(values));
+}
+
+
+//- Construct by parsing string for scalar ranges
+//  The individual items are space, comma or semicolon delimited.
+static labelList parseIOranks
+(
+    const Foam::string& str,
+    label numProcs = -1
+)
+{
+    bool byHostName = false;
+
+    labelList ranks;
+
+    // Info<< "parsing: " << str << endl;
+
+    if (!str.empty())
+    {
+        if (str.contains('('))
+        {
+            // Looks like a list - tokenise it
+
+            ITstream is(str);
+            if (!is.empty())
+            {
+                is >> ranks;
+            }
+        }
+        else if (str == "host")
+        {
+            // Select by host
+            byHostName = true;
+        }
+        else
+        {
+            // Manual parse
+            ranks = splitStringToList<label>(str);
+        }
+    }
+
+    if (ranks.size())
+    {
+        // Never trust user input.
+        // Sort and eliminate any duplicates
+
+        std::sort(ranks.begin(), ranks.end());
+
+        auto last = std::unique(ranks.begin(), ranks.end());
+
+        label newLen = label(last - ranks.begin());
+
+        // Detect any values that are too large
+        if (numProcs > 0)
+        {
+            auto iter = std::find_if
+            (
+                ranks.begin(),
+                last,
+                [=](label proci) { return proci >= numProcs; }
+            );
+
+            if (last != iter)
+            {
+                newLen = label(iter - ranks.begin());
+            }
+        }
+
+        ranks.resize(newLen);
+    }
+    else if (byHostName)
+    {
+        // Use hostname
+        // Lowest rank per hostname is the IO rank
+
+        numProcs = UPstream::nProcs(UPstream::worldComm);
+
+        List<SHA1Digest> digests;
+        if (UPstream::master(UPstream::worldComm))
+        {
+            digests.resize(numProcs);
+        }
+
+        // Could also add lowercase etc, but since hostName()
+        // will be consistent within the same node, there is no need.
+        SHA1Digest myDigest(SHA1(hostName()).digest());
+
+        // The fixed-length digest allows use of MPI_Gather
+        UPstream::mpiGather
+        (
+            myDigest.cdata_bytes(),     // Send
+            digests.data_bytes(),       // Recv
+            SHA1Digest::max_size(),     // Num send/recv per rank
+            UPstream::worldComm
+        );
+
+        if (UPstream::master(UPstream::worldComm))
+        {
+            DynamicList<label> dynRanks(numProcs);
+
+            dynRanks.push_back(0);  // Always include master
+            label previ = 0;
+
+            for (label proci = 1; proci < digests.size(); ++proci)
+            {
+                if (digests[proci] != digests[previ])
+                {
+                    dynRanks.push_back(proci);
+                    previ = proci;
+                }
+            }
+
+            ranks.transfer(dynRanks);
+        }
+
+        Pstream::broadcast(ranks, UPstream::worldComm);
+    }
+
+    return ranks;
+}
+
+
+// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
+// Main program:
+
+int main(int argc, char *argv[])
+{
+    argList::noBanner();
+    argList::noCheckProcessorDirectories();
+
+    argList::addOption("io-ranks", "list", "shadow for -ioRanks (testing)");
+    argList::addOption("pick", "list", "limited subset of procs");
+
+    labelList ioRanks;
+
+    // Pre-check
+    for (int argi = 1; argi < argc; ++argi)
+    {
+        if (strcmp(argv[argi], "-io-ranks") == 0)
+        {
+            if (argi < argc-1)
+            {
+                ioRanks = parseIOranks(argv[argi+1]);
+                break;
+            }
+        }
+    }
+
+    #include "setRootCase.H"
+
+    bitSet useProc;
+
+    if (args.found("pick"))
+    {
+        useProc = bitSet
+        (
+            UPstream::nProcs(),
+            parseIOranks(args.get<string>("pick"))
+        );
+    }
+
+    Info<< "procs: " << UPstream::nProcs() << endl;
+    Info<< "io-ranks: " << flatOutput(ioRanks) << endl;
+    Info<< "-ioRanks: "
+        << flatOutput(fileOperation::getGlobalIORanks()) << endl;
+
+    Info<< "pick: " << flatOutput(useProc.toc()) << endl;
+
+    // labelList subRanks = fileOperation::getGlobalSubRanks(useProc);
+    //
+    // Pout<< "sub ranks: "
+    //     << flatOutput(fileOperation::getGlobalSubRanks(useProc)) << endl;
+
+    Info<< "\nEnd\n" << endl;
+    return 0;
+}
+
+
+// ************************************************************************* //

src/OpenFOAM/Make/files

@@ -20,6 +20,7 @@ fileOps = global/fileOperations
 $(fileOps)/fileOperation/fileOperation.C
 $(fileOps)/fileOperation/fileOperationBroadcast.C
 $(fileOps)/fileOperation/fileOperationNew.C
+$(fileOps)/fileOperation/fileOperationRanks.C
 $(fileOps)/fileOperationInitialise/fileOperationInitialise.C
 $(fileOps)/uncollatedFileOperation/uncollatedFileOperation.C
 $(fileOps)/masterUncollatedFileOperation/masterUncollatedFileOperation.C

src/OpenFOAM/global/fileOperations/collatedFileOperation/collatedFileOperation.C

@@ -78,7 +78,7 @@ namespace fileOperations
 
 void Foam::fileOperations::collatedFileOperation::printBanner
 (
-    const bool printRanks
+    const bool withRanks
 ) const
 {
     DetailInfo
@@ -105,7 +105,7 @@ void Foam::fileOperations::collatedFileOperation::printBanner
             << "         OpenFOAM etc/controlDict" << endl;
     }
 
-    if (printRanks)
+    if (withRanks)
     {
         // Information about the ranks
         stringList hosts(Pstream::nProcs());
@@ -277,11 +277,11 @@ Foam::fileOperations::collatedFileOperation::collatedFileOperation
     masterUncollatedFileOperation
     (
         (
-            ioRanks().size()
+            fileOperation::getGlobalIORanks().size()
           ? UPstream::allocateCommunicator
             (
                 UPstream::worldComm,
-                subRanks(Pstream::nProcs())
+                subRanks(UPstream::nProcs())
             )
           : UPstream::worldComm
         ),
@@ -290,7 +290,7 @@ Foam::fileOperations::collatedFileOperation::collatedFileOperation
     managedComm_(getManagedComm(comm_)),  // Possibly locally allocated
     writer_(mag(maxThreadFileBufferSize), comm_),
     nProcs_(Pstream::nProcs()),
-    ioRanks_(ioRanks())
+    ioRanks_(fileOperation::getGlobalIORanks())
 {
     init(verbose);
 }

src/OpenFOAM/global/fileOperations/collatedFileOperation/collatedFileOperation.H

@@ -98,7 +98,7 @@ protected:
     // Protected Member Functions
 
         //- Print banner information, optionally with io ranks
-        void printBanner(const bool printRanks = false) const;
+        void printBanner(const bool withRanks = false) const;
 
         //- Is proci master of communicator (in parallel) or master of
         //- the io ranks (non-parallel)

src/OpenFOAM/global/fileOperations/collatedFileOperation/hostCollatedFileOperation.C

@@ -64,60 +64,16 @@ Foam::labelList Foam::fileOperations::hostCollatedFileOperation::subRanks
     const label n
 )
 {
-    DynamicList<label> subRanks(64);
+    labelList mainIOranks(fileOperation::getGlobalIORanks());
 
-    labelList mainRanks(fileOperation::ioRanks());
+    if (mainIOranks.empty())
-    if (!mainRanks.empty())
     {
-        if (!mainRanks.found(0))
+        mainIOranks = fileOperation::getGlobalHostIORanks();
-        {
-            FatalErrorInFunction
-                << "Rank 0 (master) should be in the IO ranks. Currently "
-                << mainRanks << nl
-                << exit(FatalError);
-        }
-
-        // The lowest numbered rank is the IO rank
-        const bitSet isIOrank(n, mainRanks);
-
-        for (label proci = Pstream::myProcNo(); proci >= 0; --proci)
-        {
-            if (isIOrank[proci])
-            {
-                // Found my master. Collect all processors with same master
-                subRanks.append(proci);
-                for
-                (
-                    label rank = proci+1;
-                    rank < n && !isIOrank[rank];
-                    ++rank
-                )
-                {
-                    subRanks.append(rank);
-                }
-                break;
-            }
-        }
     }
-    else
-    {
-        // Normal operation: one lowest rank per hostname is the writer
-        const string myHostName(hostName());
 
-        stringList hosts(Pstream::nProcs());
+    labelRange subRange = fileOperation::subRanks(mainIOranks);
-        hosts[Pstream::myProcNo()] = myHostName;
-        Pstream::allGatherList(hosts);
 
-        // Collect procs with same hostname
+    return identity(subRange);
-        forAll(hosts, proci)
-        {
-            if (hosts[proci] == myHostName)
-            {
-                subRanks.append(proci);
-            }
-        }
-    }
-    return subRanks;
 }
 
 
@@ -146,7 +102,7 @@ Foam::fileOperations::hostCollatedFileOperation::hostCollatedFileOperation
             UPstream::worldComm,
             subRanks(UPstream::nProcs())
         ),
-        (UPstream::parRun() ? labelList() : ioRanks()), // processor dirs
+        (UPstream::parRun() ? labelList() : getGlobalIORanks()),
         false  // verbose
     ),
     managedComm_(getManagedComm(comm_))  // Possibly locally allocated

src/OpenFOAM/global/fileOperations/collatedFileOperation/hostCollatedFileOperation.H

@@ -40,17 +40,25 @@ Description
     FOAM_IORANKS environment variable (also when running non-parallel), e.g.
     when decomposing into 4:
 
+    \verbatim
         FOAM_IORANKS='(0 2)' decomposePar -fileHandler hostCollated
+        FOAM_IORANKS='0,2'   decomposePar -fileHandler hostCollated
+        FOAM_IORANKS='0 2'   decomposePar -fileHandler hostCollated
+    \endverbatim
 
     will generate
 
-        processors4_0-1/
+    \verbatim
-            containing data for processors 0 to 1
+        processors4_0-1/  : containing data for processors 0 to 1
-        processors4_2-3/
+        processors4_2-3/  : containing data for processors 2 to 3
-            containing data for processors 2 to 3
+    \endverbatim
+
+Environment
+    - \c FOAM_ENV : list of io-ranks as plain space or comma separated
+      list or as an OpenFOAM formatted list. Eg, '(0 4 8)'
 
 See also
-    collatedFileOperation
+    Foam::collatedFileOperation
 
 SourceFiles
     hostCollatedFileOperation.C

src/OpenFOAM/global/fileOperations/fileOperation/fileOperation.C

@@ -219,20 +219,6 @@ void sortProcessorDirs(Foam::UList<Foam::fileOperation::dirIndex>& dirs)
 
 // * * * * * * * * * * * * * Static Member Functions * * * * * * * * * * * * //
 
-Foam::labelList Foam::fileOperation::ioRanks()
-{
-    labelList ranks;
-
-    ITstream is(Foam::getEnv("FOAM_IORANKS"));
-    if (!is.empty())
-    {
-        is >> ranks;
-    }
-
-    return ranks;
-}
-
-
 Foam::instantList
 Foam::fileOperation::sortTimes
 (

src/OpenFOAM/global/fileOperations/fileOperation/fileOperation.H

@@ -30,6 +30,38 @@ Class
 Description
     An encapsulation of filesystem-related operations.
 
+    Several of the file handlers can be configured to use specific ranks for
+    IO operations. These can either be defined from the command-line
+    \c -ioRanks option, or via the \c FOAM_ENV environment variable.
+    In either case, the list of IO ranks shall always include the value
+    \b 0 (master rank). The ranks may be specified as a plain list, or as an
+    OpenFOAM list. The special value \em host can be used to specify selection
+    based on hostname.
+
+    For example,
+    \verbatim
+        FOAM_IORANKS='0 4 8'   decomposePar -fileHandler hostCollated
+        FOAM_IORANKS='0,4,8'   decomposePar -fileHandler hostCollated
+        FOAM_IORANKS='(0 4 8)' decomposePar -fileHandler hostCollated
+    \endverbatim
+    will generate
+    \verbatim
+        processors12_0-3/   : containing data for processors 0 to 3
+        processors12_4-7/   : containing data for processors 4 to 7
+        processors12_8-11/  : containing data for processors 8 to 11
+    \endverbatim
+
+    The value \em host can be used to specify a single IO rank for each
+    host. For example,
+    \verbatim
+        decomposePar -fileHandler collated -ioRanks host
+        decomposePar -fileHandler hostCollated
+    \endverbatim
+
+Environment
+    - \c FOAM_ENV : list of io-ranks as plain space or comma separated
+      list or as an OpenFOAM formatted list. Eg, '(0 4 8)'
+
 Namespace
     Foam::fileOperations
 
@@ -109,6 +141,7 @@ public:
         //- identical to UPstream::rangeType
         typedef IntRange<int> procRangeType;
 
+
 protected:
 
     // Protected Data
@@ -131,9 +164,6 @@ protected:
         //- Get or create fileMonitor singleton
         fileMonitor& monitor() const;
 
-        //- Retrieve list of IO ranks from FOAM_IORANKS env variable
-        static labelList ioRanks();
-
         //- Merge two times
         static void mergeTimes
         (
@@ -161,7 +191,7 @@ protected:
             const fileName& objectPath
         ) const;
 
-        //- Does IOObject exist.
+        //- Does IOobject exist?
         //- Is either a directory (empty name()) or a file
         bool exists(IOobject& io) const;
 
@@ -647,15 +677,15 @@ public:
 
         // Other
 
-            //- Actual name of processors dir (for use in mode PROCOBJECT,
+            //- Actual name of processors dir
-            //  PROCINSTANCE)
+            //- (for use in mode PROCOBJECT, PROCINSTANCE)
             virtual word processorsDir(const IOobject& io) const
             {
                 return processorsBaseDir;
             }
 
-            //- Actual name of processors dir (for use in mode PROCOBJECT,
+            //- Actual name of processors dir
-            //  PROCINSTANCE)
+            //- (for use in mode PROCOBJECT, PROCINSTANCE)
             virtual word processorsDir(const fileName&) const
             {
                 return processorsBaseDir;
@@ -733,6 +763,21 @@ public:
 
             //- Detect processor number from '/aa/bb/processorDDD/cc'
             static label detectProcessorPath(const fileName& objPath);
+
+
+    // Rank selection/sub-selection
+
+        //- Get (contiguous) range/bounds of ranks addressed
+        //- within the given main io-ranks.
+        static labelRange subRanks(const labelUList& mainIOranks);
+
+        //- Get list of global IO master ranks based on the hostname.
+        //- It is assumed that each host range is contiguous.
+        static labelList getGlobalHostIORanks();
+
+        //- Get list of global IO ranks from FOAM_IORANKS env variable.
+        //- If set, these correspond to the IO master ranks.
+        static labelList getGlobalIORanks();
 };
 
 

src/OpenFOAM/global/fileOperations/fileOperation/fileOperationRanks.C

@@ -0,0 +1,319 @@
+/*---------------------------------------------------------------------------*\
+  =========                 |
+  \\      /  F ield         | OpenFOAM: The Open Source CFD Toolbox
+   \\    /   O peration     |
+    \\  /    A nd           | www.openfoam.com
+     \\/     M anipulation  |
+-------------------------------------------------------------------------------
+    Copyright (C) 2022-2023 OpenCFD Ltd.
+-------------------------------------------------------------------------------
+License
+    This file is part of OpenFOAM.
+
+    OpenFOAM is free software: you can redistribute it and/or modify it
+    under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    OpenFOAM is distributed in the hope that it will be useful, but WITHOUT
+    ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+    FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+    for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with OpenFOAM.  If not, see <http://www.gnu.org/licenses/>.
+
+\*---------------------------------------------------------------------------*/
+
+#include "fileOperation.H"
+#include "stringOps.H"
+#include "ITstream.H"
+#include "Pstream.H"
+#include "SHA1.H"
+#include "OSspecific.H"  // for hostName()
+#include <cinttypes>
+
+// * * * * * * * * * * * * * * * Local Functions * * * * * * * * * * * * * * //
+
+namespace Foam
+{
+
+// Parse space, comma, semicolon separated list of integers, floats etc...
+template<class PrimitiveType>
+static List<PrimitiveType> splitStringToList(const std::string& str)
+{
+    const SubStrings<std::string> items = stringOps::splitAny(str, " ,;");
+
+    DynamicList<PrimitiveType> values(items.size());
+
+    for (const auto& item : items)
+    {
+        const std::string s(item.str());
+
+        PrimitiveType val;
+
+        if (Foam::read(s, val))
+        {
+            values.push_back(val);
+        }
+        else
+        {
+            // Report errors? Could get noisy...
+        }
+    }
+
+    return List<PrimitiveType>(std::move(values));
+}
+
+} // End namespace Foam
+
+
+// * * * * * * * * * * * * * Static Member Functions * * * * * * * * * * * * //
+
+Foam::labelRange Foam::fileOperation::subRanks(const labelUList& mainIOranks)
+{
+    // Fast path - no IO ranks.
+    if (mainIOranks.empty())
+    {
+        return labelRange();
+    }
+
+    // The lowest numbered rank is the IO rank
+    // - linear search for the enclosing range
+    // - fallback is proc = 0, which silently adds master (0) into IO ranks
+
+    label begProc = 0;
+    label endProc = UPstream::nProcs(UPstream::worldComm);
+
+    const label myProci = UPstream::myProcNo(UPstream::worldComm);
+
+    forAllReverse(mainIOranks, i)
+    {
+        if (mainIOranks[i] <= myProci)
+        {
+            begProc = mainIOranks[i];
+            if (i+1 < mainIOranks.size())
+            {
+                endProc = mainIOranks[i+1];
+            }
+            break;
+        }
+    }
+
+    Pout<< "subRanks: " << labelRange(begProc, (endProc-begProc)) << endl;
+
+    return labelRange(begProc, (endProc-begProc));
+}
+
+
+Foam::labelList Foam::fileOperation::getGlobalHostIORanks()
+{
+    const label numProcs = UPstream::nProcs(UPstream::worldComm);
+
+    // Use hostname
+    // Lowest rank per hostname is the IO rank
+
+    List<SHA1Digest> digests;
+    if (UPstream::master(UPstream::worldComm))
+    {
+        digests.resize(numProcs);
+    }
+
+    // Could also add lowercase etc, but since hostName()
+    // will be consistent within the same node, there is no need.
+    SHA1Digest myDigest(SHA1(hostName()).digest());
+
+    // The fixed-length digest allows use of MPI_Gather
+    UPstream::mpiGather
+    (
+        myDigest.cdata_bytes(),     // Send
+        digests.data_bytes(),       // Recv
+        SHA1Digest::max_size(),     // Num send/recv per rank
+        UPstream::worldComm
+    );
+
+    labelList ranks;
+    DynamicList<label> dynRanks;
+
+    if (UPstream::master(UPstream::worldComm))
+    {
+        dynRanks.reserve(numProcs);
+
+        dynRanks.push_back(0);  // Always include master
+        label previ = 0;
+
+        for (label proci = 1; proci < digests.size(); ++proci)
+        {
+            if (digests[proci] != digests[previ])
+            {
+                dynRanks.push_back(proci);
+                previ = proci;
+            }
+        }
+
+        ranks.transfer(dynRanks);
+    }
+
+    Pstream::broadcast(ranks, UPstream::worldComm);
+    return ranks;
+}
+
+
+Foam::labelList Foam::fileOperation::getGlobalIORanks
+(
+    // const bool useHost
+)
+{
+    // bool byHostName = useHost;
+    bool byHostName = false;
+
+    DynamicList<label> dynRanks;
+
+    Foam::string str(Foam::getEnv("FOAM_IORANKS"));
+
+    if (!str.empty())
+    {
+        if (str.contains('('))
+        {
+            // Looks like a list - tokenise it
+            ITstream is(str);
+            if (!is.empty())
+            {
+                is >> dynRanks;
+            }
+        }
+        else if (str == "host")
+        {
+            // Select by hostname
+            byHostName = true;
+        }
+        else
+        {
+            // Manual parse
+            dynRanks = splitStringToList<label>(str);
+        }
+    }
+
+    if (dynRanks.size())
+    {
+        if (!dynRanks.contains(0))
+        {
+            // Could also add silently
+            // dynRanks.push_back(0);
+            FatalErrorInFunction
+                << "Rank 0 (master) should be in the IO ranks. Currently:" << nl
+                << "    " << flatOutput(dynRanks) << nl
+                << exit(FatalError);
+        }
+
+        // Never trust user input.
+        // Sort and eliminate any duplicates
+
+        std::sort(dynRanks.begin(), dynRanks.end());
+
+        if (dynRanks.front() < 0)
+        {
+            FatalErrorInFunction
+                << "Cannot have negative ranks! Currently:" << nl
+                << "    " << flatOutput(dynRanks) << nl
+                << exit(FatalError);
+        }
+
+        labelList ranks;
+
+        auto last = std::unique(dynRanks.begin(), dynRanks.end());
+
+        if (last < dynRanks.end())
+        {
+            ranks = dynRanks.slice(0, (last - dynRanks.begin()));
+        }
+        else
+        {
+            ranks = dynRanks;
+        }
+
+        return ranks;
+    }
+    else if (byHostName)
+    {
+        return fileOperation::getGlobalHostIORanks();
+    }
+
+    return labelList();
+}
+
+
+#if 0  // FUTURE
+bool Foam::fileOperation::isIOrank(const label proci) const
+{
+    return
+    (
+        UPstream::parRun()
+      ? UPstream::master(comm_)
+      : ioRanks_.empty()
+      ? (proci == 0)                // No io-ranks, assume single communicator
+      : ioRanks_.contains(proci)    // Found proci in IO rank
+    );
+}
+
+
+void Foam::fileOperation::printRanks() const
+{
+    // Collect the names of the IO masters
+    stringList hosts(UPstream::nProcs(UPstream::worldComm));
+    if (UPstream::master(comm_))
+    {
+        hosts[UPstream::myProcNo(UPstream::worldComm)] = hostName();
+    }
+    Pstream::gatherList(hosts, UPstream::msgType(), UPstream::worldComm);
+
+
+    DynamicList<label> offsetMaster;
+
+    // Calculate the offsets/counts
+
+    if (UPstream::master(UPstream::worldComm))
+    {
+        label nHostRanks = 0;
+        forAll(hosts, ranki)
+        {
+            if (!hosts[ranki].empty())
+            {
+                ++nHostRanks;
+            }
+        }
+        offsetMaster.reserve(nHostRanks+1);
+
+        forAll(hosts, ranki)
+        {
+            if (!hosts[ranki].empty())
+            {
+                offsetMaster.push_back(ranki);
+            }
+        }
+
+        // End of range is nProcs
+        offsetMaster.push_back(hosts.size());
+    }
+
+    if (offsetMaster.size() > 2)
+    {
+        DetailInfo
+            << "I/O on :" << nl << '(' << nl;
+        for (label group = 1; group < offsetMaster.size(); ++group)
+        {
+            const label beg = offsetMaster[group-1];
+            const label end = offsetMaster[group];
+
+            DetailInfo
+                << "    (" << hosts[beg].c_str() << ' '
+                << (end-beg) << ')' << nl;
+        }
+        DetailInfo
+            << ')' << nl;
+    }
+}
+#endif
+
+
+// ************************************************************************* //

src/OpenFOAM/global/fileOperations/masterUncollatedFileOperation/masterUncollatedFileOperation.C

@@ -85,46 +85,17 @@ Foam::labelList Foam::fileOperations::masterUncollatedFileOperation::subRanks
     const label n
 )
 {
-    labelList mainRanks(fileOperation::ioRanks());
+    labelList mainIOranks(fileOperation::getGlobalIORanks());
 
-    if (mainRanks.empty())
+    if (mainIOranks.empty())
     {
         return identity(n);
     }
     else
     {
-        DynamicList<label> subRanks(n);
+        labelRange subRange = fileOperation::subRanks(mainIOranks);
 
-        if (!mainRanks.found(0))
+        return identity(subRange);
-        {
-            FatalErrorInFunction
-                << "Rank 0 (master) should be in the IO ranks. Currently "
-                << mainRanks << nl
-                << exit(FatalError);
-        }
-
-        // The lowest numbered rank is the IO rank
-        const bitSet isIOrank(n, mainRanks);
-
-        for (label proci = Pstream::myProcNo(); proci >= 0; --proci)
-        {
-            if (isIOrank[proci])
-            {
-                // Found my master. Collect all processors with same master
-                subRanks.append(proci);
-                for
-                (
-                    label rank = proci+1;
-                    rank < n && !isIOrank[rank];
-                    ++rank
-                )
-                {
-                    subRanks.append(rank);
-                }
-                break;
-            }
-        }
-        return subRanks;
     }
 }
 

tutorials/IO/fileHandler/Allrun

@@ -47,7 +47,7 @@ mkdir -p machineB/fileHandler
 (   d=$PWD && \
     cd machineA/fileHandler && \
     runParallel -s multiRoot $application \
-    -fileHandler masterUncollated -ioRanks '(0 2)' \
+    -fileHandler masterUncollated -ioRanks '0,2' \
     -roots "(\"$d/machineA\" \"$d/machineB\" \"$d/machineB\")"
 )
 
@@ -60,13 +60,13 @@ runParallel foamFormatConvert -fileHandler uncollated
 
 #- Restart with multiple IO ranks
 runParallel -s multiCollated \
-    $application -fileHandler collated -ioRanks '(0 2)'
+    $application -fileHandler collated -ioRanks '0,2'
 
 #- Reconstruct the multi-rank format. Delete the collated directory
 #  since conflicts with the multi-collated directory
 rm -rf processors4
 runApplication -s multiCollated reconstructPar -latestTime \
-    -fileHandler collated -ioRanks '(0 2)'
+    -fileHandler collated -ioRanks '0,2'
 
 #- Convert the multi-rank format to uncollated
 runParallel -s uncollated foamFormatConvert -fileHandler uncollated
@@ -99,7 +99,7 @@ mkdir -p machineB/fileHandler
 (   d=$PWD && \
     cd machineA/fileHandler && \
     runParallel -s distributed_multiCollated $application \
-    -fileHandler collated -ioRanks '(0 2)' \
+    -fileHandler collated -ioRanks '0,2' \
     -roots "(\"$d/machineA\" \"$d/machineB\" \"$d/machineB\")"
 )