From 627d79dba68d9436a8ce3e4df5e92f40d976cfed Mon Sep 17 00:00:00 2001
From: Mark Olesen <Mark.Olesen@esi-group.com>
Date: Wed, 9 Dec 2020 14:15:24 +0100
Subject: [PATCH] ENH: reduce use of readdir on individual processors (#1946)

- implicitly enabled when timeStampMaster (default) is used
  for the fileModificationChecking

- When running with non-distributed roots (eg, NFS-share) read for
  processor directories on master only and send to sub-processes
  instead individual reads.

- If disabled (old default, or when running with distributed roots),
  uses the regular fileHandler readDir, which may perform readDir
  on each processor. Potentially slow startup times on large systems.

Improvements based on analysis from T.Aoyagi(RIST), A.Azami(RIST)
---
 src/OpenFOAM/db/Time/TimeIO.C                 | 11 +++-
 src/OpenFOAM/global/argList/argList.C         |  8 +++
 .../fileOperation/fileOperation.C             | 63 ++++++++++++++-----
 3 files changed, 67 insertions(+), 15 deletions(-)

diff --git a/src/OpenFOAM/db/Time/TimeIO.C b/src/OpenFOAM/db/Time/TimeIO.C
index 1b9232392c..1a57de375c 100644
--- a/src/OpenFOAM/db/Time/TimeIO.C
+++ b/src/OpenFOAM/db/Time/TimeIO.C
@@ -166,8 +166,17 @@ void Foam::Time::readDict()
         }
         controlDict_.watchIndices().clear();
 
+        // The new handler, with verbosity
+        autoPtr<fileOperation> newHandler =
+            fileOperation::New(fileHandlerName, true);
+
+        if (TimePaths::distributed() && newHandler)
+        {
+            newHandler->distributed(true);
+        }
+
         // Installing the new handler
-        Foam::fileHandler(fileOperation::New(fileHandlerName, true));
+        Foam::fileHandler(std::move(newHandler));
 
         // Reinstall old watches
         fileHandler().addWatches(controlDict_, oldWatched);
diff --git a/src/OpenFOAM/global/argList/argList.C b/src/OpenFOAM/global/argList/argList.C
index 735a2487c9..2e1d2fc48d 100644
--- a/src/OpenFOAM/global/argList/argList.C
+++ b/src/OpenFOAM/global/argList/argList.C
@@ -1414,6 +1414,14 @@ void Foam::argList::parse
         case_ = globalCase_;   // Redundant, but extra safety?
     }
 
+    // If needed, adjust fileHandler for distributed roots
+    if (parRunControl_.distributed())
+    {
+        if (fileOperation::fileHandlerPtr_)
+        {
+            fileOperation::fileHandlerPtr_->distributed(true);
+        }
+    }
 
     // Keep or discard slave and root information for reporting:
     if (Pstream::master() && parRunControl_.parRun())
diff --git a/src/OpenFOAM/global/fileOperations/fileOperation/fileOperation.C b/src/OpenFOAM/global/fileOperations/fileOperation/fileOperation.C
index 4ee216eb7b..fa5345096d 100644
--- a/src/OpenFOAM/global/fileOperations/fileOperation/fileOperation.C
+++ b/src/OpenFOAM/global/fileOperations/fileOperation/fileOperation.C
@@ -361,21 +361,62 @@ Foam::fileOperation::lookupAndCacheProcessorsPath
             return iter.val();
         }
 
+        DynamicList<dirIndex> procDirs;
+        fileNameList dirEntries;
+
         // Read all directories to see any beginning with processor
         // ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-        DynamicList<dirIndex> procDirs;
-
         // Note: use parallel synchronised reading so cache will be same
         //       order on all processors
-        fileNameList dirNames(readDir(path, fileName::Type::DIRECTORY));
+
+        const bool readDirMasterOnly
+        (
+            regIOobject::fileModificationChecking == IOobject::timeStampMaster
+         || regIOobject::fileModificationChecking == IOobject::inotifyMaster
+        );
+
+        // As byproduct of the above selection, we exclude masterUncollated
+        // from using read/send, but that doesn't matter since that is what
+        // its own internals for readDir() do anyhow.
+
+        if (readDirMasterOnly && Pstream::parRun() && !distributed())
+        {
+            // Non-distributed.
+            // Read on master only and send to subProcs
+
+            if (Pstream::master())
+            {
+                dirEntries = Foam::readDir(path, fileName::Type::DIRECTORY);
+
+                DebugInfo
+                    << "readDir on master: send " << dirEntries.size()
+                    << " names to sub-processes" << endl;
+            }
+
+            Pstream::scatter(dirEntries, Pstream::msgType(), comm_);
+        }
+        else
+        {
+            // Serial or distributed roots.
+            // Handle readDir() with virtual method
+
+            if (debug)
+            {
+                Pout<< "readDir without special master/send treatment"
+                    << endl;
+            }
+
+            dirEntries = readDir(path, fileName::Type::DIRECTORY);
+        }
+
 
         // Extract info from processorsDDD or processorDDD:
         // - highest processor number
         // - directory+offset containing data for proci
         label maxProc = -1;
 
-        for (const fileName& dirN : dirNames)
+        for (const fileName& dirN : dirEntries)
         {
             // Analyse directory name
             fileName rp, rd, rl;
@@ -775,16 +816,11 @@ Foam::instantList Foam::fileOperation::findTimes
             << directory << endl;
     }
 
-    // Read directory entries into a list
-    fileNameList dirEntries
-    (
-        Foam::readDir
-        (
-            directory,
-            fileName::DIRECTORY
-        )
-    );
+    // Note: do NOT use master-only reading here (as per lookupProcessorsPath)
+    // since this routine is called on an individual processorN directory
 
+    // Read directory entries into a list
+    fileNameList dirEntries(Foam::readDir(directory, fileName::DIRECTORY));
     instantList times = sortTimes(dirEntries, constantName);
 
 
@@ -1269,7 +1305,6 @@ Foam::label Foam::fileOperation::splitProcessorPath
             // We are done!
             break;
         }
-
     }
 
     if (pos != string::npos)