From aec4ba30a3e8ca996401fb759db2ad6a1d4bea09 Mon Sep 17 00:00:00 2001
From: mattijs <mattijs>
Date: Thu, 24 Nov 2022 09:00:00 +0000
Subject: [PATCH] ENH: handle watching included files

---
 .../includeEntry/includeEntry.C               | 51 ++++++++++++-------
 .../includeEtcEntry/includeEtcEntry.C         | 32 +++++++++++-
 .../masterUncollatedFileOperation.C           | 50 +++++++++++-------
 3 files changed, 95 insertions(+), 38 deletions(-)

diff --git a/src/OpenFOAM/db/dictionary/functionEntries/includeEntry/includeEntry.C b/src/OpenFOAM/db/dictionary/functionEntries/includeEntry/includeEntry.C
index 78620c7da3..96db436c4a 100644
--- a/src/OpenFOAM/db/dictionary/functionEntries/includeEntry/includeEntry.C
+++ b/src/OpenFOAM/db/dictionary/functionEntries/includeEntry/includeEntry.C
@@ -6,7 +6,7 @@
      \\/     M anipulation  |
 -------------------------------------------------------------------------------
     Copyright (C) 2011-2017 OpenFOAM Foundation
-    Copyright (C) 2018-2021 OpenCFD Ltd.
+    Copyright (C) 2018-2022 OpenCFD Ltd.
 -------------------------------------------------------------------------------
 License
     This file is part of OpenFOAM.
@@ -27,12 +27,13 @@ License
 \*---------------------------------------------------------------------------*/
 
 #include "includeEntry.H"
-#include "addToMemberFunctionSelectionTable.H"
 #include "stringOps.H"
 #include "IFstream.H"
 #include "IOstreams.H"
-#include "Time.H"
+#include "UPstream.H"
 #include "fileOperation.H"
+#include "regIOobject.H"
+#include "addToMemberFunctionSelectionTable.H"
 
 // * * * * * * * * * * * * * * Static Data Members * * * * * * * * * * * * * //
 
@@ -133,6 +134,15 @@ bool Foam::functionEntries::includeEntry::execute
     Istream& is
 )
 {
+    const auto* rioPtr = isA<regIOobject>(parentDict.topDict());
+
+    const label oldComm
+    (
+        rioPtr && rioPtr->global()
+      ? fileHandler().comm(UPstream::worldComm)
+      : fileHandler().comm()
+    );
+
     const fileName rawName(is);
     const fileName fName(resolveFile(is.name().path(), rawName, parentDict));
 
@@ -148,20 +158,19 @@ bool Foam::functionEntries::includeEntry::execute
         }
 
         // Add watch on included file
-        const dictionary& top = parentDict.topDict();
-        if (isA<regIOobject>(top))
+        if (rioPtr)
         {
-            regIOobject& rio = const_cast<regIOobject&>
-            (
-                dynamic_cast<const regIOobject&>(top)
-            );
-            rio.addWatch(fName);
+            const_cast<regIOobject&>(*rioPtr).addWatch(fName);
         }
 
         parentDict.read(ifs);
+
+        fileHandler().comm(oldComm);
         return true;
     }
 
+    fileHandler().comm(oldComm);
+
     if (!mandatory)
     {
         return true; // Never fails if optional
@@ -185,6 +194,15 @@ bool Foam::functionEntries::includeEntry::execute
     Istream& is
 )
 {
+    const auto* rioPtr = isA<regIOobject>(parentDict.topDict());
+
+    const label oldComm
+    (
+        rioPtr && rioPtr->global()
+      ? fileHandler().comm(UPstream::worldComm)
+      : fileHandler().comm()
+    );
+
     const fileName rawName(is);
     const fileName fName(resolveFile(is.name().path(), rawName, parentDict));
 
@@ -200,20 +218,19 @@ bool Foam::functionEntries::includeEntry::execute
         }
 
         // Add watch on included file
-        const dictionary& top = parentDict.topDict();
-        if (isA<regIOobject>(top))
+        if (rioPtr)
         {
-            regIOobject& rio = const_cast<regIOobject&>
-            (
-                dynamic_cast<const regIOobject&>(top)
-            );
-            rio.addWatch(fName);
+            const_cast<regIOobject&>(*rioPtr).addWatch(fName);
         }
 
         entry.read(parentDict, ifs);
+
+        fileHandler().comm(oldComm);
         return true;
     }
 
+    fileHandler().comm(oldComm);
+
     if (!mandatory)
     {
         return true; // Never fails if optional
diff --git a/src/OpenFOAM/db/dictionary/functionEntries/includeEtcEntry/includeEtcEntry.C b/src/OpenFOAM/db/dictionary/functionEntries/includeEtcEntry/includeEtcEntry.C
index 9c55a6f0c9..504673640d 100644
--- a/src/OpenFOAM/db/dictionary/functionEntries/includeEtcEntry/includeEtcEntry.C
+++ b/src/OpenFOAM/db/dictionary/functionEntries/includeEtcEntry/includeEtcEntry.C
@@ -6,7 +6,7 @@
      \\/     M anipulation  |
 -------------------------------------------------------------------------------
     Copyright (C) 2015-2017 OpenFOAM Foundation
-    Copyright (C) 2019-2021 OpenCFD Ltd.
+    Copyright (C) 2019-2022 OpenCFD Ltd.
 -------------------------------------------------------------------------------
 License
     This file is part of OpenFOAM.
@@ -27,12 +27,14 @@ License
 \*---------------------------------------------------------------------------*/
 
 #include "includeEtcEntry.H"
-#include "addToMemberFunctionSelectionTable.H"
 #include "etcFiles.H"
 #include "stringOps.H"
 #include "IFstream.H"
 #include "IOstreams.H"
+#include "UPstream.H"
 #include "fileOperation.H"
+#include "regIOobject.H"
+#include "addToMemberFunctionSelectionTable.H"
 
 // * * * * * * * * * * * * * * Static Data Members * * * * * * * * * * * * * //
 
@@ -113,6 +115,15 @@ bool Foam::functionEntries::includeEtcEntry::execute
     Istream& is
 )
 {
+    const regIOobject* rioPtr = isA<regIOobject>(parentDict.topDict());
+
+    const label oldComm
+    (
+        rioPtr && rioPtr->global()
+      ? fileHandler().comm(UPstream::worldComm)
+      : fileHandler().comm()
+    );
+
     const fileName rawName(is);
     const fileName fName(resolveEtcFile(rawName, parentDict));
 
@@ -127,9 +138,13 @@ bool Foam::functionEntries::includeEtcEntry::execute
             Info<< fName << nl;
         }
         parentDict.read(ifs);
+
+        fileHandler().comm(oldComm);
         return true;
     }
 
+    fileHandler().comm(oldComm);
+
     if (!mandatory)
     {
         return true; // Never fails if optional
@@ -153,6 +168,15 @@ bool Foam::functionEntries::includeEtcEntry::execute
     Istream& is
 )
 {
+    const regIOobject* rioPtr = isA<regIOobject>(parentDict.topDict());
+
+    const label oldComm
+    (
+        rioPtr && rioPtr->global()
+      ? fileHandler().comm(UPstream::worldComm)
+      : fileHandler().comm()
+    );
+
     const fileName rawName(is);
     const fileName fName(resolveEtcFile(rawName, parentDict));
 
@@ -167,9 +191,13 @@ bool Foam::functionEntries::includeEtcEntry::execute
             Info<< fName << nl;
         }
         entry.read(parentDict, ifs);
+
+        fileHandler().comm(oldComm);
         return true;
     }
 
+    fileHandler().comm(oldComm);
+
     if (!mandatory)
     {
         return true; // Never fails if optional
diff --git a/src/OpenFOAM/global/fileOperations/masterUncollatedFileOperation/masterUncollatedFileOperation.C b/src/OpenFOAM/global/fileOperations/masterUncollatedFileOperation/masterUncollatedFileOperation.C
index f7781c32f2..f48b12c8d6 100644
--- a/src/OpenFOAM/global/fileOperations/masterUncollatedFileOperation/masterUncollatedFileOperation.C
+++ b/src/OpenFOAM/global/fileOperations/masterUncollatedFileOperation/masterUncollatedFileOperation.C
@@ -2766,30 +2766,42 @@ void Foam::fileOperations::masterUncollatedFileOperation::addWatches
 {
     const labelList& watchIndices = rio.watchIndices();
 
+    // Do on master and distribute effect to subprocs such that after
+    // all have consistent numbering & files
+
     DynamicList<label> newWatchIndices;
-    labelHashSet removedWatches(watchIndices);
-
-    for (const fileName& f : files)
+    if (UPstream::master())
     {
-        const label index = findWatch(watchIndices, f);
+        // Switch off comms inside findWatch/addWatch etc.
+        const bool oldParRun = UPstream::parRun(false);
 
-        if (index == -1)
-        {
-            newWatchIndices.append(addWatch(f));
-        }
-        else
-        {
-            // Existing watch
-            newWatchIndices.append(watchIndices[index]);
-            removedWatches.erase(index);
-        }
-    }
+        labelHashSet removedWatches(watchIndices);
 
-    // Remove any unused watches
-    for (const label index : removedWatches)
-    {
-        removeWatch(watchIndices[index]);
+        for (const fileName& f : files)
+        {
+            const label index = findWatch(watchIndices, f);
+
+            if (index == -1)
+            {
+                newWatchIndices.push_back(addWatch(f));
+            }
+            else
+            {
+                // Existing watch
+                newWatchIndices.push_back(watchIndices[index]);
+                removedWatches.erase(index);
+            }
+        }
+
+        // Remove any unused watches
+        for (const label index : removedWatches)
+        {
+            removeWatch(watchIndices[index]);
+        }
+
+        UPstream::parRun(oldParRun);
     }
+    Pstream::broadcast(newWatchIndices);
 
     rio.watchIndices() = newWatchIndices;
 }