fileHandler: Added flush method

This method waits until all the threads have completed IO operations and
then clears any cached information about the files on disk. This
replaces the deactivation of threading by means of zeroing the buffer
size when writing and reading of a file happen in sequence. It also
allows paraFoam to update the list of available times.

Patch contributed by Mattijs Janssens
Resolves bug report https://bugs.openfoam.org/view.php?id=2962

applications/solvers/multiphase/compressibleInterFoam/twoPhaseMixtureThermo/twoPhaseMixtureThermo.C

@@ -50,12 +50,6 @@ Foam::twoPhaseMixtureThermo::twoPhaseMixtureThermo
     thermo1_(nullptr),
     thermo2_(nullptr)
 {
-    // Note: we're writing files to be read in immediately afterwards.
-    //       Avoid any thread-writing problems.
-    float bufSz =
-        fileOperations::collatedFileOperation::maxThreadFileBufferSize;
-    fileOperations::collatedFileOperation::maxThreadFileBufferSize = 0;
-
     {
         volScalarField T1
         (
@@ -86,9 +80,9 @@ Foam::twoPhaseMixtureThermo::twoPhaseMixtureThermo
         T2.write();
     }
 
-    fileOperations::collatedFileOperation::maxThreadFileBufferSize =
+    // Note: we're writing files to be read in immediately afterwards.
-        bufSz;
+    //       Avoid any thread-writing problems.
-
+    fileHandler().flush();
 
     thermo1_ = rhoThermo::New(U.mesh(), phase1Name());
     thermo2_ = rhoThermo::New(U.mesh(), phase2Name());

applications/utilities/mesh/manipulation/setSet/setSet.C

@@ -2,7 +2,7 @@
   =========                 |
   \\      /  F ield         | OpenFOAM: The Open Source CFD Toolbox
    \\    /   O peration     |
-    \\  /    A nd           | Copyright (C) 2011-2017 OpenFOAM Foundation
+    \\  /    A nd           | Copyright (C) 2011-2018 OpenFOAM Foundation
      \\/     M anipulation  |
 -------------------------------------------------------------------------------
 License
@@ -48,7 +48,6 @@ Description
 #include "faceZoneSet.H"
 #include "pointZoneSet.H"
 #include "timeSelector.H"
-#include "collatedFileOperation.H"
 
 #include <stdio.h>
 
@@ -349,6 +348,8 @@ void removeZone
         zones.setSize(zones.size()-1);
         zones.clearAddressing();
         zones.write();
+        // Force flushing so we know it has finished writing
+        fileHandler().flush();
     }
 }
 
@@ -603,6 +604,8 @@ bool doCommand
                     currentSet.instance() = mesh.time().timeName();
                 }
                 currentSet.write();
+                // Make sure writing is finished
+                fileHandler().flush();
             }
         }
     }
@@ -809,8 +812,6 @@ int main(int argc, char *argv[])
     // Specific to topoSet/setSet: quite often we want to block upon writing
     // a set so we can immediately re-read it. So avoid use of threading
     // for set writing.
-    fileOperations::collatedFileOperation::maxThreadFileBufferSize = 0;
-
     timeSelector::addOptions(true, false);
     #include "addRegionOption.H"
     argList::addBoolOption("noVTK", "do not write VTK files");

applications/utilities/mesh/manipulation/topoSet/topoSet.C

@@ -2,7 +2,7 @@
   =========                 |
   \\      /  F ield         | OpenFOAM: The Open Source CFD Toolbox
    \\    /   O peration     |
-    \\  /    A nd           | Copyright (C) 2011-2017 OpenFOAM Foundation
+    \\  /    A nd           | Copyright (C) 2011-2018 OpenFOAM Foundation
      \\/     M anipulation  |
 -------------------------------------------------------------------------------
 License
@@ -40,7 +40,6 @@ Description
 #include "faceZoneSet.H"
 #include "pointZoneSet.H"
 #include "IOdictionary.H"
-#include "collatedFileOperation.H"
 
 using namespace Foam;
 
@@ -85,6 +84,7 @@ void removeZone
         zones.setSize(zones.size()-1);
         zones.clearAddressing();
         zones.write();
+        fileHandler().flush();
     }
 }
 
@@ -193,11 +193,6 @@ polyMesh::readUpdateState meshReadUpdate(polyMesh& mesh)
 
 int main(int argc, char *argv[])
 {
-    // Specific to topoSet/setSet: quite often we want to block upon writing
-    // a set so we can immediately re-read it. So avoid use of threading
-    // for set writing.
-    fileOperations::collatedFileOperation::maxThreadFileBufferSize = 0;
-
     timeSelector::addOptions(true, false);
     #include "addDictOption.H"
     #include "addRegionOption.H"
@@ -302,6 +297,7 @@ int main(int argc, char *argv[])
                     // Synchronize for coupled patches.
                     if (!noSync) currentSet().sync(mesh);
                     currentSet().write();
+                    fileHandler().flush();
                 }
                 break;
 
@@ -336,6 +332,7 @@ int main(int argc, char *argv[])
                     // Synchronize for coupled patches.
                     if (!noSync) currentSet().sync(mesh);
                     currentSet().write();
+                    fileHandler().flush();
                 }
                 break;
 
@@ -343,12 +340,14 @@ int main(int argc, char *argv[])
                     Info<< "    Clearing " << currentSet().type() << endl;
                     currentSet().clear();
                     currentSet().write();
+                    fileHandler().flush();
                 break;
 
                 case topoSetSource::INVERT:
                     Info<< "    Inverting " << currentSet().type() << endl;
                     currentSet().invert(currentSet().maxSize(mesh));
                     currentSet().write();
+                    fileHandler().flush();
                 break;
 
                 case topoSetSource::REMOVE:

applications/utilities/parallelProcessing/decomposePar/decomposePar.C

@@ -104,7 +104,6 @@ Usage
 #include "pointFieldDecomposer.H"
 #include "lagrangianFieldDecomposer.H"
 #include "decompositionModel.H"
-#include "collatedFileOperation.H"
 
 // * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
 
@@ -459,12 +458,6 @@ int main(int argc, char *argv[])
         // Decompose the mesh
         if (!decomposeFieldsOnly)
         {
-            // Disable buffering when writing mesh since we need to read
-            // it later on when decomposing the fields
-            float bufSz =
-                fileOperations::collatedFileOperation::maxThreadFileBufferSize;
-            fileOperations::collatedFileOperation::maxThreadFileBufferSize = 0;
-
             mesh.decomposeMesh(dictIO.objectPath());
 
             mesh.writeDecomposition(decomposeSets);
@@ -521,8 +514,7 @@ int main(int argc, char *argv[])
                     << endl;
             }
 
-            fileOperations::collatedFileOperation::maxThreadFileBufferSize =
+            fileHandler().flush();
-                bufSz;
         }
 
 

applications/utilities/postProcessing/graphics/PVReaders/vtkPVFoam/vtkPVFoam.C

@@ -97,7 +97,8 @@ int Foam::vtkPVFoam::setTime(int nRequest, const double requestTimes[])
 {
     Time& runTime = dbPtr_();
 
-    // Get times list
+    // Get times list. Flush first to force refresh.
+    fileHandler().flush();
     instantList Times = runTime.times();
 
     int nearestIndex = timeIndex_;
@@ -248,11 +249,6 @@ Foam::vtkPVFoam::vtkPVFoam
 
     fileName FileName(vtkFileName);
 
-    // Make sure not to use the threaded version - it does not like
-    // being loaded as a shared library - static cleanup order is problematic.
-    // For now just disable the threaded writer.
-    fileOperations::collatedFileOperation::maxThreadFileBufferSize = 0;
-
     // avoid argList and get rootPath/caseName directly from the file
     fileName fullCasePath(FileName.path());
 
@@ -575,6 +571,8 @@ double* Foam::vtkPVFoam::findTimes(int& nTimeSteps)
     if (dbPtr_.valid())
     {
         Time& runTime = dbPtr_();
+        // Get times list. Flush first to force refresh.
+        fileHandler().flush();
         instantList timeLst = runTime.times();
 
         // find the first time for which this mesh appears to exist

applications/utilities/postProcessing/graphics/PVReaders/vtkPVFoam/vtkPVFoamUpdateInfo.C

@@ -2,7 +2,7 @@
   =========                 |
   \\      /  F ield         | OpenFOAM: The Open Source CFD Toolbox
    \\    /   O peration     |
-    \\  /    A nd           | Copyright (C) 2011-2017 OpenFOAM Foundation
+    \\  /    A nd           | Copyright (C) 2011-2018 OpenFOAM Foundation
      \\/     M anipulation  |
 -------------------------------------------------------------------------------
 License
@@ -192,6 +192,9 @@ void Foam::vtkPVFoam::updateInfoLagrangian
 
     // Generate a list of lagrangian clouds across all times
     HashSet<fileName> cloudDirs;
+
+    // Get times list. Flush first to force refresh.
+    fileHandler().flush();
     instantList times = dbPtr_().times();
     forAll(times, timei)
     {
@@ -706,6 +709,8 @@ void Foam::vtkPVFoam::updateInfoLagrangianFields()
     // set. ParaView will display "(partial)" after field names that only apply
     // to some of the clouds.
     const arrayRange& range = arrayRangeLagrangian_;
+
+    fileHandler().flush();
     for (label partId = range.start(); partId < range.end(); ++ partId)
     {
         const instantList times = dbPtr_().times();

applications/utilities/postProcessing/graphics/PVReaders/vtkPVblockMesh/vtkPVblockMesh.C

@@ -31,7 +31,6 @@ License
 #include "Time.H"
 #include "patchZones.H"
 #include "OStringStream.H"
-#include "collatedFileOperation.H"
 
 // VTK includes
 #include "vtkDataArraySelection.h"
@@ -171,12 +170,6 @@ Foam::vtkPVblockMesh::vtkPVblockMesh
             << FileName << endl;
     }
 
-    // Make sure not to use the threaded version - it does not like
-    // being loaded as a shared library - static cleanup order is problematic.
-    // For now just disable the threaded writer.
-    fileOperations::collatedFileOperation::maxThreadFileBufferSize = 0;
-
-
     // avoid argList and get rootPath/caseName directly from the file
     fileName fullCasePath(fileName(FileName).path());
 

src/OpenFOAM/global/fileOperations/collatedFileOperation/collatedFileOperation.C

@@ -608,6 +608,18 @@ bool Foam::fileOperations::collatedFileOperation::writeObject
     }
 }
 
+void Foam::fileOperations::collatedFileOperation::flush() const
+{
+    if (debug)
+    {
+        Pout<< "collatedFileOperation::flush : clearing and waiting for thread"
+            << endl;
+    }
+    masterUncollatedFileOperation::flush();
+    // Wait for thread to finish (note: also removes thread)
+    writer_.waitAll();
+}
+
 
 Foam::word Foam::fileOperations::collatedFileOperation::processorsDir
 (

src/OpenFOAM/global/fileOperations/collatedFileOperation/collatedFileOperation.H

@@ -155,6 +155,9 @@ public:
 
         // Other
 
+            //- Forcibly wait until all output done. Flush any cached data
+            virtual void flush() const;
+
             //- Actual name of processors dir
             virtual word processorsDir(const IOobject&) const;
 

src/OpenFOAM/global/fileOperations/fileOperation/fileOperation.C

@@ -982,6 +982,17 @@ Foam::label Foam::fileOperation::nProcs
 }
 
 
+void Foam::fileOperation::flush() const
+{
+    if (debug)
+    {
+        Pout<< "fileOperation::flush : clearing processor directories cache"
+            << endl;
+    }
+    procsDirs_.clear();
+}
+
+
 Foam::fileName Foam::fileOperation::processorsCasePath
 (
     const IOobject& io,

src/OpenFOAM/global/fileOperations/fileOperation/fileOperation.H

@@ -485,6 +485,9 @@ public:
             virtual void setTime(const Time&) const
             {}
 
+            //- Forcibly wait until all output done. Flush any cached data
+            virtual void flush() const;
+
             //- Generate path (like io.path) from root+casename with any
             //  'processorXXX' replaced by procDir (usually 'processsors')
             fileName processorsCasePath

src/OpenFOAM/global/fileOperations/masterUncollatedFileOperation/masterUncollatedFileOperation.C

@@ -2532,6 +2532,13 @@ Foam::fileOperations::masterUncollatedFileOperation::NewOFstream
 }
 
 
+void Foam::fileOperations::masterUncollatedFileOperation::flush() const
+{
+    fileOperation::flush();
+    times_.clear();
+}
+
+
 Foam::label Foam::fileOperations::masterUncollatedFileOperation::addWatch
 (
     const fileName& fName

src/OpenFOAM/global/fileOperations/masterUncollatedFileOperation/masterUncollatedFileOperation.H

@@ -752,6 +752,9 @@ public:
             //- Callback for time change
             virtual void setTime(const Time&) const;
 
+            //- Forcibly wait until all output done. Flush any cached data
+            virtual void flush() const;
+
             //- Return cached times
             const HashPtrTable<instantList>& times() const
             {