Merge branch 'externalCoupled' into 'master'

External coupled

Improvements and bugfixes for externalCoupledFunctionObject - issues #278 #269

See merge request !72

src/postProcessing/functionObjects/jobControl/externalCoupled/externalCoupledFunctionObject.C

@@ -49,7 +49,54 @@ namespace Foam
 
 Foam::word Foam::externalCoupledFunctionObject::lockName = "OpenFOAM";
 
-Foam::string Foam::externalCoupledFunctionObject::patchKey = "# Patch: ";
+Foam::string Foam::externalCoupledFunctionObject::patchKey = "// Patch:";
+
+template<>
+const char* Foam::NamedEnum
+<
+    Foam::externalCoupledFunctionObject::stateEnd,
+    2
+>::names[] =
+{
+    "remove",
+    "done"
+};
+
+const Foam::NamedEnum
+<
+    Foam::externalCoupledFunctionObject::stateEnd,
+    2
+> Foam::externalCoupledFunctionObject::stateEndNames_;
+
+
+namespace Foam
+{
+//! \cond fileScope
+//- Write list content with size, bracket, content, bracket one-per-line.
+//  This makes for consistent for parsing, regardless of the list length.
+template <class T>
+static void writeList(Ostream& os, const string& header, const UList<T>& L)
+{
+    // Header string
+    os  << header.c_str() << nl;
+
+    // Write size and start delimiter
+    os  << L.size() << nl
+        << token::BEGIN_LIST;
+
+    // Write contents
+    forAll(L, i)
+    {
+        os << nl << L[i];
+    }
+
+    // Write end delimiter
+    os << nl << token::END_LIST << nl << endl;
+}
+//! \endcond
+
+}
+// namespace Foam
 
 
 // * * * * * * * * * * * * * Private Member Functions  * * * * * * * * * * * //
@@ -88,7 +135,7 @@ Foam::fileName Foam::externalCoupledFunctionObject::lockFile() const
 }
 
 
-void Foam::externalCoupledFunctionObject::createLockFile() const
+void Foam::externalCoupledFunctionObject::useMaster() const
 {
     if (!Pstream::master())
     {
@@ -104,13 +151,13 @@ void Foam::externalCoupledFunctionObject::createLockFile() const
         if (log_) Info<< type() << ": creating lock file" << endl;
 
         OFstream os(fName);
-        os  << "lock file";
+        os  << "status=openfoam\n";
         os.flush();
     }
 }
 
 
-void Foam::externalCoupledFunctionObject::removeLockFile() const
+void Foam::externalCoupledFunctionObject::useSlave() const
 {
     if (!Pstream::master())
     {
@@ -119,7 +166,35 @@ void Foam::externalCoupledFunctionObject::removeLockFile() const
 
     if (log_) Info<< type() << ": removing lock file" << endl;
 
-    rm(lockFile());
+    Foam::rm(lockFile());
+}
+
+
+void Foam::externalCoupledFunctionObject::cleanup() const
+{
+    if (!Pstream::master())
+    {
+        return;
+    }
+
+    const fileName lck(lockFile());
+    switch (stateEnd_)
+    {
+        case REMOVE:
+            {
+                if (log_) Info<< type() << ": removing lock file" << endl;
+                Foam::rm(lck);
+            }
+            break;
+        case DONE:
+            {
+                if (log_) Info<< type() << ": lock file status=done" << endl;
+                OFstream os(lck);
+                os  << "status=done\n";
+                os.flush();
+            }
+            break;
+    }
 }
 
 
@@ -189,30 +264,29 @@ void Foam::externalCoupledFunctionObject::removeWriteFiles() const
 }
 
 
-void Foam::externalCoupledFunctionObject::wait() const
+void Foam::externalCoupledFunctionObject::waitForSlave() const
 {
     const fileName fName(lockFile());
-    label found = 0;
     label totalTime = 0;
+    bool found = false;
 
     if (log_) Info<< type() << ": beginning wait for lock file " << fName << nl;
 
-    while (found == 0)
+    while (!found)
     {
         if (Pstream::master())
         {
             if (totalTime > timeOut_)
             {
                 FatalErrorInFunction
-                    << "Wait time exceeded time out time of " << timeOut_
+                    << "Wait time exceeded timeout of " << timeOut_
                     << " s" << abort(FatalError);
             }
 
             IFstream is(fName);
-
             if (is.good())
             {
-                found++;
+                found = true;
 
                 if (log_)
                 {
@@ -232,7 +306,7 @@ void Foam::externalCoupledFunctionObject::wait() const
         }
 
         // prevent other procs from racing ahead
-        reduce(found, sumOp<label>());
+        reduce(found, orOp<bool>());
     }
 }
 
@@ -387,11 +461,6 @@ void Foam::externalCoupledFunctionObject::writeGeometry
 
     fileName dir(groupDir(commsDir, compositeName(regionNames), groupName));
 
-    //if (log_)
-    {
-        Info<< typeName << ": writing geometry to " << dir << endl;
-    }
-
     autoPtr<OFstream> osPointsPtr;
     autoPtr<OFstream> osFacesPtr;
     if (Pstream::master())
@@ -399,12 +468,20 @@ void Foam::externalCoupledFunctionObject::writeGeometry
         mkDir(dir);
         osPointsPtr.reset(new OFstream(dir/"patchPoints"));
         osFacesPtr.reset(new OFstream(dir/"patchFaces"));
+
+        osPointsPtr() << "// Group: " << groupName << endl;
+        osFacesPtr()  << "// Group: " << groupName << endl;
+
+        Info<< typeName << ": writing geometry to " << dir << endl;
     }
 
+    // Individual region/patch entries
 
-    DynamicList<face> allMeshesFaces;
+    DynamicList<face> allFaces;
-    DynamicField<point> allMeshesPoints;
+    DynamicField<point> allPoints;
 
+    labelList pointToGlobal;
+    labelList uniquePointIDs;
     forAll(meshes, meshI)
     {
         const fvMesh& mesh = meshes[meshI];
@@ -417,109 +494,61 @@ void Foam::externalCoupledFunctionObject::writeGeometry
             ).sortedToc()
         );
 
-        // Count faces
-        label nFaces = 0;
-        forAll(patchIDs, i)
-        {
-            nFaces += mesh.boundaryMesh()[patchIDs[i]].size();
-        }
-
-        // Collect faces
-        DynamicList<label> allFaceIDs(nFaces);
         forAll(patchIDs, i)
         {
             const polyPatch& p = mesh.boundaryMesh()[patchIDs[i]];
 
-            forAll(p, pI)
+            mesh.globalData().mergePoints
+            (
+                p.meshPoints(),
+                p.meshPointMap(),
+                pointToGlobal,
+                uniquePointIDs
+            );
+
+            label procI = Pstream::myProcNo();
+
+            List<pointField> collectedPoints(Pstream::nProcs());
+            collectedPoints[procI] = pointField(mesh.points(), uniquePointIDs);
+            Pstream::gatherList(collectedPoints);
+
+            List<faceList> collectedFaces(Pstream::nProcs());
+            faceList& patchFaces = collectedFaces[procI];
+            patchFaces = p.localFaces();
+            forAll(patchFaces, faceI)
             {
-                allFaceIDs.append(p.start()+pI);
+                inplaceRenumber(pointToGlobal, patchFaces[faceI]);
             }
-        }
+            Pstream::gatherList(collectedFaces);
 
-        // Construct overall patch
+            if (Pstream::master())
-        indirectPrimitivePatch allPatch
-        (
-            IndirectList<face>(mesh.faces(), allFaceIDs),
-            mesh.points()
-        );
-
-        labelList pointToGlobal;
-        labelList uniquePointIDs;
-        mesh.globalData().mergePoints
-        (
-            allPatch.meshPoints(),
-            allPatch.meshPointMap(),
-            pointToGlobal,
-            uniquePointIDs
-        );
-
-        label procI = Pstream::myProcNo();
-
-        List<pointField> collectedPoints(Pstream::nProcs());
-        collectedPoints[procI] = pointField(mesh.points(), uniquePointIDs);
-        Pstream::gatherList(collectedPoints);
-
-        List<faceList> collectedFaces(Pstream::nProcs());
-        faceList& patchFaces = collectedFaces[procI];
-        patchFaces = allPatch.localFaces();
-        forAll(patchFaces, faceI)
-        {
-            inplaceRenumber(pointToGlobal, patchFaces[faceI]);
-        }
-        Pstream::gatherList(collectedFaces);
-
-        if (Pstream::master())
-        {
-            // Append and renumber
-            label nPoints = allMeshesPoints.size();
-
-            forAll(collectedPoints, procI)
             {
-                allMeshesPoints.append(collectedPoints[procI]);
+                allPoints.clear();
+                allFaces.clear();
 
-            }
+                for (label procI=0; procI < Pstream::nProcs(); ++procI)
-            face newFace;
-            forAll(collectedFaces, procI)
-            {
-                const faceList& procFaces = collectedFaces[procI];
-
-                forAll(procFaces, faceI)
                 {
-                    const face& f = procFaces[faceI];
+                    allPoints.append(collectedPoints[procI]);
-
+                    allFaces.append(collectedFaces[procI]);
-                    newFace.setSize(f.size());
-                    forAll(f, fp)
-                    {
-                        newFace[fp] = f[fp]+nPoints;
-                    }
-                    allMeshesFaces.append(newFace);
                 }
 
-                nPoints += collectedPoints[procI].size();
+                Info<< typeName << ": mesh " << mesh.name()
+                    << ", patch " << p.name()
+                    << ": writing " << allPoints.size() << " points to "
+                    << osPointsPtr().name() << nl
+                    << typeName << ": mesh " << mesh.name()
+                    << ", patch " << p.name()
+                    << ": writing " << allFaces.size() << " faces to "
+                    << osFacesPtr().name() << endl;
+
+                // The entry name (region / patch)
+                const string entryHeader =
+                    patchKey + ' ' + mesh.name() + ' ' + p.name();
+
+                writeList(osPointsPtr(), entryHeader, allPoints);
+                writeList(osFacesPtr(),  entryHeader, allFaces);
             }
         }
-
-        //if (log_)
-        {
-            Info<< typeName << ": for mesh " << mesh.name()
-                << " writing " << allMeshesPoints.size() << " points to "
-                << osPointsPtr().name() << endl;
-            Info<< typeName << ": for mesh " << mesh.name()
-                << " writing " << allMeshesFaces.size() << " faces to "
-                << osFacesPtr().name() << endl;
-        }
-    }
-
-    // Write points
-    if (osPointsPtr.valid())
-    {
-        osPointsPtr() << allMeshesPoints << endl;
-    }
-
-    // Write faces
-    if (osFacesPtr.valid())
-    {
-        osFacesPtr() << allMeshesFaces << endl;
     }
 }
 
@@ -541,7 +570,7 @@ Foam::word Foam::externalCoupledFunctionObject::compositeName
         {
             // For compatibility with single region cases suppress single
             // region name
-            return word("");
+            return word::null;
         }
         else
         {
@@ -768,7 +797,7 @@ void Foam::externalCoupledFunctionObject::initialise()
     if (initByExternal_)
     {
         // Wait for initial data to be made available
-        wait();
+        waitForSlave();
 
         // Read data passed back from external source
         readData();
@@ -790,6 +819,7 @@ Foam::externalCoupledFunctionObject::externalCoupledFunctionObject
     functionObject(name),
     time_(runTime),
     enabled_(true),
+    stateEnd_(REMOVE),
     initialised_(false)
 {
     read(dict);
@@ -801,7 +831,7 @@ Foam::externalCoupledFunctionObject::externalCoupledFunctionObject
 
     if (!initByExternal_)
     {
-        createLockFile();
+        useMaster();
     }
 }
 
@@ -809,7 +839,12 @@ Foam::externalCoupledFunctionObject::externalCoupledFunctionObject
 // * * * * * * * * * * * * * * * * Destructor  * * * * * * * * * * * * * * * //
 
 Foam::externalCoupledFunctionObject::~externalCoupledFunctionObject()
-{}
+{
+    if (enabled())
+    {
+        cleanup();
+    }
+}
 
 
 // * * * * * * * * * * * * * * * Member Functions  * * * * * * * * * * * * * //
@@ -846,11 +881,12 @@ bool Foam::externalCoupledFunctionObject::execute(const bool forceWrite)
         // Write data for external source
         writeData();
 
-        // remove lock file, signalling external source to execute
+        // Signal external source to execute (by removing lock file)
-        removeLockFile();
+        // - Wait for slave to provide data
+        useSlave();
 
         // Wait for response
-        wait();
+        waitForSlave();
 
         // Remove old data files from OpenFOAM
         removeWriteFiles();
@@ -858,8 +894,8 @@ bool Foam::externalCoupledFunctionObject::execute(const bool forceWrite)
         // Read data passed back from external source
         readData();
 
-        // create lock file for external source
+        // Signal external source to wait (by creating the lock file)
-        createLockFile();
+        useMaster();
 
         return true;
     }
@@ -877,7 +913,7 @@ bool Foam::externalCoupledFunctionObject::end()
         // Remove old data files
         removeReadFiles();
         removeWriteFiles();
-        removeLockFile();
+        cleanup();
     }
 
     return true;
@@ -906,13 +942,19 @@ bool Foam::externalCoupledFunctionObject::read(const dictionary& dict)
         return true;
     }
 
+    calcFrequency_  = dict.lookupOrDefault("calcFrequency", 1);
+
     dict.lookup("commsDir") >> commsDir_;
     commsDir_.expand();
+    commsDir_.clean();
 
-    waitInterval_ = dict.lookupOrDefault("waitInterval", 1);
+    waitInterval_   = dict.lookupOrDefault("waitInterval", 1);
-    timeOut_ = dict.lookupOrDefault("timeOut", 100*waitInterval_);
+    timeOut_        = dict.lookupOrDefault("timeOut", 100*waitInterval_);
-    calcFrequency_ = dict.lookupOrDefault("calcFrequency", 1);
     initByExternal_ = readBool(dict.lookup("initByExternal"));
+    // initByExternal_ = dict.lookupOrDefault<Switch>("initByExternal", false);
+    stateEnd_       =
+        stateEndNames_[dict.lookupOrDefault<word>("stateEnd", "remove")];
+
     log_ = dict.lookupOrDefault("log", false);
 
 
@@ -920,7 +962,6 @@ bool Foam::externalCoupledFunctionObject::read(const dictionary& dict)
     wordList allRegionNames(time_.lookupClass<fvMesh>().sortedToc());
 
 
-
     const dictionary& allRegionsDict = dict.subDict("regions");
 
     forAllConstIter(dictionary, allRegionsDict, iter)

src/postProcessing/functionObjects/jobControl/externalCoupled/externalCoupledFunctionObject.H

@@ -2,7 +2,7 @@
   =========                 |
   \\      /  F ield         | OpenFOAM: The Open Source CFD Toolbox
    \\    /   O peration     |
-    \\  /    A nd           | Copyright (C) 2015 OpenCFD Ltd.
+    \\  /    A nd           | Copyright (C) 2015-2016 OpenCFD Ltd.
      \\/     M anipulation  |
 -------------------------------------------------------------------------------
 License
@@ -82,6 +82,7 @@ Description
         log             yes;
         commsDir        "${FOAM_CASE}/comms";
         initByExternal  yes;
+        stateEnd        remove;  // (remove | done)
 
         regions
         {
@@ -122,9 +123,11 @@ SourceFiles
 #include "DynamicList.H"
 #include "wordReList.H"
 #include "scalarField.H"
+#include "NamedEnum.H"
 #include "Switch.H"
 #include "UPtrList.H"
 
+
 // * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
 
 namespace Foam
@@ -145,6 +148,23 @@ class externalCoupledFunctionObject
 :
     public functionObject
 {
+public:
+
+    // Public data types
+
+        //- Lockfile state on termination
+        enum stateEnd
+        {
+            REMOVE,     //!< Remove lock file on end
+            DONE        //!< Lock file contains status=done on end
+        };
+
+private:
+
+        //- State end names
+        static const NamedEnum<stateEnd, 2> stateEndNames_;
+
+
     // Private data
 
         //- Reference to the time database
@@ -168,6 +188,9 @@ class externalCoupledFunctionObject
         //- Flag to indicate values are initialised by external application
         bool initByExternal_;
 
+        //- Lockfile state on termination
+        stateEnd stateEnd_;
+
         //- Log flag
         bool log_;
 
@@ -209,11 +232,15 @@ class externalCoupledFunctionObject
         //- Return the file path to the lock file
         fileName lockFile() const;
 
-        //- Create lock file
-        void createLockFile() const;
 
-        //- Remove lock file
+        //- Create lock file to indicate that OpenFOAM is in charge
-        void removeLockFile() const;
+        void useMaster() const;
+
+        //- Remove lock file to indicate that the external program is in charge
+        void useSlave() const;
+
+        //- Remove lock file or status=done in lock.
+        void cleanup() const;
 
         //- Remove files written by OpenFOAM
         void removeWriteFiles() const;
@@ -221,8 +248,9 @@ class externalCoupledFunctionObject
         //- Remove files written by external code
         void removeReadFiles() const;
 
-        //- Wait for response from external source
+        //- Wait for indication that the external program has supplied input
-        void wait() const;
+        //  (ie, for the lock file to reappear).
+        void waitForSlave() const;
 
 
         //- Read data for a single region, single field
@@ -291,10 +319,10 @@ public:
     //- Runtime type information
     TypeName("externalCoupled");
 
-    //- Name of lock file
+    //- Name of lock file (normally 'OpenFOAM.lock')
     static word lockName;
 
-    //- Name of patch key, e.g. '# Patch:' when looking for start of patch data
+    //- Name of patch key, e.g. '// Patch:' when looking for start of patch data
     static string patchKey;
 
 
@@ -363,7 +391,7 @@ public:
             //  separated by '_'
             static word compositeName(const wordList&);
 
-            //- Write geometry for the group/patch
+            //- Write geometry for the group as region/patch
             static void writeGeometry
             (
                 const UPtrList<const fvMesh>& meshes,

tutorials/heatTransfer/chtMultiRegionFoam/externalCoupledMultiRegionHeater/Allrun

@@ -15,6 +15,12 @@ cd ${0%/*} || exit 1    # Run from this directory
 # Decompose
 runApplication decomposePar -allRegions
 
+## Can verify parallel operation of createExternalCoupledPatchGeometry
+# \rm -f log.createExternalCoupledPatchGeometry
+# runParallel createExternalCoupledPatchGeometry \
+#     -regions '(topAir heater)' coupleGroup \
+#     -commsDir $PWD/comms
+
 # Run OpenFOAM
 runParallel $(getApplication) &
 

tutorials/heatTransfer/chtMultiRegionFoam/externalCoupledMultiRegionHeater/Allrun.pre

@@ -28,6 +28,6 @@ runApplication createExternalCoupledPatchGeometry \
 echo
 echo "creating files for paraview post-processing"
 echo
-paraFoam -touchAll
+paraFoam -touchAll 2>/dev/null
 
 # ----------------------------------------------------------------- end-of-file

tutorials/heatTransfer/chtMultiRegionFoam/externalCoupledMultiRegionHeater/externalSolver

@@ -20,13 +20,17 @@ fieldName="T"
 
 lockFile="${commsDir}/OpenFOAM.lock"
 dataFile="${commsDir}/${regionGroupName}/${patchGroupName}/${fieldName}"
-waitSec=1
+waitSec=5
-timeOut=10
+timeOut=100
-nSteps=200  # maximum number of time steps. Note: should be more than
+nSteps=1000 # maximum number of time steps. Note: should be more than
             # number of iterations on the OpenFOAM side
 refGrad=0
 valueFraction=1
 
+
+# Remove any old junk
+\rm -f $lockFile 2>/dev/null
+
 log()
 {
     echo "External: $@"
@@ -53,7 +57,7 @@ init()
         echo "$refValue2 $refGrad $valueFraction" >> "${dataFile}.in"
     done
 
-    # create lock file to pass control to OF
+    # Create lock file to pass control to OpenFOAM
     touch ${lockFile}
 }
 
@@ -61,7 +65,6 @@ init()
 # create the comms directory
 mkdir -p ${commsDir}/${regionGroupName}/${patchGroupName}
 
-
 # tutorial case employs the 'initByExternalOption', so we need to provide
 # the initial values
 init
@@ -69,11 +72,24 @@ init
 
 totalWait=0
 step=0
-while [ $step -lt $nSteps ]; do
+while [ $step -lt $nSteps ]
-    if [ -f $lockFile ]; then
+do
-        log "found lock file ${lockFile} - waiting"
+    if [ -f $lockFile ]
+    then
+        if grep -q "status=done" ${lockFile}
+        then
+             log "found lock file ${lockFile} with 'status=done' - finished"
+             break
+        elif [ -s $lockFile ]
+        then
+             log "found lock file ${lockFile} containing '$(cat $lockFile)' - waiting"
+        else
+             log "found lock file ${lockFile} - waiting"
+        fi
+
         totalWait=$(expr $totalWait + $waitSec)
-        if [ $totalWait -gt $timeOut ]; then
+        if [ $totalWait -gt $timeOut ]
+        then
             log "timeout"
             break
         else
@@ -91,7 +107,7 @@ while [ $step -lt $nSteps ]; do
         log "updating ${dataFile}.in from ${dataFile}.out"
 
         awk '{if( $1 != "#" ){print $1+1 " 0 1"}}' \
-            ${dataFile}.out | tee ${dataFile}.in
+            ${dataFile}.out >| ${dataFile}.in
 
         log "creating lock file ${lockFile}"
         touch ${lockFile}
@@ -100,5 +116,7 @@ done
 
 log "done"
 
+# Remove the lock file too
+\rm -f $lockFile 2>/dev/null
 
 #------------------------------------------------------------------------------

tutorials/heatTransfer/chtMultiRegionFoam/externalCoupledMultiRegionHeater/system/controlDict

@@ -26,11 +26,12 @@ startTime       0;
 
 stopAt          endTime;
 
-endTime         1;
+endTime         0.5;
 
 deltaT          0.001;
 
 writeControl    adjustableRunTime;
+
 writeInterval   0.1;
 
 purgeWrite      0;
@@ -56,40 +57,7 @@ adjustTimeStep  yes;
 
 functions
 {
-    externalCoupled
+    #include "externalCoupled"
-    {
-        // Where to load it from (if not already in solver)
-        functionObjectLibs ("libjobControl.so");
-
-        type            externalCoupled;
-
-        // Directory to use for communication
-        commsDir        "${FOAM_CASE}/comms";
-
-        // Does external process start first
-        initByExternal  true;
-
-        // Additional output
-        log             true;
-
-        regions
-        {
-            // Region name (wildcards allowed)
-            "(topAir|heater)"
-            {
-                // In topAir adjust the minX patch (fixedValue)
-
-                // Patch or patchGroup
-                coupleGroup
-                {
-                    // Fields to output in commsDir
-                    writeFields (T);
-                    // Fields to read from commsDir
-                    readFields  (T);
-                }
-            }
-        }
-    }
 }
 
 

tutorials/heatTransfer/chtMultiRegionFoam/externalCoupledMultiRegionHeater/system/externalCoupled

@@ -0,0 +1,43 @@
+// -*- C++ -*-
+
+// control for external coupled simulation
+externalCoupled
+{
+    // Where to load it from (if not already in solver)
+    functionObjectLibs ("libjobControl.so");
+
+    type            externalCoupled;
+
+    // Directory to use for communication
+    commsDir        "${FOAM_CASE}/comms";
+
+    // Does external process start first
+    initByExternal  true;
+
+    // Cleanup behaviour on termination (remove|done)
+    stateEnd        done;
+
+    // Additional output
+    log             true;
+
+    regions
+    {
+        // Region name (wildcards allowed)
+        "(topAir|heater)"
+        {
+            // In topAir adjust the minX patch (fixedValue)
+
+            // Patch or patchGroup
+            coupleGroup
+            {
+                // Fields to output in commsDir
+                writeFields (T);
+                // Fields to read from commsDir
+                readFields  (T);
+            }
+        }
+    }
+}
+
+
+// ************************************************************************* //