zhyang-dev/openfoam
1
0
Fork 0
mirror of https://develop.openfoam.com/Development/openfoam.git synced 2025-11-28 03:28:01 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

ENH: avoid reduce for MPI barrier in externalCoupled (issue #419)

Browse Source 
- rationalized waiting logic

- timeout and wait are unsigned int (not label) since this is what
  the underlying sleep uses anyhow.
This commit is contained in:
Mark Olesen
2017-06-14 14:40:59 +02:00
parent b5f091e74f
commit 4a3ead4732
3 changed files with 204 additions and 213 deletions
Download Patch File Download Diff File Expand all files Collapse all files

340
src/functionObjects/field/externalCoupled/externalCoupled.C
View File

@ -50,28 +50,19 @@ namespace functionObjects
} }
} }
const Foam::Enum<Foam::functionObjects::externalCoupled::stateEnd>
Foam::functionObjects::externalCoupled::stateEndNames_
{
{ stateEnd::REMOVE, "remove" },
{ stateEnd::DONE, "done" }
// 'IGNORE' is internal use only and thus without a name
};
Foam::word Foam::functionObjects::externalCoupled::lockName = "OpenFOAM"; Foam::word Foam::functionObjects::externalCoupled::lockName = "OpenFOAM";
Foam::string Foam::functionObjects::externalCoupled::patchKey = "// Patch:"; Foam::string Foam::functionObjects::externalCoupled::patchKey = "// Patch:";
template<>
const char* Foam::NamedEnum
<
Foam::functionObjects::externalCoupled::stateEnd,
2
>::names[] =
{
"remove",
"done"
// The 'IGNORE' enumeration is internal use only and thus has no name
};
const Foam::NamedEnum
<
Foam::functionObjects::externalCoupled::stateEnd,
2
> Foam::functionObjects::externalCoupled::stateEndNames_;
namespace Foam namespace Foam
{ {
@ -86,16 +77,16 @@ static void writeList(Ostream& os, const string& header, const UList<T>& L)
// Write size and start delimiter // Write size and start delimiter
os << L.size() << nl os << L.size() << nl
<< token::BEGIN_LIST; << token::BEGIN_LIST << nl;
// Write contents // Write contents
forAll(L, i) forAll(L, i)
{ {
os << nl << L[i]; os << L[i] << nl;
} }
// Write end delimiter // Write end delimiter
os << nl << token::END_LIST << nl << endl; os << token::END_LIST << nl << endl;
} }
//! \endcond //! \endcond
@ -141,77 +132,72 @@ Foam::fileName Foam::functionObjects::externalCoupled::lockFile() const
void Foam::functionObjects::externalCoupled::useMaster() const void Foam::functionObjects::externalCoupled::useMaster() const
{ {
if (!Pstream::master()) if (Pstream::master())
{ {
return; const fileName lck(lockFile());
}
const fileName fName(lockFile()); // Only create lock file if it doesn't already exist
IFstream is(fName); if (!Foam::isFile(lck))
{
Log << type() << ": creating lock file" << endl;
// Only create lock file if it doesn't already exist OFstream os(lck);
if (!is.good()) os << "status=openfoam\n";
{ os.flush();
Log << type() << ": creating lock file" << endl; }
OFstream os(fName);
os << "status=openfoam\n";
os.flush();
} }
} }
void Foam::functionObjects::externalCoupled::useSlave() const void Foam::functionObjects::externalCoupled::useSlave() const
{ {
if (!Pstream::master()) if (Pstream::master())
{ {
return; Log << type() << ": removing lock file" << endl;
Foam::rm(lockFile());
} }
Log << type() << ": removing lock file" << endl;
Foam::rm(lockFile());
} }
void Foam::functionObjects::externalCoupled::cleanup() const void Foam::functionObjects::externalCoupled::cleanup() const
{ {
if (!Pstream::master()) if (Pstream::master())
{ {
return; const fileName lck(lockFile());
} switch (stateEnd_)
{
const fileName lck(lockFile()); case REMOVE:
switch (stateEnd_)
{
case REMOVE:
{ {
Log << type() << ": removing lock file" << endl; Log << type() << ": removing lock file" << endl;
Foam::rm(lck); Foam::rm(lck);
break;
} }
break; case DONE:
case DONE:
{ {
Log << type() << ": lock file status=done" << endl; Log << type() << ": lock file status=done" << endl;
OFstream os(lck); OFstream os(lck);
os << "status=done\n"; os << "status=done\n";
os.flush(); os.flush();
break;
} }
break; case IGNORE:
case IGNORE: break;
break; }
stateEnd_ = IGNORE; // Avoid re-triggering in destructor
} }
} }
void Foam::functionObjects::externalCoupled::removeReadFiles() const void Foam::functionObjects::externalCoupled::removeDataSlave() const
{ {
if (!Pstream::master()) if (!Pstream::master())
{ {
return; return;
} }
Log << type() << ": removing all read files" << endl; Log << type() << ": removing data files written by slave" << nl;
forAll(regionGroupNames_, regioni) forAll(regionGroupNames_, regioni)
{ {
@ -237,14 +223,14 @@ void Foam::functionObjects::externalCoupled::removeReadFiles() const
} }
void Foam::functionObjects::externalCoupled::removeWriteFiles() const void Foam::functionObjects::externalCoupled::removeDataMaster() const
{ {
if (!Pstream::master()) if (!Pstream::master())
{ {
return; return;
} }
Log << type() << ": removing all write files" << endl; Log << type() << ": removing data files written by master" << nl;
forAll(regionGroupNames_, regioni) forAll(regionGroupNames_, regioni)
{ {
@ -272,42 +258,34 @@ void Foam::functionObjects::externalCoupled::removeWriteFiles() const
void Foam::functionObjects::externalCoupled::waitForSlave() const void Foam::functionObjects::externalCoupled::waitForSlave() const
{ {
const fileName fName(lockFile()); bool waiting = true;
label totalTime = 0; if (Pstream::master())
bool found = false;
Log << type() << ": beginning wait for lock file " << fName << nl;
while (!found)
{ {
if (Pstream::master()) const fileName lck(lockFile());
unsigned totalTime = 0;
Log << type() << ": beginning wait for lock file " << lck << nl;
while ((waiting = !Foam::isFile(lck)) == true)
{ {
if (totalTime > timeOut_) sleep(waitInterval_);
totalTime += waitInterval_;
if (timeOut_ && totalTime > timeOut_)
{ {
FatalErrorInFunction FatalErrorInFunction
<< "Wait time exceeded timeout of " << timeOut_ << "Wait time exceeded timeout of " << timeOut_
<< " s" << abort(FatalError); << " s" << abort(FatalError);
} }
IFstream is(fName); Log << type() << ": wait time = " << totalTime << endl;
if (is.good())
{
found = true;
Log << type() << ": found lock file " << fName << endl;
}
else
{
sleep(waitInterval_);
totalTime += waitInterval_;
Log << type() << ": wait time = " << totalTime << endl;
}
} }
// Prevent other procs from racing ahead Log << type() << ": found lock file " << lck << endl;
reduce(found, orOp<bool>());
} }
// MPI barrier
Pstream::scatter(waiting);
} }
@ -329,7 +307,7 @@ void Foam::functionObjects::externalCoupled::readColumns
// Read data from file and send to destination processor // Read data from file and send to destination processor
for (label proci = 0; proci < Pstream::nProcs(); proci++) for (label proci = 0; proci < Pstream::nProcs(); ++proci)
{ {
// Temporary storage // Temporary storage
List<scalarField> values(nColumns); List<scalarField> values(nColumns);
@ -342,7 +320,7 @@ void Foam::functionObjects::externalCoupled::readColumns
values[columni].setSize(procNRows); values[columni].setSize(procNRows);
} }
for (label rowi = 0; rowi < procNRows; rowi++) for (label rowi = 0; rowi < procNRows; ++rowi)
{ {
// Get a line // Get a line
do do
@ -358,11 +336,12 @@ void Foam::functionObjects::externalCoupled::readColumns
} }
masterFilePtr().getLine(line); masterFilePtr().getLine(line);
} while (line.empty() || line[0] == '#'); }
while (line.empty() || line[0] == '#');
IStringStream lineStr(line); IStringStream lineStr(line);
for (label columni = 0; columni < nColumns; columni++) for (label columni = 0; columni < nColumns; ++columni)
{ {
lineStr >> values[columni][rowi]; lineStr >> values[columni][rowi];
} }
@ -399,14 +378,14 @@ void Foam::functionObjects::externalCoupled::readLines
// Read line from file and send to destination processor // Read line from file and send to destination processor
for (label proci = 0; proci < Pstream::nProcs(); proci++) for (label proci = 0; proci < Pstream::nProcs(); ++proci)
{ {
// Number of rows to read for processor proci // Number of rows to read for processor proci
label procNRows = globalFaces.localSize(proci); label procNRows = globalFaces.localSize(proci);
UOPstream toProc(proci, pBufs); UOPstream toProc(proci, pBufs);
for (label rowi = 0; rowi < procNRows; rowi++) for (label rowi = 0; rowi < procNRows; ++rowi)
{ {
// Get a line // Get a line
do do
@ -422,7 +401,8 @@ void Foam::functionObjects::externalCoupled::readLines
} }
masterFilePtr().getLine(line); masterFilePtr().getLine(line);
} while (line.empty() || line[0] == '#'); }
while (line.empty() || line[0] == '#');
// Send line to the destination processor // Send line to the destination processor
toProc << line; toProc << line;
@ -435,7 +415,7 @@ void Foam::functionObjects::externalCoupled::readLines
// Read lines from PstreamBuffers // Read lines from PstreamBuffers
UIPstream str(Pstream::masterNo(), pBufs); UIPstream str(Pstream::masterNo(), pBufs);
for (label rowi = 0; rowi < nRows; rowi++) for (label rowi = 0; rowi < nRows; ++rowi)
{ {
string line(str); string line(str);
lines << line.c_str() << nl; lines << line.c_str() << nl;
@ -568,8 +548,8 @@ Foam::word Foam::functionObjects::externalCoupled::compositeName
{ {
if (regionNames[0] == polyMesh::defaultRegion) if (regionNames[0] == polyMesh::defaultRegion)
{ {
// For compatibility with single region cases suppress single // For compatibility with single region cases
// region name // - suppress single region name
return word::null; return word::null;
} }
else else
@ -636,35 +616,38 @@ void Foam::functionObjects::externalCoupled::readData()
{ {
const word& fieldName = fieldNames[fieldi]; const word& fieldName = fieldNames[fieldi];
bool ok = readData<scalar> const bool ok =
( (
meshes, readData<scalar>
groupName, (
fieldName meshes,
); groupName,
ok = ok || readData<vector> fieldName
( )
meshes, || readData<vector>
groupName, (
fieldName meshes,
); groupName,
ok = ok || readData<sphericalTensor> fieldName
( )
meshes, || readData<sphericalTensor>
groupName, (
fieldName meshes,
); groupName,
ok = ok || readData<symmTensor> fieldName
( )
meshes, || readData<symmTensor>
groupName, (
fieldName meshes,
); groupName,
ok = ok || readData<tensor> fieldName
( )
meshes, || readData<tensor>
groupName, (
fieldName meshes,
groupName,
fieldName
)
); );
if (!ok) if (!ok)
@ -706,35 +689,38 @@ void Foam::functionObjects::externalCoupled::writeData() const
{ {
const word& fieldName = fieldNames[fieldi]; const word& fieldName = fieldNames[fieldi];
bool ok = writeData<scalar> const bool ok =
( (
meshes, writeData<scalar>
groupName, (
fieldName meshes,
); groupName,
ok = ok || writeData<vector> fieldName
( )
meshes, || writeData<vector>
groupName, (
fieldName meshes,
); groupName,
ok = ok || writeData<sphericalTensor> fieldName
( )
meshes, || writeData<sphericalTensor>
groupName, (
fieldName meshes,
); groupName,
ok = ok || writeData<symmTensor> fieldName
( )
meshes, || writeData<symmTensor>
groupName, (
fieldName meshes,
); groupName,
ok = ok || writeData<tensor> fieldName
( )
meshes, || writeData<tensor>
groupName, (
fieldName meshes,
groupName,
fieldName
)
); );
if (!ok) if (!ok)
@ -794,7 +780,7 @@ void Foam::functionObjects::externalCoupled::initialise()
} }
} }
if (initByExternal_) if (slaveFirst_)
{ {
// Wait for initial data to be made available // Wait for initial data to be made available
waitForSlave(); waitForSlave();
@ -828,7 +814,7 @@ Foam::functionObjects::externalCoupled::externalCoupled
mkDir(baseDir()); mkDir(baseDir());
} }
if (!initByExternal_) if (!slaveFirst_)
{ {
useMaster(); useMaster();
} }
@ -863,7 +849,7 @@ bool Foam::functionObjects::externalCoupled::execute()
waitForSlave(); waitForSlave();
// Remove old data files from OpenFOAM // Remove old data files from OpenFOAM
removeWriteFiles(); removeDataMaster();
// Read data passed back from external source // Read data passed back from external source
readData(); readData();
@ -885,12 +871,10 @@ bool Foam::functionObjects::externalCoupled::end()
functionObject::end(); functionObject::end();
// Remove old data files // Remove old data files
removeReadFiles(); removeDataMaster();
removeWriteFiles(); removeDataSlave();
cleanup(); cleanup();
stateEnd_ = IGNORE; // Avoid running cleanup() again in destructor
return true; return true;
} }
@ -899,25 +883,37 @@ bool Foam::functionObjects::externalCoupled::read(const dictionary& dict)
{ {
functionObject::read(dict); functionObject::read(dict);
calcFrequency_ = dict.lookupOrDefault("calcFrequency", 1); // NB: Cannot change directory or initialization
// if things have already been initialized
if (!initialised_)
{
dict.lookup("commsDir") >> commsDir_;
commsDir_.expand();
commsDir_.clean();
dict.lookup("commsDir") >> commsDir_; slaveFirst_ = readBool(dict.lookup("initByExternal"));
commsDir_.expand(); // slaveFirst_ = dict.lookupOrDefault<bool>("initByExternal", false);
commsDir_.clean(); }
waitInterval_ = dict.lookupOrDefault("waitInterval", 1); calcFrequency_ = dict.lookupOrDefault("calcFrequency", 1);
timeOut_ = dict.lookupOrDefault("timeOut", 100*waitInterval_);
initByExternal_ = readBool(dict.lookup("initByExternal")); waitInterval_ = dict.lookupOrDefault("waitInterval", 1u);
// initByExternal_ = dict.lookupOrDefault<Switch>("initByExternal", false); if (!waitInterval_)
stateEnd_ = {
stateEndNames_[dict.lookupOrDefault<word>("stateEnd", "remove")]; // Enforce non-zero sleep
waitInterval_ = 1u;
}
timeOut_ = dict.lookupOrDefault("timeOut", 100*waitInterval_);
stateEnd_ =
stateEndNames_.lookupOrDefault("stateEnd", dict, stateEnd::REMOVE);
// Get names of all fvMeshes (and derived types) // Get names of all fvMeshes (and derived types)
wordList allRegionNames(time_.lookupClass<fvMesh>().sortedToc()); wordList allRegionNames(time_.lookupClass<fvMesh>().sortedToc());
const dictionary& allRegionsDict = dict.subDict("regions"); const dictionary& allRegionsDict = dict.subDict("regions");
forAllConstIter(dictionary, allRegionsDict, iter) forAllConstIters(allRegionsDict, iter)
{ {
if (!iter().isDict()) if (!iter().isDict())
{ {
@ -936,8 +932,7 @@ bool Foam::functionObjects::externalCoupled::read(const dictionary& dict)
regionGroupNames_.append(compositeName(regionNames)); regionGroupNames_.append(compositeName(regionNames));
regionGroupRegions_.append(regionNames); regionGroupRegions_.append(regionNames);
forAllConstIters(regionDict, regionIter)
forAllConstIter(dictionary, regionDict, regionIter)
{ {
if (!regionIter().isDict()) if (!regionIter().isDict())
{ {
@ -948,15 +943,12 @@ bool Foam::functionObjects::externalCoupled::read(const dictionary& dict)
const wordRe groupName(regionIter().keyword()); const wordRe groupName(regionIter().keyword());
const dictionary& groupDict = regionIter().dict(); const dictionary& groupDict = regionIter().dict();
label nGroups = groupNames_.size(); const label nGroups = groupNames_.size();
const wordList readFields(groupDict.lookup("readFields")); const wordList readFields(groupDict.lookup("readFields"));
const wordList writeFields(groupDict.lookup("writeFields")); const wordList writeFields(groupDict.lookup("writeFields"));
HashTable<labelList>::iterator fnd = regionToGroups_.find auto fnd = regionToGroups_.find(regionGroupNames_.last());
( if (fnd.found())
regionGroupNames_.last()
);
if (fnd != regionToGroups_.end())
{ {
fnd().append(nGroups); fnd().append(nGroups);
} }

60
src/functionObjects/field/externalCoupled/externalCoupled.H
View File

@ -126,7 +126,7 @@ SourceFiles
#include "DynamicList.H" #include "DynamicList.H"
#include "wordReList.H" #include "wordReList.H"
#include "scalarField.H" #include "scalarField.H"
#include "NamedEnum.H" #include "Enum.H"
#include "Switch.H" #include "Switch.H"
#include "UPtrList.H" #include "UPtrList.H"
@ -164,8 +164,8 @@ public:
private: private:
//- State end names //- State end names (NB, only selectable values itemized)
static const NamedEnum<stateEnd, 2> stateEndNames_; static const Enum<stateEnd> stateEndNames_;
// Private data // Private data
@ -177,19 +177,19 @@ private:
fileName commsDir_; fileName commsDir_;
//- Interval time between checking for return data [s] //- Interval time between checking for return data [s]
label waitInterval_; unsigned waitInterval_;
//- Time out time [s] //- Time out time [s]
label timeOut_; unsigned timeOut_;
//- Calculation frequency //- Calculation frequency
label calcFrequency_; label calcFrequency_;
//- Flag to indicate values are initialised by external application //- Flag to indicate values are initialised by external application
bool initByExternal_; bool slaveFirst_;
//- Lockfile state on termination //- Lockfile state on termination
stateEnd stateEnd_; mutable stateEnd stateEnd_;
//- Names of (composite) regions //- Names of (composite) regions
DynamicList<word> regionGroupNames_; DynamicList<word> regionGroupNames_;
@ -240,10 +240,10 @@ private:
void cleanup() const; void cleanup() const;
//- Remove files written by OpenFOAM //- Remove files written by OpenFOAM
void removeWriteFiles() const; void removeDataMaster() const;
//- Remove files written by external code //- Remove files written by external code
void removeReadFiles() const; void removeDataSlave() const;
//- Wait for indication that the external program has supplied input //- Wait for indication that the external program has supplied input
// (ie, for the lock file to reappear). // (ie, for the lock file to reappear).
@ -302,7 +302,7 @@ private:
template<class Type> template<class Type>
static tmp<Field<Type>> gatherAndCombine(const Field<Type>& fld); static tmp<Field<Type>> gatherAndCombine(const Field<Type>& fld);
static void checkOrder(const wordList&); static void checkOrder(const wordList& regionNames);
//- Disallow default bitwise copy constructor //- Disallow default bitwise copy constructor
externalCoupled(const externalCoupled&) = delete; externalCoupled(const externalCoupled&) = delete;
@ -340,34 +340,34 @@ public:
// Member Functions // Member Functions
// Function object control // Function object control
//- Called at each ++ or += of the time-loop //- Called at each ++ or += of the time-loop
virtual bool execute(); virtual bool execute();
//- Called when Time::run() determines that the time-loop exits //- Called when Time::run() determines that the time-loop exits
virtual bool end(); virtual bool end();
//- Read and set the function object if its data have changed //- Read and set the function object if its data have changed
virtual bool read(const dictionary&); virtual bool read(const dictionary& dict);
//- Write //- Write, currently a no-op
virtual bool write(); virtual bool write();
// Other // Other
//- Create single name by appending words (in sorted order), //- Create single name by appending words (in sorted order),
// separated by '_' // separated by '_'
static word compositeName(const wordList&); static word compositeName(const wordList&);
//- Write geometry for the group as region/patch //- Write geometry for the group as region/patch
static void writeGeometry static void writeGeometry
( (
const UPtrList<const fvMesh>& meshes, const UPtrList<const fvMesh>& meshes,
const fileName& commsDir, const fileName& commsDir,
const wordRe& groupName const wordRe& groupName
); );
}; };

17
tutorials/heatTransfer/chtMultiRegionFoam/externalCoupledMultiRegionHeater/externalSolver
View File

@ -38,13 +38,13 @@ log()
init() init()
{ {
log "initialisation: creating ${dataFile}.in" log "init - creating ${dataFile}.in"
# Hard-coded for patch of size 8 (heater/minY) # Hard-coded for patch of size 8 (heater/minY)
n1=8 n1=8
refValue1=500 refValue1=500
touch "${dataFile}.in" touch "${dataFile}.in"
log "initialisation: adding $n1 data elements with refValue $refValue1" log "init - adding $n1 data elements with refValue $refValue1"
for i in $(seq 1 $n1); do for i in $(seq 1 $n1); do
echo "$refValue1 $refGrad $valueFraction" >> "${dataFile}.in" echo "$refValue1 $refGrad $valueFraction" >> "${dataFile}.in"
done done
@ -52,7 +52,7 @@ init()
# Hard-coded for patch of size 40 (topAir/minX) # Hard-coded for patch of size 40 (topAir/minX)
n2=40 n2=40
refValue2=300 refValue2=300
log "initialisation: adding $n2 data elements with refValue $refValue2" log "init - adding $n2 data elements with refValue $refValue2"
for i in $(seq 1 $n2); do for i in $(seq 1 $n2); do
echo "$refValue2 $refGrad $valueFraction" >> "${dataFile}.in" echo "$refValue2 $refGrad $valueFraction" >> "${dataFile}.in"
done done
@ -65,8 +65,7 @@ init()
# create the comms directory # create the comms directory
mkdir -p ${commsDir}/${regionGroupName}/${patchGroupName} mkdir -p ${commsDir}/${regionGroupName}/${patchGroupName}
# tutorial case employs the 'initByExternalOption', so we need to provide # Tutorial case uses 'initByExternalOption', so we must provide initial values
# the initial values
init init
@ -78,13 +77,13 @@ do
then then
if grep -q "status=done" ${lockFile} if grep -q "status=done" ${lockFile}
then then
log "found lock file ${lockFile} with 'status=done' - finished" log "found lock file '${lockFile}' with 'status=done' - finished"
break break
elif [ -s $lockFile ] elif [ -s $lockFile ]
then then
log "found lock file ${lockFile} containing '$(< $lockFile)' - waiting" log "found lock file '${lockFile}' containing '$(< $lockFile)' - waiting"
else else
log "found lock file ${lockFile} - waiting" log "found lock file '${lockFile}' - waiting"
fi fi
totalWait=$(expr $totalWait + $waitSec) totalWait=$(expr $totalWait + $waitSec)
@ -109,7 +108,7 @@ do
awk '{if( $1 != "#" ){print $1+1 " 0 1"}}' \ awk '{if( $1 != "#" ){print $1+1 " 0 1"}}' \
${dataFile}.out >| ${dataFile}.in ${dataFile}.out >| ${dataFile}.in
log "creating lock file ${lockFile}" log "creating lock file '${lockFile}'"
touch ${lockFile} touch ${lockFile}
fi fi
done done