From 76de104daa5af1bed83eadb0e6772f357c5c2ded Mon Sep 17 00:00:00 2001 From: mattijs Date: Wed, 27 Nov 2019 14:13:01 +0000 Subject: [PATCH] ENH: fileModificationSkew: add polling. Fixes #1472. --- etc/controlDict | 4 + .../functionEntries/codeStream/codeStream.C | 79 ++++++++++++----- .../db/dynamicLibrary/codedBase/codedBase.C | 85 ++++++++++++------- src/OpenFOAM/db/regIOobject/regIOobject.C | 11 +++ src/OpenFOAM/db/regIOobject/regIOobject.H | 2 + src/OpenFOAM/global/argList/argList.C | 23 ++++- 6 files changed, 149 insertions(+), 55 deletions(-) diff --git a/etc/controlDict b/etc/controlDict index 5be9a11664..02bdf85dbd 100644 --- a/etc/controlDict +++ b/etc/controlDict @@ -82,6 +82,10 @@ OptimisationSwitches // updated. Set to 0 on distributed case. fileModificationSkew 10; + // How often retest (using above fileModificationSkew) before giving up. + // Default is 1. + //maxFileModificationPolls 100; + //- Modification checking: // - timeStamp : use modification time on file // - inotify : use inotify framework diff --git a/src/OpenFOAM/db/dictionary/functionEntries/codeStream/codeStream.C b/src/OpenFOAM/db/dictionary/functionEntries/codeStream/codeStream.C index bc9c22c355..72a1f92778 100644 --- a/src/OpenFOAM/db/dictionary/functionEntries/codeStream/codeStream.C +++ b/src/OpenFOAM/db/dictionary/functionEntries/codeStream/codeStream.C @@ -222,44 +222,77 @@ Foam::functionEntries::codeStream::getFunction off_t masterSize = mySize; Pstream::scatter(masterSize); - DebugPout - << nl << "on processor " << Pstream::myProcNo() - << " have masterSize:" << masterSize - << " and localSize:" << mySize << endl; - - if (mySize < masterSize) + for + ( + label iter = 0; + iter < regIOobject::maxFileModificationPolls; + iter++ + ) { DebugPout - << "Local file " << libPath - << " not of same size (" << mySize - << ") as master (" - << masterSize << "). Waiting for " - << regIOobject::fileModificationSkew - << " seconds." << endl; + << "on processor " << Pstream::myProcNo() + << "masterSize:" << masterSize + << " and localSize:" << mySize + << endl; - Foam::sleep(regIOobject::fileModificationSkew); - - // Recheck local size - mySize = Foam::fileSize(libPath); - - if (mySize < masterSize) + if (mySize == masterSize) { - FatalIOErrorInFunction(parentDict) - << "Cannot read (NFS mounted) library " << nl - << libPath << nl + break; + } + else if (mySize > masterSize) + { + FatalIOErrorInFunction(context.dict()) + << "Excessive size when reading (NFS mounted) library " + << nl << libPath << nl << "on processor " << Pstream::myProcNo() << " detected size " << mySize << " whereas master size is " << masterSize << " bytes." << nl - << "If your case is not NFS mounted" + << "If your case is NFS mounted increase" + << " fileModificationSkew or maxFileModificationPolls;" + << nl << "If your case is not NFS mounted" << " (so distributed) set fileModificationSkew" << " to 0" << exit(FatalIOError); } + else + { + DebugPout + << "Local file " << libPath + << " not of same size (" << mySize + << ") as master (" + << masterSize << "). Waiting for " + << regIOobject::fileModificationSkew + << " seconds." << endl; + + Foam::sleep(regIOobject::fileModificationSkew); + + // Recheck local size + mySize = Foam::fileSize(libPath); + } + } + + + // Finished doing iterations. Do final check + if (mySize != masterSize) + { + FatalIOErrorInFunction(context.dict()) + << "Cannot read (NFS mounted) library " << nl + << libPath << nl + << "on processor " << Pstream::myProcNo() + << " detected size " << mySize + << " whereas master size is " << masterSize + << " bytes." << nl + << "If your case is NFS mounted increase" + << " fileModificationSkew or maxFileModificationPolls;" + << nl << "If your case is not NFS mounted" + << " (so distributed) set fileModificationSkew" + << " to 0" + << exit(FatalIOError); } DebugPout - << nl << "on processor " << Pstream::myProcNo() + << "on processor " << Pstream::myProcNo() << " after waiting: have masterSize:" << masterSize << " and localSize:" << mySize << endl; } diff --git a/src/OpenFOAM/db/dynamicLibrary/codedBase/codedBase.C b/src/OpenFOAM/db/dynamicLibrary/codedBase/codedBase.C index 2be000110e..37731c909a 100644 --- a/src/OpenFOAM/db/dynamicLibrary/codedBase/codedBase.C +++ b/src/OpenFOAM/db/dynamicLibrary/codedBase/codedBase.C @@ -250,54 +250,79 @@ void Foam::codedBase::createLibrary off_t masterSize = mySize; Pstream::scatter(masterSize); - if (debug) + for + ( + label iter = 0; + iter < regIOobject::maxFileModificationPolls; + iter++ + ) { - Pout<< endl<< "on processor " << Pstream::myProcNo() + DebugPout + << "on processor " << Pstream::myProcNo() << " have masterSize:" << masterSize << " and localSize:" << mySize << endl; - } - - if (mySize < masterSize) - { - if (debug) + if (mySize == masterSize) { - Pout<< "Local file " << libPath + break; + } + else if (mySize > masterSize) + { + FatalIOErrorInFunction(context.dict()) + << "Excessive size when reading (NFS mounted) library " + << nl << libPath << nl + << "on processor " << Pstream::myProcNo() + << " detected size " << mySize + << " whereas master size is " << masterSize + << " bytes." << nl + << "If your case is NFS mounted increase" + << " fileModificationSkew or maxFileModificationPolls;" + << nl << "If your case is not NFS mounted" + << " (so distributed) set fileModificationSkew" + << " to 0" + << exit(FatalIOError); + } + else + { + DebugPout + << "Local file " << libPath << " not of same size (" << mySize << ") as master (" << masterSize << "). Waiting for " << regIOobject::fileModificationSkew << " seconds." << endl; - } - Foam::sleep(regIOobject::fileModificationSkew); - // Recheck local size - mySize = Foam::fileSize(libPath); + Foam::sleep(regIOobject::fileModificationSkew); - if (mySize < masterSize) - { - FatalIOErrorInFunction(context.dict()) - << "Cannot read (NFS mounted) library " << nl - << libPath << nl - << "on processor " << Pstream::myProcNo() - << " detected size " << mySize - << " whereas master size is " << masterSize - << " bytes." << nl - << "If your case is not NFS mounted" - << " (so distributed) set fileModificationSkew" - << " to 0" - << exit(FatalIOError); + // Recheck local size + mySize = Foam::fileSize(libPath); } } - if (debug) + + // Finished doing iterations. Do final check + if (mySize != masterSize) { - Pout<< endl<< "on processor " << Pstream::myProcNo() - << " after waiting: have masterSize:" << masterSize - << " and localSize:" << mySize - << endl; + FatalIOErrorInFunction(context.dict()) + << "Cannot read (NFS mounted) library " << nl + << libPath << nl + << "on processor " << Pstream::myProcNo() + << " detected size " << mySize + << " whereas master size is " << masterSize + << " bytes." << nl + << "If your case is NFS mounted increase" + << " fileModificationSkew or maxFileModificationPolls;" + << nl << "If your case is not NFS mounted" + << " (so distributed) set fileModificationSkew" + << " to 0" + << exit(FatalIOError); } + + DebugPout + << "on processor " << Pstream::myProcNo() + << " after waiting: have masterSize:" << masterSize + << " and localSize:" << mySize << endl; } reduce(create, orOp()); } diff --git a/src/OpenFOAM/db/regIOobject/regIOobject.C b/src/OpenFOAM/db/regIOobject/regIOobject.C index 3fe337f1b6..eb938a6605 100644 --- a/src/OpenFOAM/db/regIOobject/regIOobject.C +++ b/src/OpenFOAM/db/regIOobject/regIOobject.C @@ -50,6 +50,17 @@ registerOptSwitch Foam::regIOobject::fileModificationSkew ); +int Foam::regIOobject::maxFileModificationPolls +( + Foam::debug::optimisationSwitch("maxFileModificationPolls", 1) +); +registerOptSwitch +( + "maxFileModificationPolls", + int, + Foam::regIOobject::maxFileModificationPolls +); + bool Foam::regIOobject::masterOnlyReading = false; diff --git a/src/OpenFOAM/db/regIOobject/regIOobject.H b/src/OpenFOAM/db/regIOobject/regIOobject.H index 0f9ae1196c..e990fa7ab6 100644 --- a/src/OpenFOAM/db/regIOobject/regIOobject.H +++ b/src/OpenFOAM/db/regIOobject/regIOobject.H @@ -131,6 +131,8 @@ public: static float fileModificationSkew; + static int maxFileModificationPolls; + // Constructors diff --git a/src/OpenFOAM/global/argList/argList.C b/src/OpenFOAM/global/argList/argList.C index 99d2f04e2e..01f1bddc5e 100644 --- a/src/OpenFOAM/global/argList/argList.C +++ b/src/OpenFOAM/global/argList/argList.C @@ -1501,8 +1501,27 @@ void Foam::argList::parse ) ) { - Info<< " (fileModificationSkew " - << regIOobject::fileModificationSkew << ")"; + if (regIOobject::maxFileModificationPolls == 1) + { + Info<< " (fileModificationSkew " + << regIOobject::fileModificationSkew + << ")"; + } + else if (regIOobject::maxFileModificationPolls > 1) + { + Info<< " (fileModificationSkew " + << regIOobject::fileModificationSkew + << ", maxFileModificationPolls " + << regIOobject::maxFileModificationPolls + << ")"; + } + else + { + FatalErrorInFunction + << "Invalid setting for maxFileModificationPolls " + << regIOobject::maxFileModificationPolls + << exit(FatalError); + } } Info<< nl;