zhyang-dev/openfoam
1
0
Fork 0
mirror of https://develop.openfoam.com/Development/openfoam.git synced 2025-11-28 03:28:01 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

ENH: reduce some code duplication in fileOperations

Browse Source
This commit is contained in:
Mark Olesen
2021-03-17 14:09:55 +01:00
parent cb6dedacfc
commit 6ccc587bea
14 changed files with 160 additions and 130 deletions
Download Patch File Download Diff File Expand all files Collapse all files

18
src/OpenFOAM/db/IOobjects/decomposedBlockData/decomposedBlockData.H
View File

@ -93,6 +93,9 @@ SourceFiles
namespace Foam
{
// Forward Declarations
class dictionary;
/*---------------------------------------------------------------------------*\
Class decomposedBlockData Declaration
\*---------------------------------------------------------------------------*/
@ -177,8 +180,8 @@ public:
//- Read object
virtual bool read();
//- Write separated content. Assumes content is the serialised data
// and that the master data contains a header
//- Write separated content (assumes content is the serialised data)
// The serialised master data should also contain a FoamFile header
virtual bool writeData(Ostream& os) const;
//- Write using stream options
@ -209,7 +212,8 @@ public:
const word& objectType,
const string& note,
const fileName& location,
const word& objectName
const word& objectName,
const dictionary* extraEntries = nullptr
);
//- Helper: write FoamFile IOobject header
@ -220,6 +224,14 @@ public:
const IOobject& io
);
//- Helper: generate additional entries for FoamFile header
static void writeExtraHeaderContent
(
dictionary& dict,
IOstreamOption streamOptData,
const IOobject& io
);
//- Helper: read block of (binary) character data
static bool readBlockEntry
(

27
src/OpenFOAM/db/IOobjects/decomposedBlockData/decomposedBlockDataHeader.C
View File

@ -140,7 +140,8 @@ void Foam::decomposedBlockData::writeHeader
const word& objectType,
const string& note,
const fileName& location,
const word& objectName
const word& objectName,
const dictionary* extraEntries
)
{
if (IOobject::bannerEnabled())
@ -160,6 +161,11 @@ void Foam::decomposedBlockData::writeHeader
objectName
);
if (extraEntries)
{
extraEntries->writeEntries(os);
}
os.endBlock();
if (IOobject::bannerEnabled())
@ -169,6 +175,25 @@ void Foam::decomposedBlockData::writeHeader
}
void Foam::decomposedBlockData::writeExtraHeaderContent
(
dictionary& dict,
IOstreamOption streamOptData,
const IOobject& io
)
{
dict.set("data.format", streamOptData.format());
dict.set("data.class", io.type());
// Deep-copy of meta-data (if any)
const dictionary* metaDataDict = io.findMetaData();
if (metaDataDict && !metaDataDict->empty())
{
dict.add("meta", *metaDataDict);
}
}
void Foam::decomposedBlockData::writeHeader
(
Ostream& os,

15
src/OpenFOAM/global/fileOperations/collatedFileOperation/OFstreamCollator.C
View File

@ -79,18 +79,21 @@ bool Foam::OFstreamCollator::writeFile
{
Foam::mkDir(fName.path());
osPtr.reset(new OFstream(fName, streamOpt, append));
auto& os = *osPtr;
// We don't have IOobject so cannot use IOobject::writeHeader
if (!append)
{
// No IOobject so cannot use IOobject::writeHeader
// FoamFile
decomposedBlockData::writeHeader
(
*osPtr,
streamOpt, // streamOpt for container
os,
streamOpt, // streamOpt for container
objectType,
"", // note
"", // location (leave empty instead inaccurate)
fName.name() // object name
"", // note
"", // location (leave empty instead inaccurate)
fName.name() // object name
);
}
}

2
src/OpenFOAM/global/fileOperations/collatedFileOperation/OFstreamCollator.H
View File

@ -119,7 +119,7 @@ class OFstreamCollator
};
// Private data
// Private Data
//- Total amount of storage to use for object stack below
const off_t maxBufferSize_;

34
src/OpenFOAM/global/fileOperations/collatedFileOperation/collatedFileOperation.C
View File

@ -74,23 +74,6 @@ namespace fileOperations
}
// * * * * * * * * * * * * * Static Member Functions * * * * * * * * * * * * //
Foam::labelList Foam::fileOperations::collatedFileOperation::ioRanks()
{
labelList ioRanks;
string ioRanksString(Foam::getEnv("FOAM_IORANKS"));
if (!ioRanksString.empty())
{
IStringStream is(ioRanksString);
is >> ioRanks;
}
return ioRanks;
}
// * * * * * * * * * * * * Protected Member Functions * * * * * * * * * * * //
void Foam::fileOperations::collatedFileOperation::printBanner
@ -125,20 +108,20 @@ void Foam::fileOperations::collatedFileOperation::printBanner
if (printRanks)
{
// Information about the ranks
stringList ioRanks(Pstream::nProcs());
stringList hosts(Pstream::nProcs());
if (Pstream::master(comm_))
{
// Don't usually need the pid
// ioRanks[Pstream::myProcNo()] = hostName()+"."+name(pid());
ioRanks[Pstream::myProcNo()] = hostName();
// hosts[Pstream::myProcNo()] = hostName()+"."+name(pid());
hosts[Pstream::myProcNo()] = hostName();
}
Pstream::gatherList(ioRanks);
Pstream::gatherList(hosts);
DynamicList<label> offsetMaster(Pstream::nProcs());
forAll(ioRanks, ranki)
forAll(hosts, ranki)
{
if (!ioRanks[ranki].empty())
if (!hosts[ranki].empty())
{
offsetMaster.append(ranki);
}
@ -157,7 +140,7 @@ void Foam::fileOperations::collatedFileOperation::printBanner
const label end = offsetMaster[group];
DetailInfo
<< " (" << ioRanks[beg].c_str() << ' '
<< " (" << hosts[beg].c_str() << ' '
<< (end-beg) << ')' << nl;
}
DetailInfo
@ -500,9 +483,6 @@ bool Foam::fileOperations::collatedFileOperation::writeObject
useThread
);
// If any of these fail, return
// (leave error handling to Ostream class)
bool ok = os.good();
if (Pstream::master(comm_))

6
src/OpenFOAM/global/fileOperations/collatedFileOperation/collatedFileOperation.H
View File

@ -46,7 +46,6 @@ SourceFiles
#include "masterUncollatedFileOperation.H"
#include "OFstreamCollator.H"
#include "fileOperationInitialise.H"
// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
@ -84,9 +83,6 @@ protected:
// Protected Member Functions
//- Retrieve list of IO ranks from FOAM_IORANKS env variable
static labelList ioRanks();
//- Print banner information, optionally with io ranks
void printBanner(const bool printRanks = false) const;
@ -198,7 +194,7 @@ public:
// Member Functions
//- Needs threading
//- Requires threading for non-zero maxThreadFileBufferSize
virtual bool needsThreading() const
{
return (collatedFileOperation::maxThreadFileBufferSize > 0);

14
src/OpenFOAM/global/fileOperations/collatedFileOperation/hostCollatedFileOperation.C
View File

@ -66,21 +66,19 @@ Foam::labelList Foam::fileOperations::hostCollatedFileOperation::subRanks
{
DynamicList<label> subRanks(64);
string ioRanksString(getEnv("FOAM_IORANKS"));
if (!ioRanksString.empty())
labelList mainRanks(fileOperation::ioRanks());
if (!mainRanks.empty())
{
IStringStream is(ioRanksString);
labelList ioRanks(is);
if (!ioRanks.found(0))
if (!mainRanks.found(0))
{
FatalErrorInFunction
<< "Rank 0 (master) should be in the IO ranks. Currently "
<< ioRanks << exit(FatalError);
<< mainRanks << nl
<< exit(FatalError);
}
// The lowest numbered rank is the IO rank
const bitSet isIOrank(n, ioRanks);
const bitSet isIOrank(n, mainRanks);
for (label proci = Pstream::myProcNo(); proci >= 0; --proci)
{

17
src/OpenFOAM/global/fileOperations/fileOperation/fileOperation.C
View File

@ -36,6 +36,7 @@ License
#include "polyMesh.H"
#include "registerSwitch.H"
#include "Time.H"
#include "ITstream.H"
#include <cerrno>
#include <cinttypes>
@ -181,6 +182,20 @@ static bool parseProcsNumRange
// * * * * * * * * * * * * * Static Member Functions * * * * * * * * * * * * //
Foam::labelList Foam::fileOperation::ioRanks()
{
labelList ranks;
ITstream is("ioranks", Foam::getEnv("FOAM_IORANKS"));
if (!is.empty())
{
is >> ranks;
}
return ranks;
}
Foam::instantList
Foam::fileOperation::sortTimes
(
@ -241,7 +256,7 @@ Foam::fileOperation::sortTimes
}
// * * * * * * * * * * * * * Private Member Functions * * * * * * * * * * * //
// * * * * * * * * * * * * Protected Member Functions * * * * * * * * * * * //
Foam::fileMonitor& Foam::fileOperation::monitor() const
{

58
src/OpenFOAM/global/fileOperations/fileOperation/fileOperation.H
View File

@ -47,8 +47,6 @@ Description
#include "fileNameList.H"
#include "instantList.H"
#include "fileMonitor.H"
#include "labelList.H"
#include "Switch.H"
#include "refPtr.H"
#include "Enum.H"
#include "Tuple2.H"
@ -64,13 +62,6 @@ class regIOobject;
class objectRegistry;
class Time;
// Description of processor directory naming:
// - processor directory naming
// - whether directory contains a range (so differs on different processors)
// - index in range
//typedef Tuple2<fileName, Tuple2<bool, label>> dirIndex;
//typedef List<dirIndex> dirIndexList;
/*---------------------------------------------------------------------------*\
Class fileOperation Declaration
\*---------------------------------------------------------------------------*/
@ -80,28 +71,32 @@ class fileOperation
public:
//- Enumeration for the location of an IOobject
enum pathType
enum pathType : int
{
NOTFOUND, // not found
ABSOLUTE, // instance is absolute directory
OBJECT, // io.objectPath() exists
WRITEOBJECT, // write path exists
PROCUNCOLLATED, // objectPath exists in processor0
PROCBASEOBJECT, // objectPath exists in specified, constant
// processorsDir (usually 'processorsDDD')
PROCOBJECT, // objectPath exists in locally differing
// processorsDir (e.g. 'processorsDDD_0-1')
PARENTOBJECT, // parent of object path
FINDINSTANCE, // file found in time directory
PROCUNCOLLATEDINSTANCE, // as PROCUNCOLLATED but with instance
PROCBASEINSTANCE, // as PROCBASEOBJECT but with instance
PROCINSTANCE // as PROCOBJECT but with instance
NOTFOUND = 0, //!< Not found
ABSOLUTE, //!< instance is absolute directory
OBJECT, //!< io.objectPath() exists
WRITEOBJECT, //!< write path exists
// NOTE: increasing precedence (uncollated, collated, rank-collated)
PROCUNCOLLATED,
//!< objectPath exists in 'processorN'
PROCBASEOBJECT = PROCUNCOLLATED + 1,
//!< objectPath exists in 'processorsNN'
PROCOBJECT = PROCBASEOBJECT + 1,
//!< objectPath exists in 'processorsNN_first-last'
PARENTOBJECT, //!< parent of object path
FINDINSTANCE, //!< file found in time directory
PROCUNCOLLATEDINSTANCE, //!< as PROCUNCOLLATED but with instance
PROCBASEINSTANCE, //!< as PROCBASEOBJECT but with instance
PROCINSTANCE //!< as PROCOBJECT but with instance
};
static const Enum<pathType> pathTypeNames_;
//- A dirIndex adds the path type and local offset to a fileName
typedef Tuple2<fileName, Tuple2<pathType, label>> dirIndex;
//- Augment fileName with pathType and local offset
typedef Tuple2<fileName, Tuple2<pathType, int>> dirIndex;
typedef List<dirIndex> dirIndexList;
//- For addressing a range of processors,
@ -131,6 +126,9 @@ protected:
//- Get or create fileMonitor singleton
fileMonitor& monitor() const;
//- Retrieve list of IO ranks from FOAM_IORANKS env variable
static labelList ioRanks();
//- Merge two times
static void mergeTimes
(
@ -146,7 +144,7 @@ protected:
// \return empty fileName if not found.
refPtr<dirIndexList> lookupAndCacheProcessorsPath
(
const fileName&,
const fileName& objectPath,
const bool syncPar
) const;
@ -158,8 +156,8 @@ protected:
const fileName& objectPath
) const;
//- Does ioobject exist. Is either a directory (empty name()) or
// a file
//- Does IOObject exist.
//- Is either a directory (empty name()) or a file
bool exists(IOobject& io) const;

28
src/OpenFOAM/global/fileOperations/fileOperationInitialise/fileOperationInitialise.C
View File

@ -6,7 +6,7 @@
\\/ M anipulation |
-------------------------------------------------------------------------------
Copyright (C) 2017-2018 OpenFOAM Foundation
Copyright (C) 2019 OpenCFD Ltd.
Copyright (C) 2019-2021 OpenCFD Ltd.
-------------------------------------------------------------------------------
License
This file is part of OpenFOAM.
@ -27,6 +27,7 @@ License
\*---------------------------------------------------------------------------*/
#include "fileOperationInitialise.H"
#include "OSspecific.H"
#include "addToRunTimeSelectionTable.H"
/* * * * * * * * * * * * * * * Static Member Data * * * * * * * * * * * * * */
@ -48,7 +49,30 @@ Foam::fileOperations::fileOperationInitialise::fileOperationInitialise
int& argc,
char**& argv
)
{}
{
// Filter out commonly known arguments
const string s("-ioRanks");
int index = -1;
for (int i=1; i<argc-1; i++)
{
if (argv[i] == s)
{
index = i;
Foam::setEnv("FOAM_IORANKS", argv[i+1], true);
break;
}
}
if (index > 0)
{
for (int i=index+2; i<argc; i++)
{
argv[i-2] = argv[i];
}
argc -= 2;
}
}
Foam::autoPtr<Foam::fileOperations::fileOperationInitialise>

9
src/OpenFOAM/global/fileOperations/fileOperationInitialise/fileOperationInitialise.H
View File

@ -26,6 +26,11 @@ License
Class
Foam::fileOperationInitialise
Description
General fileOperation initialiser.
Handles \c -ioRanks option, using it to set the FOAM_IORANKS environment
variable.
\*---------------------------------------------------------------------------*/
#ifndef fileOperationInitialise_H
@ -54,7 +59,7 @@ public:
// Constructors
//- Construct components
//- Construct from components
fileOperationInitialise(int& argc, char**& argv);
@ -87,7 +92,7 @@ public:
// Member Functions
//- Needs threading
//- Threading required?
virtual bool needsThreading() const = 0;
};

5
src/OpenFOAM/global/fileOperations/fileOperationInitialise/unthreadedInitialise.H
View File

@ -26,6 +26,9 @@ License
Class
Foam::unthreadedInitialise
Description
A fileOperation initialiser for unthreaded file handlers
\*---------------------------------------------------------------------------*/
#ifndef unthreadedInitialise_H
@ -65,7 +68,7 @@ public:
// Member Functions
//- Needs threading
//- No threading required
virtual bool needsThreading() const
{
return false;

49
src/OpenFOAM/global/fileOperations/masterUncollatedFileOperation/masterUncollatedFileOperation.C
View File

@ -32,6 +32,7 @@ License
#include "Time.H"
#include "instant.H"
#include "IFstream.H"
#include "IListStream.H"
#include "masterOFstream.H"
#include "decomposedBlockData.H"
#include "registerSwitch.H"
@ -39,7 +40,6 @@ License
#include "SubList.H"
#include "unthreadedInitialise.H"
#include "bitSet.H"
#include "IListStream.H"
/* * * * * * * * * * * * * * * Static Member Data * * * * * * * * * * * * * */
@ -85,8 +85,9 @@ Foam::labelList Foam::fileOperations::masterUncollatedFileOperation::subRanks
const label n
)
{
string ioRanksString(getEnv("FOAM_IORANKS"));
if (ioRanksString.empty())
labelList mainRanks(fileOperation::ioRanks());
if (mainRanks.empty())
{
return identity(n);
}
@ -94,18 +95,16 @@ Foam::labelList Foam::fileOperations::masterUncollatedFileOperation::subRanks
{
DynamicList<label> subRanks(n);
IStringStream is(ioRanksString);
labelList ioRanks(is);
if (!ioRanks.found(0))
if (!mainRanks.found(0))
{
FatalErrorInFunction
<< "Rank 0 (master) should be in the IO ranks. Currently "
<< ioRanks << exit(FatalError);
<< mainRanks << nl
<< exit(FatalError);
}
// The lowest numbered rank is the IO rank
const bitSet isIOrank(n, ioRanks);
const bitSet isIOrank(n, mainRanks);
for (label proci = Pstream::myProcNo(); proci >= 0; --proci)
{
@ -810,36 +809,6 @@ masterUncollatedFileOperation
}
Foam::fileOperations::masterUncollatedFileOperationInitialise::
masterUncollatedFileOperationInitialise(int& argc, char**& argv)
:
unthreadedInitialise(argc, argv)
{
// Filter out any of my arguments
const string s("-ioRanks");
int index = -1;
for (int i=1; i<argc-1; i++)
{
if (argv[i] == s)
{
index = i;
Foam::setEnv("FOAM_IORANKS", argv[i+1], true);
break;
}
}
if (index != -1)
{
for (int i=index+2; i<argc; i++)
{
argv[i-2] = argv[i];
}
argc -= 2;
}
}
// * * * * * * * * * * * * * * * * Destructor * * * * * * * * * * * * * * * //
Foam::fileOperations::masterUncollatedFileOperation::
@ -1754,7 +1723,7 @@ bool Foam::fileOperations::masterUncollatedFileOperation::readHeader
{
Pout<< "masterUncollatedFileOperation::readHeader :" << endl
<< " objectPath:" << io.objectPath() << endl
<< " fName :" << fName << endl;
<< " filePath :" << fName << endl;
}
// Get filePaths on world master

8
src/OpenFOAM/global/fileOperations/masterUncollatedFileOperation/masterUncollatedFileOperation.H
View File

@ -65,9 +65,8 @@ Description
#include "fileOperation.H"
#include "OSspecific.H"
#include "HashPtrTable.H"
#include "Switch.H"
#include "List.H"
#include "unthreadedInitialise.H"
#include "boolList.H"
// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
@ -784,7 +783,10 @@ public:
// Constructors
//- Construct from components
masterUncollatedFileOperationInitialise(int& argc, char**& argv);
masterUncollatedFileOperationInitialise(int& argc, char**& argv)
:
unthreadedInitialise(argc, argv)
{}
//- Destructor