diff --git a/applications/test/broadcastCopy/Make/files b/applications/test/broadcastCopy/Make/files new file mode 100644 index 0000000000..14f89956da --- /dev/null +++ b/applications/test/broadcastCopy/Make/files @@ -0,0 +1,3 @@ +Test-broadcastCopy.C + +EXE = $(FOAM_USER_APPBIN)/Test-broadcastCopy diff --git a/applications/test/broadcastCopy/Make/options b/applications/test/broadcastCopy/Make/options new file mode 100644 index 0000000000..18e6fe47af --- /dev/null +++ b/applications/test/broadcastCopy/Make/options @@ -0,0 +1,2 @@ +/* EXE_INC = */ +/* EXE_LIBS = */ diff --git a/applications/test/broadcastCopy/Test-broadcastCopy.C b/applications/test/broadcastCopy/Test-broadcastCopy.C new file mode 100644 index 0000000000..4c0ff2febf --- /dev/null +++ b/applications/test/broadcastCopy/Test-broadcastCopy.C @@ -0,0 +1,97 @@ +/*---------------------------------------------------------------------------*\ + ========= | + \\ / F ield | OpenFOAM: The Open Source CFD Toolbox + \\ / O peration | + \\ / A nd | www.openfoam.com + \\/ M anipulation | +------------------------------------------------------------------------------- + Copyright (C) 2023 OpenCFD Ltd. +------------------------------------------------------------------------------- +License + This file is part of OpenFOAM. + + OpenFOAM is free software: you can redistribute it and/or modify it + under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + OpenFOAM is distributed in the hope that it will be useful, but WITHOUT + ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + for more details. + + You should have received a copy of the GNU General Public License + along with OpenFOAM. If not, see . + +Description + Test file/directory broadcasting + +\*---------------------------------------------------------------------------*/ + +#include "argList.H" +#include "OSspecific.H" +#include "fileOperation.H" +#include "Pstream.H" +#include "Switch.H" + +using namespace Foam; + +// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * // +// Main program: + +int main(int argc, char *argv[]) +{ + argList::noBanner(); + argList::noFunctionObjects(); + argList::noCheckProcessorDirectories(); + + argList::addNote("Test broadcast file via MPI"); + + argList::addArgument("srcFile"); + argList::addBoolOption("even", "Broadcast to even directories only"); + argList::addBoolOption("relative", "Copy relative to output dir"); + + #include "setRootCase.H" + + const auto srcFile = args.get(1); + + // const auto dstFile = args.get(2); + fileName dstFile("proc" + Foam::name(UPstream::myProcNo())); + + + const bool writeOnProc = + ( + !args.found("even") || 0 == (UPstream::myProcNo() % 2) + ); + + if (args.found("relative")) + { + // if (writeOnProc) + // { + // Foam::mkDir(dstFile); + // } + } + else + { + dstFile /= srcFile + ".copy"; + } + + Pout<< "writing: " << writeOnProc << " : " << dstFile << endl; + + const auto& fp = fileHandler(); + + fp.broadcastCopy + ( + UPstream::worldComm, + writeOnProc, + srcFile, + dstFile + ); + + Info<< "\nEnd\n" << endl; + + return 0; +} + + +// ************************************************************************* // diff --git a/src/OpenFOAM/Make/files b/src/OpenFOAM/Make/files index f9bb85e565..93f42ea300 100644 --- a/src/OpenFOAM/Make/files +++ b/src/OpenFOAM/Make/files @@ -18,6 +18,7 @@ global/etcFiles/etcFiles.C fileOps = global/fileOperations $(fileOps)/fileOperation/fileOperation.C +$(fileOps)/fileOperation/fileOperationBroadcast.C $(fileOps)/fileOperationInitialise/fileOperationInitialise.C $(fileOps)/uncollatedFileOperation/uncollatedFileOperation.C $(fileOps)/masterUncollatedFileOperation/masterUncollatedFileOperation.C diff --git a/src/OpenFOAM/global/fileOperations/fileOperation/fileOperation.H b/src/OpenFOAM/global/fileOperations/fileOperation/fileOperation.H index 8da0f46275..7f020e8878 100644 --- a/src/OpenFOAM/global/fileOperations/fileOperation/fileOperation.H +++ b/src/OpenFOAM/global/fileOperations/fileOperation/fileOperation.H @@ -379,6 +379,23 @@ public: const bool emptyOnly = false ) const = 0; + //- Read dir/file (recursively if necessary) on master of the + //- communicator, send and write contents to all 'writeOnProc' + //- processors with local file name + // \param comm the communicator for broadcasting + // \param writeOnProc write on the processor + // \param src the source file/directory + // \param dst the target file/directory. + // If empty, treat as being identical to the src. + virtual bool broadcastCopy + ( + const label comm, + const bool writeOnProc, + const fileName& src, + const fileName& dst + // always recreates links + ) const; + // (reg)IOobject functionality diff --git a/src/OpenFOAM/global/fileOperations/fileOperation/fileOperationBroadcast.C b/src/OpenFOAM/global/fileOperations/fileOperation/fileOperationBroadcast.C new file mode 100644 index 0000000000..2d61a91f94 --- /dev/null +++ b/src/OpenFOAM/global/fileOperations/fileOperation/fileOperationBroadcast.C @@ -0,0 +1,415 @@ +/*---------------------------------------------------------------------------*\ + ========= | + \\ / F ield | OpenFOAM: The Open Source CFD Toolbox + \\ / O peration | + \\ / A nd | www.openfoam.com + \\/ M anipulation | +------------------------------------------------------------------------------- + Copyright (C) 2022-2023 OpenCFD Ltd. +------------------------------------------------------------------------------- +License + This file is part of OpenFOAM. + + OpenFOAM is free software: you can redistribute it and/or modify it + under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + OpenFOAM is distributed in the hope that it will be useful, but WITHOUT + ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + for more details. + + You should have received a copy of the GNU General Public License + along with OpenFOAM. If not, see . + +\*---------------------------------------------------------------------------*/ + +#include "fileOperation.H" +#include "Pstream.H" +#include "OSspecific.H" +#include +#include + +// * * * * * * * * * * * * * * * Local Functions * * * * * * * * * * * * * * // + +namespace Foam +{ + +// Implementation for broadcasting an individual file +static void broadcastFile_single +( + const label comm, + const bool writeOnProc, + const fileName& srcName, + const fileName& dstName, + DynamicList& buffer +) +{ + if (fileOperation::debug) + { + InfoInFunction + << "Reading file " << srcName + << " on master processor and copying to " << dstName + << endl; + } + + // Read file on master, broadcast to all but write on IOranks only. + // This is a lot easier / possibly quicker? + // than -allocating our own IO communicator + // -send to all IO subProcs -releasing communicator + + // The file length uses uint64_t instead of off_t + // since MINGW has something odd for off_t: + // + // ... error: call of overloaded 'min(off_t&, off_t&)' is ambiguous + + // Tuple2 lengthAndMode(0, 0); + Tuple2 lengthAndMode(0, 0); + + std::unique_ptr srcStream; + std::unique_ptr dstStream; + + if (UPstream::master(comm)) + { + // Read (see newIFstream) + lengthAndMode.first() = Foam::fileSize(srcName); + lengthAndMode.second() = Foam::mode(srcName); + + srcStream.reset + ( + new std::ifstream + ( + srcName, + std::ios_base::in | std::ios_base::binary + ) + ); + if (!srcStream->good()) + { + FatalIOErrorInFunction(srcName) + << "Could not open file for reading!" + << exit(FatalIOError); + } + } + + if + ( + writeOnProc + && + ( + UPstream::master(comm) + ? (srcName != dstName) + : UPstream::is_subrank(comm) + ) + ) + { + // Make sure the destination directory exists. + // - will fail itself if not possible + // - no-op if directory already exists + Foam::mkDir(dstName.path()); + + dstStream.reset + ( + new std::ofstream + ( + dstName, + std::ios_base::out | std::ios_base::binary + ) + ); + + if (!dstStream->good()) + { + // Fail noisily or silently? + if (!dstStream->good()) + { + dstStream.reset(nullptr); + } + } + + // Adjust mode? + // Foam::chMode(dstName, fileMode); + } + + // Broadcast size and mode (contiguous data) + UPstream::broadcast + ( + reinterpret_cast(&lengthAndMode), + sizeof(lengthAndMode), + comm + ); + + uint64_t fileLength = lengthAndMode.first(); + + const uint64_t maxChunkSize = + ( + UPstream::maxCommsSize > 0 + ? uint64_t(UPstream::maxCommsSize) + : uint64_t(pTraits::max) + ); + + + while (fileLength > 0) + { + const uint64_t sendSize = min(fileLength, maxChunkSize); + fileLength -= sendSize; + + // Read file contents into a character buffer + buffer.resize_nocopy(static_cast