From 5dcacbdcaf68357e4e58b17f5342739a0a81c298 Mon Sep 17 00:00:00 2001 From: mattijs Date: Tue, 30 Nov 2010 16:35:58 +0000 Subject: [PATCH 1/5] ENH: ptscotchDecomp : redistribute graph so ptscotch does not abort --- .../redistributeMeshPar/redistributeMeshPar.C | 42 ++- .../decompose/ptscotchDecomp/ptscotchDecomp.C | 270 +++++++++++++++++- .../decompose/ptscotchDecomp/ptscotchDecomp.H | 29 +- 3 files changed, 316 insertions(+), 25 deletions(-) diff --git a/applications/utilities/parallelProcessing/redistributeMeshPar/redistributeMeshPar.C b/applications/utilities/parallelProcessing/redistributeMeshPar/redistributeMeshPar.C index 4bb427665e..985349019f 100644 --- a/applications/utilities/parallelProcessing/redistributeMeshPar/redistributeMeshPar.C +++ b/applications/utilities/parallelProcessing/redistributeMeshPar/redistributeMeshPar.C @@ -86,9 +86,35 @@ autoPtr createMesh if (!haveMesh) { // Create dummy mesh. Only used on procs that don't have mesh. + // WIP: how to avoid parallel comms when loading IOdictionaries? + // For now just give error message. + if + ( + regIOobject::fileModificationChecking + == regIOobject::timeStampMaster + || regIOobject::fileModificationChecking + == regIOobject::inotifyMaster + ) + { + FatalErrorIn("createMesh(..)") + << "Cannot use 'fileModificationChecking' mode " + << regIOobject::fileCheckTypesNames + [ + regIOobject::fileModificationChecking + ] + << " since this uses parallel communication." + << exit(FatalError); + } + fvMesh dummyMesh ( - io, + IOobject + ( + regionName, + instDir, + runTime, + IOobject::NO_READ + ), xferCopy(pointField()), xferCopy(faceList()), xferCopy(labelList()), @@ -510,16 +536,14 @@ int main(int argc, char *argv[]) "specify the merge distance relative to the bounding box size " "(default 1E-6)" ); - // Create argList. This will check for non-existing processor dirs. # include "setRootCase.H" - //- Not useful anymore. See above. - //// Create processor directory if non-existing - //if (!Pstream::master() && !isDir(args.path())) - //{ - // Pout<< "Creating case directory " << args.path() << endl; - // mkDir(args.path()); - //} + // Create processor directory if non-existing + if (!Pstream::master() && !isDir(args.path())) + { + Pout<< "Creating case directory " << args.path() << endl; + mkDir(args.path()); + } # include "createTime.H" diff --git a/src/parallel/decompose/ptscotchDecomp/ptscotchDecomp.C b/src/parallel/decompose/ptscotchDecomp/ptscotchDecomp.C index ff262c5645..f08ab65363 100644 --- a/src/parallel/decompose/ptscotchDecomp/ptscotchDecomp.C +++ b/src/parallel/decompose/ptscotchDecomp/ptscotchDecomp.C @@ -108,6 +108,7 @@ License #include "addToRunTimeSelectionTable.H" #include "Time.H" #include "OFstream.H" +#include "globalIndex.H" extern "C" { @@ -157,16 +158,204 @@ void Foam::ptscotchDecomp::check(const int retVal, const char* str) } +//- Does prevention of 0 cell domains and calls ptscotch. +Foam::label Foam::ptscotchDecomp::decomposeZeroDomains +( + const List& initadjncy, + const List& initxadj, + const scalarField& initcWeights, + + List& finalDecomp +) const +{ + globalIndex globalCells(initxadj.size()-1); + + bool hasZeroDomain = false; + for (label procI = 0; procI < Pstream::nProcs(); procI++) + { + if (globalCells.localSize(procI) == 0) + { + hasZeroDomain = true; + break; + } + } + + if (!hasZeroDomain) + { + return decompose + ( + initadjncy, + initxadj, + initcWeights, + finalDecomp + ); + } + + + if (debug) + { + Info<< "ptscotchDecomp : have graphs with locally 0 cells." + << " trickling down." << endl; + } + + // Make sure every domain has at least one cell + // ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + // (scotch does not like zero sized domains) + // Trickle cells from processors that have them up to those that + // don't. + + + // Number of cells to send to the next processor + // (is same as number of cells next processor has to receive) + List nSendCells(Pstream::nProcs(), 0); + + for (label procI = nSendCells.size()-1; procI >=1; procI--) + { + label nLocalCells = globalCells.localSize(procI); + if (nLocalCells-nSendCells[procI] < 1) + { + nSendCells[procI-1] = nSendCells[procI]-nLocalCells+1; + } + } + + // First receive (so increasing the sizes of all arrays) + + Field xadj(initxadj); + Field adjncy(initadjncy); + scalarField cWeights(initcWeights); + + if (Pstream::myProcNo() >= 1 && nSendCells[Pstream::myProcNo()-1] > 0) + { + // Receive cells from previous processor + IPstream fromPrevProc(Pstream::blocking, Pstream::myProcNo()-1); + + Field prevXadj(fromPrevProc); + Field prevAdjncy(fromPrevProc); + scalarField prevCellWeights(fromPrevProc); + + if (prevXadj.size() != nSendCells[Pstream::myProcNo()-1]) + { + FatalErrorIn("ptscotchDecomp::decompose(..)") + << "Expected from processor " << Pstream::myProcNo()-1 + << " connectivity for " << nSendCells[Pstream::myProcNo()-1] + << " nCells but only received " << prevXadj.size() + << abort(FatalError); + } + + // Insert adjncy + prepend(prevAdjncy, adjncy); + // Adapt offsets and prepend xadj + xadj += prevAdjncy.size(); + prepend(prevXadj, xadj); + // Weights + prepend(prevCellWeights, cWeights); + } + + + // Send to my next processor + + if (nSendCells[Pstream::myProcNo()] > 0) + { + // Send cells to next processor + OPstream toNextProc(Pstream::blocking, Pstream::myProcNo()+1); + + int nCells = nSendCells[Pstream::myProcNo()]; + int startCell = xadj.size()-1 - nCells; + int startFace = xadj[startCell]; + int nFaces = adjncy.size()-startFace; + + // Send for all cell data: last nCells elements + // Send for all face data: last nFaces elements + toNextProc + << Field::subField(xadj, nCells, startCell)-startFace + << Field::subField(adjncy, nFaces, startFace) + << + ( + cWeights.size() + ? scalarField::subField(cWeights, nCells, startCell) + : scalarField(0) + ); + + // Remove data that has been sent + if (cWeights.size()) + { + cWeights.setSize(cWeights.size()-nCells); + } + adjncy.setSize(adjncy.size()-nFaces); + xadj.setSize(xadj.size() - nCells); + } + + + // Do decomposition as normal. Sets finalDecomp. + label result = decompose(adjncy, xadj, cWeights, finalDecomp); + + + if (debug) + { + Info<< "ptscotchDecomp : have graphs with locally 0 cells." + << " trickling up." << endl; + } + + + // If we sent cells across make sure we undo it + // ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + + // Receive back from next processor if I sent something + if (nSendCells[Pstream::myProcNo()] > 0) + { + IPstream fromNextProc(Pstream::blocking, Pstream::myProcNo()+1); + + List nextFinalDecomp(fromNextProc); + + if (nextFinalDecomp.size() != nSendCells[Pstream::myProcNo()]) + { + FatalErrorIn("parMetisDecomp::decompose(..)") + << "Expected from processor " << Pstream::myProcNo()+1 + << " decomposition for " << nSendCells[Pstream::myProcNo()] + << " nCells but only received " << nextFinalDecomp.size() + << abort(FatalError); + } + + append(nextFinalDecomp, finalDecomp); + } + + // Send back to previous processor. + if (Pstream::myProcNo() >= 1 && nSendCells[Pstream::myProcNo()-1] > 0) + { + OPstream toPrevProc(Pstream::blocking, Pstream::myProcNo()-1); + + int nToPrevious = nSendCells[Pstream::myProcNo()-1]; + + toPrevProc << + SubList + ( + finalDecomp, + nToPrevious, + finalDecomp.size()-nToPrevious + ); + + // Remove locally what has been sent + finalDecomp.setSize(finalDecomp.size()-nToPrevious); + } + return result; +} + + // Call scotch with options from dictionary. Foam::label Foam::ptscotchDecomp::decompose ( - List& adjncy, - List& xadj, + const List& adjncy, + const List& xadj, const scalarField& cWeights, List& finalDecomp -) +) const { + if (debug) + { + Pout<< "ptscotchDecomp : entering with xadj:" << xadj.size() << endl; + } + // // Dump graph // if (decompositionDict_.found("ptscotchCoeffs")) // { @@ -262,8 +451,7 @@ Foam::label Foam::ptscotchDecomp::decompose { WarningIn ( - "ptscotchDecomp::decompose" - "(const pointField&, const scalarField&)" + "ptscotchDecomp::decompose(..)" ) << "Illegal minimum weight " << minWeights << endl; } @@ -272,8 +460,7 @@ Foam::label Foam::ptscotchDecomp::decompose { FatalErrorIn ( - "ptscotchDecomp::decompose" - "(const pointField&, const scalarField&)" + "ptscotchDecomp::decompose(..)" ) << "Number of cell weights " << cWeights.size() << " does not equal number of cells " << xadj.size()-1 << exit(FatalError); @@ -289,8 +476,25 @@ Foam::label Foam::ptscotchDecomp::decompose + if (debug) + { + Pout<< "SCOTCH_dgraphInit" << endl; + } SCOTCH_Dgraph grafdat; check(SCOTCH_dgraphInit(&grafdat, MPI_COMM_WORLD), "SCOTCH_dgraphInit"); + + + if (debug) + { + Pout<< "SCOTCH_dgraphBuild with:" << nl + << "xadj.size() : " << xadj.size()-1 << nl + << "xadj : " << long(xadj.begin()) << nl + << "velotab : " << long(velotab.begin()) << nl + << "adjncy.size() : " << adjncy.size() << nl + << "adjncy : " << long(adjncy.begin()) << nl + << endl; + } + check ( SCOTCH_dgraphBuild @@ -302,19 +506,25 @@ Foam::label Foam::ptscotchDecomp::decompose const_cast(xadj.begin()), // vertloctab, start index per cell into // adjncy - &xadj[1], // vendloctab, end index ,, + const_cast(&xadj[1]),// vendloctab, end index ,, - velotab.begin(), // veloloctab, vertex weights + const_cast(velotab.begin()),// veloloctab, vtx weights NULL, // vlblloctab adjncy.size(), // edgelocnbr, number of arcs adjncy.size(), // edgelocsiz - adjncy.begin(), // edgeloctab + const_cast(adjncy.begin()), // edgeloctab NULL, // edgegsttab NULL // edlotab, edge weights ), "SCOTCH_dgraphBuild" ); + + + if (debug) + { + Pout<< "SCOTCH_dgraphCheck" << endl; + } check(SCOTCH_dgraphCheck(&grafdat), "SCOTCH_dgraphCheck"); @@ -322,6 +532,10 @@ Foam::label Foam::ptscotchDecomp::decompose // ~~~~~~~~~~~~ // (fully connected network topology since using switch) + if (debug) + { + Pout<< "SCOTCH_archInit" << endl; + } SCOTCH_Arch archdat; check(SCOTCH_archInit(&archdat), "SCOTCH_archInit"); @@ -349,6 +563,10 @@ Foam::label Foam::ptscotchDecomp::decompose } else { + if (debug) + { + Pout<< "SCOTCH_archCmplt" << endl; + } check ( SCOTCH_archCmplt(&archdat, nProcessors_), @@ -373,6 +591,10 @@ Foam::label Foam::ptscotchDecomp::decompose ); # endif + if (debug) + { + Pout<< "SCOTCH_dgraphMap" << endl; + } finalDecomp.setSize(xadj.size()-1); finalDecomp = 0; check @@ -406,6 +628,10 @@ Foam::label Foam::ptscotchDecomp::decompose // "SCOTCH_graphPart" //); + if (debug) + { + Pout<< "SCOTCH_dgraphExit" << endl; + } // Release storage for graph SCOTCH_dgraphExit(&grafdat); // Release storage for strategy @@ -465,7 +691,13 @@ Foam::labelList Foam::ptscotchDecomp::decompose // Decompose using default weights List finalDecomp; - decompose(cellCells.m(), cellCells.offsets(), pointWeights, finalDecomp); + decomposeZeroDomains + ( + cellCells.m(), + cellCells.offsets(), + pointWeights, + finalDecomp + ); // Copy back to labelList labelList decomp(finalDecomp.size()); @@ -510,7 +742,13 @@ Foam::labelList Foam::ptscotchDecomp::decompose // Decompose using weights List finalDecomp; - decompose(cellCells.m(), cellCells.offsets(), pointWeights, finalDecomp); + decomposeZeroDomains + ( + cellCells.m(), + cellCells.offsets(), + pointWeights, + finalDecomp + ); // Rework back into decomposition for original mesh labelList fineDistribution(agglom.size()); @@ -557,7 +795,13 @@ Foam::labelList Foam::ptscotchDecomp::decompose // Decompose using weights List finalDecomp; - decompose(cellCells.m(), cellCells.offsets(), cWeights, finalDecomp); + decomposeZeroDomains + ( + cellCells.m(), + cellCells.offsets(), + cWeights, + finalDecomp + ); // Copy back to labelList labelList decomp(finalDecomp.size()); diff --git a/src/parallel/decompose/ptscotchDecomp/ptscotchDecomp.H b/src/parallel/decompose/ptscotchDecomp/ptscotchDecomp.H index 7471f5ddd7..6a6937ac79 100644 --- a/src/parallel/decompose/ptscotchDecomp/ptscotchDecomp.H +++ b/src/parallel/decompose/ptscotchDecomp/ptscotchDecomp.H @@ -50,16 +50,33 @@ class ptscotchDecomp { // Private Member Functions + //- Insert list in front of list. + template + static void prepend(const UList&, List&); + //- Insert list at end of list. + template + static void append(const UList&, List&); + //- Check and print error message static void check(const int, const char*); + //- Decompose with optionally zero sized domains + label decomposeZeroDomains + ( + const List& initadjncy, + const List& initxadj, + const scalarField& initcWeights, + List& finalDecomp + ) const; + + //- Decompose label decompose ( - List& adjncy, - List& xadj, + const List& adjncy, + const List& xadj, const scalarField& cWeights, List& finalDecomp - ); + ) const; //- Disallow default bitwise copy construct and assignment void operator=(const ptscotchDecomp&); @@ -135,6 +152,12 @@ public: // * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * // +#ifdef NoRepository +# include "ptscotchDecompTemplates.C" +#endif + +// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * // + #endif // ************************************************************************* // From a40bfad1c931ece4452bf4270987b649f724771f Mon Sep 17 00:00:00 2001 From: mattijs Date: Tue, 30 Nov 2010 21:29:11 +0000 Subject: [PATCH 2/5] BUG: ptscotchDecomp : consistent dummy variant --- .../ptscotchDecomp/dummyPtscotchDecomp.C | 28 +++++++++++++++++-- 1 file changed, 25 insertions(+), 3 deletions(-) diff --git a/src/dummyThirdParty/ptscotchDecomp/dummyPtscotchDecomp.C b/src/dummyThirdParty/ptscotchDecomp/dummyPtscotchDecomp.C index 9e59239fd1..d4c2802d58 100644 --- a/src/dummyThirdParty/ptscotchDecomp/dummyPtscotchDecomp.C +++ b/src/dummyThirdParty/ptscotchDecomp/dummyPtscotchDecomp.C @@ -58,13 +58,35 @@ void Foam::ptscotchDecomp::check(const int retVal, const char* str) {} +Foam::label Foam::ptscotchDecomp::decomposeZeroDomains +( + const List& initxadj, + const List& initadjncy, + const scalarField& initcWeights, + + List& finalDecomp +) const +{ + FatalErrorIn + ( + "label ptscotchDecomp::decompose" + "(" + "const List&, " + "const List&, " + "const scalarField&, " + "List&" + ")" + ) << notImplementedMessage << exit(FatalError); + + return -1; +} Foam::label Foam::ptscotchDecomp::decompose ( - List& adjncy, - List& xadj, + const List& adjncy, + const List& xadj, const scalarField& cWeights, List& finalDecomp -) +) const { FatalErrorIn ( From d49c96579824a6293ecbb6dab8e5737afa3b0f23 Mon Sep 17 00:00:00 2001 From: mattijs Date: Tue, 30 Nov 2010 21:29:46 +0000 Subject: [PATCH 3/5] ENH: ReleaseNotes-dev : updated --- ReleaseNotes-dev | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ReleaseNotes-dev b/ReleaseNotes-dev index 378e088c34..6af144a309 100644 --- a/ReleaseNotes-dev +++ b/ReleaseNotes-dev @@ -181,6 +181,8 @@ + =blockMesh=: specify patches via dictionary instead of type only. This makes rereading the boundary file superfluous. see e.g. pitzDailyDirectMapped tutorial. + + =setSet=: allows time range (e.g. 0:100) in combination with -batch argument + to execute the commands for multiple times. * Post-processing + =foamToEnsight=: parallel continuous data. new =-nodeValues= option to generate and output nodal field data. From a30296303286e5eb4ace54d8e1ddf3931d3b2ecc Mon Sep 17 00:00:00 2001 From: mattijs Date: Wed, 1 Dec 2010 10:46:00 +0000 Subject: [PATCH 4/5] ENH: ptscotchDecomp : allow zero sized domains --- .../ptscotchDecomp/ptscotchDecompTemplates.C | 76 +++++++++++++++++++ 1 file changed, 76 insertions(+) create mode 100644 src/parallel/decompose/ptscotchDecomp/ptscotchDecompTemplates.C diff --git a/src/parallel/decompose/ptscotchDecomp/ptscotchDecompTemplates.C b/src/parallel/decompose/ptscotchDecomp/ptscotchDecompTemplates.C new file mode 100644 index 0000000000..e133dc9ceb --- /dev/null +++ b/src/parallel/decompose/ptscotchDecomp/ptscotchDecompTemplates.C @@ -0,0 +1,76 @@ +/*---------------------------------------------------------------------------*\ + ========= | + \\ / F ield | OpenFOAM: The Open Source CFD Toolbox + \\ / O peration | + \\ / A nd | Copyright (C) 2010-2010 OpenCFD Ltd. + \\/ M anipulation | +------------------------------------------------------------------------------- +License + This file is part of OpenFOAM. + + OpenFOAM is free software: you can redistribute it and/or modify it + under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + OpenFOAM is distributed in the hope that it will be useful, but WITHOUT + ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + for more details. + + You should have received a copy of the GNU General Public License + along with OpenFOAM. If not, see . + +\*---------------------------------------------------------------------------*/ + +#include "ptscotchDecomp.H" + +// * * * * * * * * * * * * * * * Member Functions * * * * * * * * * * * * * // + +// Insert at front of list +template +void Foam::ptscotchDecomp::prepend +( + const UList& extraLst, + List& lst +) +{ + label nExtra = extraLst.size(); + + // Make space for initial elements + lst.setSize(lst.size() + nExtra); + for (label i = lst.size()-1; i >= nExtra; i--) + { + lst[i] = lst[i-nExtra]; + } + + // Insert at front + forAll(extraLst, i) + { + lst[i] = extraLst[i]; + } +} + + +// Insert at back of list +template +void Foam::ptscotchDecomp::append +( + const UList& extraLst, + List& lst +) +{ + label sz = lst.size(); + + // Make space for initial elements + lst.setSize(sz + extraLst.size()); + + // Insert at back + forAll(extraLst, i) + { + lst[sz++] = extraLst[i]; + } +} + + +// ************************************************************************* // From d775a901c5568414f3016a93a4cf1a2cc6cd75f2 Mon Sep 17 00:00:00 2001 From: mattijs Date: Thu, 2 Dec 2010 13:39:17 +0000 Subject: [PATCH 5/5] BUG: fileMonitor : non-parallel running use state directly. --- src/OSspecific/POSIX/fileMonitor.C | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/OSspecific/POSIX/fileMonitor.C b/src/OSspecific/POSIX/fileMonitor.C index 318262d9d7..329e52ccc0 100644 --- a/src/OSspecific/POSIX/fileMonitor.C +++ b/src/OSspecific/POSIX/fileMonitor.C @@ -603,6 +603,10 @@ void Foam::fileMonitor::updateStates } } } + else + { + state_ = localState_; + } }