diff --git a/src/OpenFOAM/db/IOstreams/Pstreams/PstreamReduceOps.H b/src/OpenFOAM/db/IOstreams/Pstreams/PstreamReduceOps.H index bd8f5ab1dc..a4fdb29ed8 100644 --- a/src/OpenFOAM/db/IOstreams/Pstreams/PstreamReduceOps.H +++ b/src/OpenFOAM/db/IOstreams/Pstreams/PstreamReduceOps.H @@ -2,7 +2,7 @@ ========= | \\ / F ield | OpenFOAM: The Open Source CFD Toolbox \\ / O peration | - \\ / A nd | Copyright (C) 2011-2012 OpenFOAM Foundation + \\ / A nd | Copyright (C) 2011-2013 OpenFOAM Foundation \\/ M anipulation | ------------------------------------------------------------------------------- License @@ -48,6 +48,12 @@ void reduce const label comm ) { + if (UPstream::warnComm != -1 && comm != UPstream::warnComm) + { + Pout<< "** reducing:" << Value << " with comm:" << comm + << endl; + error::printStack(Pout); + } Pstream::gather(comms, Value, bop, tag, comm); Pstream::scatter(comms, Value, tag, comm); } diff --git a/src/OpenFOAM/db/IOstreams/Pstreams/UPstream.C b/src/OpenFOAM/db/IOstreams/Pstreams/UPstream.C index 20d4609265..ab0560cd09 100644 --- a/src/OpenFOAM/db/IOstreams/Pstreams/UPstream.C +++ b/src/OpenFOAM/db/IOstreams/Pstreams/UPstream.C @@ -2,7 +2,7 @@ ========= | \\ / F ield | OpenFOAM: The Open Source CFD Toolbox \\ / O peration | - \\ / A nd | Copyright (C) 2012 OpenFOAM Foundation + \\ / A nd | Copyright (C) 2012-2013 OpenFOAM Foundation \\/ M anipulation | ------------------------------------------------------------------------------- License @@ -270,6 +270,19 @@ Foam::label Foam::UPstream::allocateCommunicator forAll(procIDs_[index], i) { procIDs_[index][i] = subRanks[i]; + + // Enforce incremental order (so index is rank in next communicator) + if (i >= 1 && subRanks[i] <= subRanks[i-1]) + { + FatalErrorIn + ( + "UPstream::allocateCommunicator" + "(const label, const labelList&, const bool)" + ) << "subranks not sorted : " << subRanks + << " when allocating subcommunicator from parent " + << parentIndex + << Foam::abort(FatalError); + } } parentCommunicator_[index] = parentIndex; @@ -277,7 +290,7 @@ Foam::label Foam::UPstream::allocateCommunicator treeCommunication_[index] = calcTreeComm(procIDs_[index].size()); - if (doPstream) + if (doPstream && parRun()) { allocatePstreamCommunicator(parentIndex, index); } @@ -297,7 +310,7 @@ void Foam::UPstream::freeCommunicator << " myProcNo : " << myProcNo_[communicator] << endl << endl; - if (doPstream) + if (doPstream && parRun()) { freePstreamCommunicator(communicator); } @@ -437,6 +450,10 @@ addcommsTypeToOpt addcommsTypeToOpt_("commsType"); Foam::label Foam::UPstream::worldComm(0); +// Warn for use of any communicator +Foam::label Foam::UPstream::warnComm(-1); + + // Number of polling cycles in processor updates int Foam::UPstream::nPollProcInterfaces ( diff --git a/src/OpenFOAM/db/IOstreams/Pstreams/UPstream.H b/src/OpenFOAM/db/IOstreams/Pstreams/UPstream.H index 2eaa66847e..89f49f9509 100644 --- a/src/OpenFOAM/db/IOstreams/Pstreams/UPstream.H +++ b/src/OpenFOAM/db/IOstreams/Pstreams/UPstream.H @@ -2,7 +2,7 @@ ========= | \\ / F ield | OpenFOAM: The Open Source CFD Toolbox \\ / O peration | - \\ / A nd | Copyright (C) 2012 OpenFOAM Foundation + \\ / A nd | Copyright (C) 2012-2013 OpenFOAM Foundation \\/ M anipulation | ------------------------------------------------------------------------------- License @@ -269,6 +269,10 @@ public: //- Default communicator (all processors) static label worldComm; + //- Debugging: warn for use of any communicator differing from warnComm + static label warnComm; + + // Constructors //- Construct given optional buffer size @@ -319,17 +323,17 @@ public: comm_(allocateCommunicator(parent, subRanks, doPstream)) {} - communicator(const label parent) - : - comm_ - ( - allocateCommunicator - ( - parent, - identity(UPstream::nProcs(parent)) - ) - ) - {} +// communicator(const label parent) +// : +// comm_ +// ( +// allocateCommunicator +// ( +// parent, +// identity(UPstream::nProcs(parent)) +// ) +// ) +// {} ~communicator() { @@ -407,17 +411,24 @@ public: return myProcNo_[communicator]; } + static label parent(const label communicator) + { + return parentCommunicator_(communicator); + } + + // //- Process IDs // static const List& procIDs() // { // return procIDs_; // } -// -// //- Process ID of given process index -// static int procID(int procNo) -// { -// return procIDs_[procNo]; -// } + + + //- Process ID of given process index + static List& procID(int communicator) + { + return procIDs_[communicator]; + } //- Process index of first slave static int firstSlave() diff --git a/src/OpenFOAM/db/error/messageStream.C b/src/OpenFOAM/db/error/messageStream.C index 8f5f6cbffa..2aabacd9ba 100644 --- a/src/OpenFOAM/db/error/messageStream.C +++ b/src/OpenFOAM/db/error/messageStream.C @@ -2,7 +2,7 @@ ========= | \\ / F ield | OpenFOAM: The Open Source CFD Toolbox \\ / O peration | - \\ / A nd | Copyright (C) 2011 OpenFOAM Foundation + \\ / A nd | Copyright (C) 2011-2013 OpenFOAM Foundation \\/ M anipulation | ------------------------------------------------------------------------------- License @@ -166,6 +166,74 @@ Foam::OSstream& Foam::messageStream::operator() } +Foam::OSstream& Foam::messageStream::operator()(const label communicator) +{ + if (UPstream::warnComm != -1 && communicator != UPstream::warnComm) + { + Pout<< "** messageStream with comm:" << communicator + << endl; + error::printStack(Pout); + } + + if (communicator == UPstream::worldComm) + { + return operator()(); + } + else + { + bool master = UPstream::master(communicator); + + if (level) + { + bool collect = (severity_ == INFO || severity_ == WARNING); + + // Report the error + if (!master && collect) + { + return Snull; + } + else + { + if (title().size()) + { + if (Pstream::parRun() && !collect) + { + Pout<< title().c_str(); + } + else + { + Sout<< title().c_str(); + } + } + + if (maxErrors_) + { + errorCount_++; + + if (errorCount_ >= maxErrors_) + { + FatalErrorIn("messageStream::operator OSstream&()") + << "Too many errors" + << abort(FatalError); + } + } + + if (Pstream::parRun() && !collect) + { + return Pout; + } + else + { + return Sout; + } + } + } + } + + return Snull; +} + + Foam::messageStream::operator Foam::OSstream&() { if (level) diff --git a/src/OpenFOAM/db/error/messageStream.H b/src/OpenFOAM/db/error/messageStream.H index eda195375e..54c41ab564 100644 --- a/src/OpenFOAM/db/error/messageStream.H +++ b/src/OpenFOAM/db/error/messageStream.H @@ -184,6 +184,11 @@ public: const dictionary& ); + //- Convert to OSstream + // Use Info for default communicator, use Pout + // on master for non-default one. + OSstream& operator()(const label communicator); + //- Convert to OSstream for << operations operator OSstream&(); diff --git a/src/OpenFOAM/fields/Fields/Field/FieldFunctions.C b/src/OpenFOAM/fields/Fields/Field/FieldFunctions.C index c26227cd00..b33a9bc091 100644 --- a/src/OpenFOAM/fields/Fields/Field/FieldFunctions.C +++ b/src/OpenFOAM/fields/Fields/Field/FieldFunctions.C @@ -2,7 +2,7 @@ ========= | \\ / F ield | OpenFOAM: The Open Source CFD Toolbox \\ / O peration | - \\ / A nd | Copyright (C) 2011-2012 OpenFOAM Foundation + \\ / A nd | Copyright (C) 2011-2013 OpenFOAM Foundation \\/ M anipulation | ------------------------------------------------------------------------------- License @@ -477,10 +477,10 @@ TMP_UNARY_FUNCTION(Type, average) #define G_UNARY_FUNCTION(ReturnType, gFunc, Func, rFunc) \ \ template \ -ReturnType gFunc(const UList& f) \ +ReturnType gFunc(const UList& f, const int comm) \ { \ ReturnType res = Func(f); \ - reduce(res, rFunc##Op()); \ + reduce(res, rFunc##Op(), Pstream::msgType(), comm); \ return res; \ } \ TMP_UNARY_FUNCTION(ReturnType, gFunc) @@ -495,27 +495,41 @@ G_UNARY_FUNCTION(Type, gSumCmptMag, sumCmptMag, sum) #undef G_UNARY_FUNCTION template -scalar gSumProd(const UList& f1, const UList& f2) +scalar gSumProd +( + const UList& f1, + const UList& f2, + const int comm +) { scalar SumProd = sumProd(f1, f2); - reduce(SumProd, sumOp()); + reduce(SumProd, sumOp(), Pstream::msgType(), comm); return SumProd; } template -Type gSumCmptProd(const UList& f1, const UList& f2) +Type gSumCmptProd +( + const UList& f1, + const UList& f2, + const int comm +) { Type SumProd = sumCmptProd(f1, f2); - reduce(SumProd, sumOp()); + reduce(SumProd, sumOp(), Pstream::msgType(), comm); return SumProd; } template -Type gAverage(const UList& f) +Type gAverage +( + const UList& f, + const int comm +) { label n = f.size(); Type s = sum(f); - sumReduce(s, n); + sumReduce(s, n, Pstream::msgType(), comm); if (n > 0) { diff --git a/src/OpenFOAM/fields/Fields/Field/FieldFunctions.H b/src/OpenFOAM/fields/Fields/Field/FieldFunctions.H index 869346e29a..f1fa2fdd96 100644 --- a/src/OpenFOAM/fields/Fields/Field/FieldFunctions.H +++ b/src/OpenFOAM/fields/Fields/Field/FieldFunctions.H @@ -2,7 +2,7 @@ ========= | \\ / F ield | OpenFOAM: The Open Source CFD Toolbox \\ / O peration | - \\ / A nd | Copyright (C) 2011-2012 OpenFOAM Foundation + \\ / A nd | Copyright (C) 2011-2013 OpenFOAM Foundation \\/ M anipulation | ------------------------------------------------------------------------------- License @@ -25,6 +25,7 @@ License #define TEMPLATE template #include "FieldFunctionsM.H" +#include "UPstream.H" // * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * // @@ -201,7 +202,7 @@ TMP_UNARY_FUNCTION(Type, average) #define G_UNARY_FUNCTION(ReturnType, gFunc, Func, rFunc) \ \ template \ -ReturnType gFunc(const UList& f); \ +ReturnType gFunc(const UList& f, const int comm = UPstream::worldComm); \ TMP_UNARY_FUNCTION(ReturnType, gFunc) G_UNARY_FUNCTION(Type, gMax, max, max) @@ -214,13 +215,27 @@ G_UNARY_FUNCTION(Type, gSumCmptMag, sumCmptMag, sum) #undef G_UNARY_FUNCTION template -scalar gSumProd(const UList& f1, const UList& f2); +scalar gSumProd +( + const UList& f1, + const UList& f2, + const int comm = UPstream::worldComm +); template -Type gSumCmptProd(const UList& f1, const UList& f2); +Type gSumCmptProd +( + const UList& f1, + const UList& f2, + const int comm = UPstream::worldComm +); template -Type gAverage(const UList& f); +Type gAverage +( + const UList& f, + const int comm = UPstream::worldComm +); TMP_UNARY_FUNCTION(Type, gAverage) diff --git a/src/OpenFOAM/matrices/LduMatrix/LduMatrix/SolverPerformance.C b/src/OpenFOAM/matrices/LduMatrix/LduMatrix/SolverPerformance.C index 5f307718eb..5cf4e60efa 100644 --- a/src/OpenFOAM/matrices/LduMatrix/LduMatrix/SolverPerformance.C +++ b/src/OpenFOAM/matrices/LduMatrix/LduMatrix/SolverPerformance.C @@ -2,7 +2,7 @@ ========= | \\ / F ield | OpenFOAM: The Open Source CFD Toolbox \\ / O peration | - \\ / A nd | Copyright (C) 2011-2012 OpenFOAM Foundation + \\ / A nd | Copyright (C) 2011-2013 OpenFOAM Foundation \\/ M anipulation | ------------------------------------------------------------------------------- License @@ -24,6 +24,7 @@ License \*---------------------------------------------------------------------------*/ #include "SolverPerformance.H" +#include "IOstreams.H" // * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * // @@ -64,7 +65,8 @@ bool Foam::SolverPerformance::checkConvergence { if (debug >= 2) { - Info<< solverName_ + //Info<< solverName_ + Pout<< solverName_ << ": Iteration " << noIterations_ << " residual = " << finalResidual_ << endl; diff --git a/src/OpenFOAM/matrices/lduMatrix/lduAddressing/lduInterface/processorLduInterface.H b/src/OpenFOAM/matrices/lduMatrix/lduAddressing/lduInterface/processorLduInterface.H index ac0f895900..65d10613ee 100644 --- a/src/OpenFOAM/matrices/lduMatrix/lduAddressing/lduInterface/processorLduInterface.H +++ b/src/OpenFOAM/matrices/lduMatrix/lduAddressing/lduInterface/processorLduInterface.H @@ -2,7 +2,7 @@ ========= | \\ / F ield | OpenFOAM: The Open Source CFD Toolbox \\ / O peration | - \\ / A nd | Copyright (C) 2011 OpenFOAM Foundation + \\ / A nd | Copyright (C) 2011-2013 OpenFOAM Foundation \\/ M anipulation | ------------------------------------------------------------------------------- License @@ -84,7 +84,7 @@ public: // Access - //- Return communicator used for sending + //- Return communicator used for parallel communication virtual int comm() const = 0; //- Return processor number (rank in communicator) diff --git a/src/OpenFOAM/matrices/lduMatrix/lduMatrix/lduMatrixSolver.C b/src/OpenFOAM/matrices/lduMatrix/lduMatrix/lduMatrixSolver.C index bedb55529e..cee5c1dc2f 100644 --- a/src/OpenFOAM/matrices/lduMatrix/lduMatrix/lduMatrixSolver.C +++ b/src/OpenFOAM/matrices/lduMatrix/lduMatrix/lduMatrixSolver.C @@ -2,7 +2,7 @@ ========= | \\ / F ield | OpenFOAM: The Open Source CFD Toolbox \\ / O peration | - \\ / A nd | Copyright (C) 2011-2012 OpenFOAM Foundation + \\ / A nd | Copyright (C) 2011-2013 OpenFOAM Foundation \\/ M anipulation | ------------------------------------------------------------------------------- License @@ -186,10 +186,15 @@ Foam::scalar Foam::lduMatrix::solver::normFactor { // --- Calculate A dot reference value of psi matrix_.sumA(tmpField, interfaceBouCoeffs_, interfaces_); - tmpField *= gAverage(psi); + + tmpField *= gAverage(psi, matrix_.lduMesh_.comm()); return - gSum(mag(Apsi - tmpField) + mag(source - tmpField)) + gSum + ( + (mag(Apsi - tmpField) + mag(source - tmpField))(), + matrix_.lduMesh_.comm() + ) + solverPerformance::small_; // At convergence this simpler method is equivalent to the above diff --git a/src/OpenFOAM/matrices/lduMatrix/solvers/GAMG/GAMGAgglomerations/GAMGAgglomeration/GAMGAgglomerateLduAddressing.C b/src/OpenFOAM/matrices/lduMatrix/solvers/GAMG/GAMGAgglomerations/GAMGAgglomeration/GAMGAgglomerateLduAddressing.C index 725433247c..1cef9aabb2 100644 --- a/src/OpenFOAM/matrices/lduMatrix/solvers/GAMG/GAMGAgglomerations/GAMGAgglomeration/GAMGAgglomerateLduAddressing.C +++ b/src/OpenFOAM/matrices/lduMatrix/solvers/GAMG/GAMGAgglomerations/GAMGAgglomeration/GAMGAgglomerateLduAddressing.C @@ -2,7 +2,7 @@ ========= | \\ / F ield | OpenFOAM: The Open Source CFD Toolbox \\ / O peration | - \\ / A nd | Copyright (C) 2011 OpenFOAM Foundation + \\ / A nd | Copyright (C) 2011-2013 OpenFOAM Foundation \\/ M anipulation | ------------------------------------------------------------------------------- License @@ -280,6 +280,7 @@ void Foam::GAMGAgglomeration::agglomerateLduAddressing coarseInterfaceAddr, coarseInterfaces, fineMeshAddr.patchSchedule(), + fineMesh.comm(), true ) ); diff --git a/src/OpenFOAM/matrices/lduMatrix/solvers/GAMG/GAMGAgglomerations/GAMGAgglomeration/GAMGAgglomeration.C b/src/OpenFOAM/matrices/lduMatrix/solvers/GAMG/GAMGAgglomerations/GAMGAgglomeration/GAMGAgglomeration.C index 51421910c2..d7c8e79080 100644 --- a/src/OpenFOAM/matrices/lduMatrix/solvers/GAMG/GAMGAgglomerations/GAMGAgglomeration/GAMGAgglomeration.C +++ b/src/OpenFOAM/matrices/lduMatrix/solvers/GAMG/GAMGAgglomerations/GAMGAgglomeration/GAMGAgglomeration.C @@ -2,7 +2,7 @@ ========= | \\ / F ield | OpenFOAM: The Open Source CFD Toolbox \\ / O peration | - \\ / A nd | Copyright (C) 2011 OpenFOAM Foundation + \\ / A nd | Copyright (C) 2011-2013 OpenFOAM Foundation \\/ M anipulation | ------------------------------------------------------------------------------- License @@ -57,7 +57,7 @@ bool Foam::GAMGAgglomeration::continueAgglomerating { // Check the need for further agglomeration on all processors bool contAgg = nCoarseCells >= nCellsInCoarsestLevel_; - reduce(contAgg, andOp()); + mesh().reduce(contAgg, andOp()); return contAgg; } diff --git a/src/OpenFOAM/matrices/lduMatrix/solvers/GAMG/GAMGAgglomerations/GAMGAgglomeration/GAMGAgglomeration.H b/src/OpenFOAM/matrices/lduMatrix/solvers/GAMG/GAMGAgglomerations/GAMGAgglomeration/GAMGAgglomeration.H index e35cdd379f..78e9aa536b 100644 --- a/src/OpenFOAM/matrices/lduMatrix/solvers/GAMG/GAMGAgglomerations/GAMGAgglomeration/GAMGAgglomeration.H +++ b/src/OpenFOAM/matrices/lduMatrix/solvers/GAMG/GAMGAgglomerations/GAMGAgglomeration/GAMGAgglomeration.H @@ -2,7 +2,7 @@ ========= | \\ / F ield | OpenFOAM: The Open Source CFD Toolbox \\ / O peration | - \\ / A nd | Copyright (C) 2011 OpenFOAM Foundation + \\ / A nd | Copyright (C) 2011-2013 OpenFOAM Foundation \\/ M anipulation | ------------------------------------------------------------------------------- License @@ -51,6 +51,7 @@ namespace Foam class lduMesh; class lduMatrix; +class mapDistribute; /*---------------------------------------------------------------------------*\ Class GAMGAgglomeration Declaration @@ -77,6 +78,9 @@ protected: // Maps from the finer to the coarser level. PtrList restrictAddressing_; + //- For cross-processor operation + autoPtr restrictDistributeMapPtr_; + //- Face restriction addressing array. // Maps from the finer to the coarser level. // Positive indices map the finer faces which form part of the boundary @@ -85,6 +89,9 @@ protected: // coarser cells to minus the corresponding coarser cell index minus 1. PtrList faceRestrictAddressing_; + //- For cross-processor operation + autoPtr faceRestrictDistributeMapPtr_; + //- Hierarchy of mesh addressing PtrList meshLevels_; diff --git a/src/OpenFOAM/matrices/lduMatrix/solvers/GAMG/GAMGAgglomerations/GAMGAgglomeration/GAMGAgglomerationTemplates.C b/src/OpenFOAM/matrices/lduMatrix/solvers/GAMG/GAMGAgglomerations/GAMGAgglomeration/GAMGAgglomerationTemplates.C index 28898f138d..7b4cbfcb8b 100644 --- a/src/OpenFOAM/matrices/lduMatrix/solvers/GAMG/GAMGAgglomerations/GAMGAgglomeration/GAMGAgglomerationTemplates.C +++ b/src/OpenFOAM/matrices/lduMatrix/solvers/GAMG/GAMGAgglomerations/GAMGAgglomeration/GAMGAgglomerationTemplates.C @@ -2,7 +2,7 @@ ========= | \\ / F ield | OpenFOAM: The Open Source CFD Toolbox \\ / O peration | - \\ / A nd | Copyright (C) 2011 OpenFOAM Foundation + \\ / A nd | Copyright (C) 2011-2013 OpenFOAM Foundation \\/ M anipulation | ------------------------------------------------------------------------------- License @@ -24,6 +24,7 @@ License \*---------------------------------------------------------------------------*/ #include "GAMGAgglomeration.H" +#include "mapDistribute.H" // * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * // @@ -37,24 +38,57 @@ void Foam::GAMGAgglomeration::restrictField { const labelList& fineToCoarse = restrictAddressing_[fineLevelIndex]; - if (ff.size() != fineToCoarse.size()) + if (restrictDistributeMapPtr_.valid()) { - FatalErrorIn - ( - "void GAMGAgglomeration::restrictField" - "(Field& cf, const Field& ff, " - "const label fineLevelIndex) const" - ) << "field does not correspond to level " << fineLevelIndex - << " sizes: field = " << ff.size() - << " level = " << fineToCoarse.size() - << abort(FatalError); + const mapDistribute& map = restrictDistributeMapPtr_(); + Field combinedFf(map.constructSize()); + forAll(ff, i) + { + combinedFf[i] = ff[i]; + } + map.distribute(combinedFf); + + if (combinedFf.size() != fineToCoarse.size()) + { + FatalErrorIn + ( + "void GAMGAgglomeration::restrictField" + "(Field& cf, const Field& ff, " + "const label fineLevelIndex) const" + ) << "field does not correspond to level " << fineLevelIndex + << " sizes: field = " << combinedFf.size() + << " level = " << fineToCoarse.size() + << abort(FatalError); + } + + cf = pTraits::zero; + + forAll(combinedFf, i) + { + cf[fineToCoarse[i]] += combinedFf[i]; + } } - - cf = pTraits::zero; - - forAll(ff, i) + else { - cf[fineToCoarse[i]] += ff[i]; + if (ff.size() != fineToCoarse.size()) + { + FatalErrorIn + ( + "void GAMGAgglomeration::restrictField" + "(Field& cf, const Field& ff, " + "const label fineLevelIndex) const" + ) << "field does not correspond to level " << fineLevelIndex + << " sizes: field = " << ff.size() + << " level = " << fineToCoarse.size() + << abort(FatalError); + } + + cf = pTraits::zero; + + forAll(ff, i) + { + cf[fineToCoarse[i]] += ff[i]; + } } } @@ -69,15 +103,40 @@ void Foam::GAMGAgglomeration::restrictFaceField { const labelList& fineToCoarse = faceRestrictAddressing_[fineLevelIndex]; - cf = pTraits::zero; - - forAll(fineToCoarse, ffacei) + if (faceRestrictDistributeMapPtr_.valid()) { - label cFace = fineToCoarse[ffacei]; - - if (cFace >= 0) + const mapDistribute& map = faceRestrictDistributeMapPtr_(); + Field combinedFf(map.constructSize()); + forAll(ff, i) { - cf[cFace] += ff[ffacei]; + combinedFf[i] = ff[i]; + } + map.distribute(combinedFf); + + cf = pTraits::zero; + + forAll(fineToCoarse, ffacei) + { + label cFace = fineToCoarse[ffacei]; + + if (cFace >= 0) + { + cf[cFace] += combinedFf[ffacei]; + } + } + } + else + { + cf = pTraits::zero; + + forAll(fineToCoarse, ffacei) + { + label cFace = fineToCoarse[ffacei]; + + if (cFace >= 0) + { + cf[cFace] += ff[ffacei]; + } } } } @@ -93,9 +152,23 @@ void Foam::GAMGAgglomeration::prolongField { const labelList& fineToCoarse = restrictAddressing_[coarseLevelIndex]; - forAll(fineToCoarse, i) + if (restrictDistributeMapPtr_.valid()) { - ff[i] = cf[fineToCoarse[i]]; + const mapDistribute& map = restrictDistributeMapPtr_(); + Field combinedCf(cf); + map.reverseDistribute(fineToCoarse.size(), combinedCf); + + forAll(fineToCoarse, i) + { + ff[i] = combinedCf[fineToCoarse[i]]; + } + } + else + { + forAll(fineToCoarse, i) + { + ff[i] = cf[fineToCoarse[i]]; + } } } diff --git a/src/OpenFOAM/matrices/lduMatrix/solvers/GAMG/GAMGSolver.H b/src/OpenFOAM/matrices/lduMatrix/solvers/GAMG/GAMGSolver.H index 99ad830f51..f62a9341a2 100644 --- a/src/OpenFOAM/matrices/lduMatrix/solvers/GAMG/GAMGSolver.H +++ b/src/OpenFOAM/matrices/lduMatrix/solvers/GAMG/GAMGSolver.H @@ -43,9 +43,7 @@ Description SourceFiles GAMGSolver.C - GAMGSolverCalcAgglomeration.C - GAMGSolverMakeCoarseMatrix.C - GAMGSolverOperations.C + GAMGSolverAgglomerateMatrix.C GAMGSolverInterpolate.C GAMGSolverScale.C GAMGSolverSolve.C diff --git a/src/OpenFOAM/matrices/lduMatrix/solvers/GAMG/GAMGSolverAgglomerateMatrix.C b/src/OpenFOAM/matrices/lduMatrix/solvers/GAMG/GAMGSolverAgglomerateMatrix.C index 4b8ef34d5d..fe5b032a09 100644 --- a/src/OpenFOAM/matrices/lduMatrix/solvers/GAMG/GAMGSolverAgglomerateMatrix.C +++ b/src/OpenFOAM/matrices/lduMatrix/solvers/GAMG/GAMGSolverAgglomerateMatrix.C @@ -2,7 +2,7 @@ ========= | \\ / F ield | OpenFOAM: The Open Source CFD Toolbox \\ / O peration | - \\ / A nd | Copyright (C) 2011 OpenFOAM Foundation + \\ / A nd | Copyright (C) 2011-2013 OpenFOAM Foundation \\/ M anipulation | ------------------------------------------------------------------------------- License @@ -126,7 +126,7 @@ void Foam::GAMGSolver::agglomerateMatrix(const label fineLevelIndex) } - // Check if matrix is assymetric and if so agglomerate both upper and lower + // Check if matrix is asymetric and if so agglomerate both upper and lower // coefficients ... if (fineMatrix.hasLower()) { diff --git a/src/OpenFOAM/matrices/lduMatrix/solvers/GAMG/GAMGSolverScale.C b/src/OpenFOAM/matrices/lduMatrix/solvers/GAMG/GAMGSolverScale.C index cbdc654d2e..cd787047bb 100644 --- a/src/OpenFOAM/matrices/lduMatrix/solvers/GAMG/GAMGSolverScale.C +++ b/src/OpenFOAM/matrices/lduMatrix/solvers/GAMG/GAMGSolverScale.C @@ -58,7 +58,11 @@ void Foam::GAMGSolver::scale } vector2D scalingVector(scalingFactorNum, scalingFactorDenom); - reduce(scalingVector, sumOp()); + matrix().mesh().reduce + ( + scalingVector, + sumOp() + ); scalar sf = scalingVector.x()/stabilise(scalingVector.y(), VSMALL); if (debug >= 2) diff --git a/src/OpenFOAM/matrices/lduMatrix/solvers/GAMG/GAMGSolverSolve.C b/src/OpenFOAM/matrices/lduMatrix/solvers/GAMG/GAMGSolverSolve.C index 21bab4c46f..ca68062193 100644 --- a/src/OpenFOAM/matrices/lduMatrix/solvers/GAMG/GAMGSolverSolve.C +++ b/src/OpenFOAM/matrices/lduMatrix/solvers/GAMG/GAMGSolverSolve.C @@ -61,7 +61,11 @@ Foam::solverPerformance Foam::GAMGSolver::solve scalarField finestResidual(source - Apsi); // Calculate normalised residual for convergence test - solverPerf.initialResidual() = gSumMag(finestResidual)/normFactor; + solverPerf.initialResidual() = gSumMag + ( + finestResidual, + matrix().mesh().comm() + )/normFactor; solverPerf.finalResidual() = solverPerf.initialResidual(); @@ -100,11 +104,15 @@ Foam::solverPerformance Foam::GAMGSolver::solve finestResidual = source; finestResidual -= Apsi; - solverPerf.finalResidual() = gSumMag(finestResidual)/normFactor; + solverPerf.finalResidual() = gSumMag + ( + finestResidual, + matrix().mesh().comm() + )/normFactor; if (debug >= 2) { - solverPerf.print(Info); + solverPerf.print(Info(matrix().mesh().comm())); } } while ( @@ -481,7 +489,7 @@ void Foam::GAMGSolver::solveCoarsestLevel if (debug >= 2) { - coarseSolverPerf.print(Info); + coarseSolverPerf.print(Info(matrix().mesh().comm())); } } } diff --git a/src/OpenFOAM/matrices/lduMatrix/solvers/GAMG/interfaceFields/processorGAMGInterfaceField/processorGAMGInterfaceField.C b/src/OpenFOAM/matrices/lduMatrix/solvers/GAMG/interfaceFields/processorGAMGInterfaceField/processorGAMGInterfaceField.C index 184630bb49..989cdff127 100644 --- a/src/OpenFOAM/matrices/lduMatrix/solvers/GAMG/interfaceFields/processorGAMGInterfaceField/processorGAMGInterfaceField.C +++ b/src/OpenFOAM/matrices/lduMatrix/solvers/GAMG/interfaceFields/processorGAMGInterfaceField/processorGAMGInterfaceField.C @@ -2,7 +2,7 @@ ========= | \\ / F ield | OpenFOAM: The Open Source CFD Toolbox \\ / O peration | - \\ / A nd | Copyright (C) 2012 OpenFOAM Foundation + \\ / A nd | Copyright (C) 2012-2013 OpenFOAM Foundation \\/ M anipulation | ------------------------------------------------------------------------------- License @@ -92,7 +92,8 @@ void Foam::processorGAMGInterfaceField::initInterfaceMatrixUpdate procInterface_.neighbProcNo(), reinterpret_cast(scalarReceiveBuf_.begin()), scalarReceiveBuf_.byteSize(), - procInterface_.tag() + procInterface_.tag(), + procInterface_.comm() ); outstandingSendRequest_ = UPstream::nRequests(); @@ -102,7 +103,8 @@ void Foam::processorGAMGInterfaceField::initInterfaceMatrixUpdate procInterface_.neighbProcNo(), reinterpret_cast(scalarSendBuf_.begin()), scalarSendBuf_.byteSize(), - procInterface_.tag() + procInterface_.tag(), + procInterface_.comm() ); } else diff --git a/src/OpenFOAM/matrices/lduMatrix/solvers/PBiCG/PBiCG.C b/src/OpenFOAM/matrices/lduMatrix/solvers/PBiCG/PBiCG.C index 8f224f85e7..1a221013e3 100644 --- a/src/OpenFOAM/matrices/lduMatrix/solvers/PBiCG/PBiCG.C +++ b/src/OpenFOAM/matrices/lduMatrix/solvers/PBiCG/PBiCG.C @@ -2,7 +2,7 @@ ========= | \\ / F ield | OpenFOAM: The Open Source CFD Toolbox \\ / O peration | - \\ / A nd | Copyright (C) 2011-2012 OpenFOAM Foundation + \\ / A nd | Copyright (C) 2011-2013 OpenFOAM Foundation \\/ M anipulation | ------------------------------------------------------------------------------- License @@ -114,7 +114,9 @@ Foam::solverPerformance Foam::PBiCG::solve } // --- Calculate normalised residual norm - solverPerf.initialResidual() = gSumMag(rA)/normFactor; + solverPerf.initialResidual() = + gSumMag(rA, matrix().mesh().comm()) + /normFactor; solverPerf.finalResidual() = solverPerf.initialResidual(); // --- Check convergence, solve if not converged @@ -139,7 +141,7 @@ Foam::solverPerformance Foam::PBiCG::solve preconPtr->preconditionT(wT, rT, cmpt); // --- Update search directions: - wArT = gSumProd(wA, rT); + wArT = gSumProd(wA, rT, matrix().mesh().comm()); if (solverPerf.nIterations() == 0) { @@ -165,7 +167,7 @@ Foam::solverPerformance Foam::PBiCG::solve matrix_.Amul(wA, pA, interfaceBouCoeffs_, interfaces_, cmpt); matrix_.Tmul(wT, pT, interfaceIntCoeffs_, interfaces_, cmpt); - scalar wApT = gSumProd(wA, pT); + scalar wApT = gSumProd(wA, pT, matrix().mesh().comm()); // --- Test for singularity if (solverPerf.checkSingularity(mag(wApT)/normFactor)) @@ -185,8 +187,9 @@ Foam::solverPerformance Foam::PBiCG::solve rTPtr[cell] -= alpha*wTPtr[cell]; } - solverPerf.finalResidual() = gSumMag(rA)/normFactor; - + solverPerf.finalResidual() = + gSumMag(rA, matrix().mesh().comm()) + /normFactor; } while ( solverPerf.nIterations()++ < maxIter_ diff --git a/src/OpenFOAM/matrices/lduMatrix/solvers/PCG/PCG.C b/src/OpenFOAM/matrices/lduMatrix/solvers/PCG/PCG.C index b4a06ec8a3..b3d0c3d4f1 100644 --- a/src/OpenFOAM/matrices/lduMatrix/solvers/PCG/PCG.C +++ b/src/OpenFOAM/matrices/lduMatrix/solvers/PCG/PCG.C @@ -2,7 +2,7 @@ ========= | \\ / F ield | OpenFOAM: The Open Source CFD Toolbox \\ / O peration | - \\ / A nd | Copyright (C) 2011-2012 OpenFOAM Foundation + \\ / A nd | Copyright (C) 2011-2013 OpenFOAM Foundation \\/ M anipulation | ------------------------------------------------------------------------------- License @@ -105,7 +105,9 @@ Foam::solverPerformance Foam::PCG::solve } // --- Calculate normalised residual norm - solverPerf.initialResidual() = gSumMag(rA)/normFactor; + solverPerf.initialResidual() = + gSumMag(rA, matrix().mesh().comm()) + /normFactor; solverPerf.finalResidual() = solverPerf.initialResidual(); // --- Check convergence, solve if not converged @@ -129,7 +131,7 @@ Foam::solverPerformance Foam::PCG::solve preconPtr->precondition(wA, rA, cmpt); // --- Update search directions: - wArA = gSumProd(wA, rA); + wArA = gSumProd(wA, rA, matrix().mesh().comm()); if (solverPerf.nIterations() == 0) { @@ -152,7 +154,7 @@ Foam::solverPerformance Foam::PCG::solve // --- Update preconditioned residual matrix_.Amul(wA, pA, interfaceBouCoeffs_, interfaces_, cmpt); - scalar wApA = gSumProd(wA, pA); + scalar wApA = gSumProd(wA, pA, matrix().mesh().comm()); // --- Test for singularity @@ -169,7 +171,9 @@ Foam::solverPerformance Foam::PCG::solve rAPtr[cell] -= alpha*wAPtr[cell]; } - solverPerf.finalResidual() = gSumMag(rA)/normFactor; + solverPerf.finalResidual() = + gSumMag(rA, matrix().mesh().comm()) + /normFactor; } while ( diff --git a/src/OpenFOAM/matrices/lduMatrix/solvers/smoothSolver/smoothSolver.C b/src/OpenFOAM/matrices/lduMatrix/solvers/smoothSolver/smoothSolver.C index 4287eb5699..a83ceaca3e 100644 --- a/src/OpenFOAM/matrices/lduMatrix/solvers/smoothSolver/smoothSolver.C +++ b/src/OpenFOAM/matrices/lduMatrix/solvers/smoothSolver/smoothSolver.C @@ -2,7 +2,7 @@ ========= | \\ / F ield | OpenFOAM: The Open Source CFD Toolbox \\ / O peration | - \\ / A nd | Copyright (C) 2011-2012 OpenFOAM Foundation + \\ / A nd | Copyright (C) 2011-2013 OpenFOAM Foundation \\/ M anipulation | ------------------------------------------------------------------------------- License @@ -122,13 +122,18 @@ Foam::solverPerformance Foam::smoothSolver::solve normFactor = this->normFactor(psi, source, Apsi, temp); // Calculate residual magnitude - solverPerf.initialResidual() = gSumMag(source - Apsi)/normFactor; + solverPerf.initialResidual() = gSumMag + ( + (source - Apsi)(), + matrix().mesh().comm() + )/normFactor; solverPerf.finalResidual() = solverPerf.initialResidual(); } if (lduMatrix::debug >= 2) { - Info<< " Normalisation factor = " << normFactor << endl; + Info(matrix().mesh().comm()) + << " Normalisation factor = " << normFactor << endl; } @@ -166,7 +171,8 @@ Foam::solverPerformance Foam::smoothSolver::solve interfaceBouCoeffs_, interfaces_, cmpt - ) + )(), + matrix().mesh().comm() )/normFactor; } while ( diff --git a/src/OpenFOAM/matrices/lduMatrix/solvers/smoothSolver/smoothSolver.H b/src/OpenFOAM/matrices/lduMatrix/solvers/smoothSolver/smoothSolver.H index 2009768401..3561969e0f 100644 --- a/src/OpenFOAM/matrices/lduMatrix/solvers/smoothSolver/smoothSolver.H +++ b/src/OpenFOAM/matrices/lduMatrix/solvers/smoothSolver/smoothSolver.H @@ -2,7 +2,7 @@ ========= | \\ / F ield | OpenFOAM: The Open Source CFD Toolbox \\ / O peration | - \\ / A nd | Copyright (C) 2011-2012 OpenFOAM Foundation + \\ / A nd | Copyright (C) 2011-2013 OpenFOAM Foundation \\/ M anipulation | ------------------------------------------------------------------------------- License @@ -25,7 +25,7 @@ Class Foam::smoothSolver Description - Iterative solver for symmetric and assymetric matrices which uses a + Iterative solver for symmetric and asymetric matrices which uses a run-time selected smoother e.g. GaussSeidel to converge the solution to the required tolerance. diff --git a/src/OpenFOAM/meshes/lduMesh/lduMesh.H b/src/OpenFOAM/meshes/lduMesh/lduMesh.H index 0c3314452a..1f88a2562b 100644 --- a/src/OpenFOAM/meshes/lduMesh/lduMesh.H +++ b/src/OpenFOAM/meshes/lduMesh/lduMesh.H @@ -2,7 +2,7 @@ ========= | \\ / F ield | OpenFOAM: The Open Source CFD Toolbox \\ / O peration | - \\ / A nd | Copyright (C) 2011 OpenFOAM Foundation + \\ / A nd | Copyright (C) 2011-2013 OpenFOAM Foundation \\/ M anipulation | ------------------------------------------------------------------------------- License @@ -77,6 +77,17 @@ public: //- Return a list of pointers for each patch // with only those pointing to interfaces being set virtual lduInterfacePtrsList interfaces() const = 0; + + //- Return communicator used for parallel communication + virtual int comm() const = 0; + + //- Helper: reduce with current communicator + template + void reduce + ( + T& Value, + const BinaryOp& bop + ) const; }; @@ -86,6 +97,12 @@ public: // * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * // +#ifdef NoRepository +# include "lduMeshTemplates.C" +#endif + +// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * // + #endif // ************************************************************************* // diff --git a/src/OpenFOAM/meshes/lduMesh/lduMeshTemplates.C b/src/OpenFOAM/meshes/lduMesh/lduMeshTemplates.C new file mode 100644 index 0000000000..122ea08d8c --- /dev/null +++ b/src/OpenFOAM/meshes/lduMesh/lduMeshTemplates.C @@ -0,0 +1,41 @@ +/*---------------------------------------------------------------------------*\ + ========= | + \\ / F ield | OpenFOAM: The Open Source CFD Toolbox + \\ / O peration | + \\ / A nd | Copyright (C) 2013 OpenFOAM Foundation + \\/ M anipulation | +------------------------------------------------------------------------------- +License + This file is part of OpenFOAM. + + OpenFOAM is free software: you can redistribute it and/or modify it + under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + OpenFOAM is distributed in the hope that it will be useful, but WITHOUT + ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + for more details. + + You should have received a copy of the GNU General Public License + along with OpenFOAM. If not, see . + +\*---------------------------------------------------------------------------*/ + +#include "lduMesh.H" + +// * * * * * * * * * * * * * * * Member Functions * * * * * * * * * * * * * // + +template +void Foam::lduMesh::reduce +( + T& Value, + const BinaryOp& bop +) const +{ + Foam::reduce(Value, bop, Pstream::msgType(), comm()); +} + + +// ************************************************************************* // diff --git a/src/OpenFOAM/meshes/lduMesh/lduPrimitiveMesh.H b/src/OpenFOAM/meshes/lduMesh/lduPrimitiveMesh.H index c5bb605340..f836135b0b 100644 --- a/src/OpenFOAM/meshes/lduMesh/lduPrimitiveMesh.H +++ b/src/OpenFOAM/meshes/lduMesh/lduPrimitiveMesh.H @@ -2,7 +2,7 @@ ========= | \\ / F ield | OpenFOAM: The Open Source CFD Toolbox \\ / O peration | - \\ / A nd | Copyright (C) 2011 OpenFOAM Foundation + \\ / A nd | Copyright (C) 2011-2013 OpenFOAM Foundation \\/ M anipulation | ------------------------------------------------------------------------------- License @@ -68,6 +68,8 @@ class lduPrimitiveMesh // Note this does not need to be held as a copy because it is invariant const lduSchedule& patchSchedule_; + //- Communicator to use for any parallel communication + const label comm_; // Private Member Functions @@ -90,7 +92,8 @@ public: const labelUList& u, const labelListList& pa, lduInterfacePtrsList interfaces, - const lduSchedule& ps + const lduSchedule& ps, + const label comm ) : lduAddressing(nCells), @@ -98,8 +101,18 @@ public: upperAddr_(u), patchAddr_(pa), interfaces_(interfaces), - patchSchedule_(ps) - {} + patchSchedule_(ps), + comm_(comm) + { + Pout<< "lduPrimitiveMesh :" + << " nCells:" << nCells + << " l:" << lowerAddr_.size() + << " u::" << upperAddr_.size() + << " pa:" << patchAddr_.size() + << " interfaces:" << interfaces_.size() + << " comm:" << comm_ + << endl; + } //- Construct from components and re-use storage as specified. @@ -111,6 +124,7 @@ public: labelListList& pa, lduInterfacePtrsList interfaces, const lduSchedule& ps, + const label comm, bool reUse ) : @@ -119,8 +133,18 @@ public: upperAddr_(u, reUse), patchAddr_(pa, reUse), interfaces_(interfaces, reUse), - patchSchedule_(ps) - {} + patchSchedule_(ps), + comm_(comm) + { + Pout<< "lduPrimitiveMesh :" + << " nCells:" << nCells + << " l:" << lowerAddr_.size() + << " u::" << upperAddr_.size() + << " pa:" << patchAddr_.size() + << " interfaces:" << interfaces_.size() + << " comm:" << comm_ + << endl; + } //- Destructor @@ -145,6 +169,12 @@ public: return interfaces_; } + //- Return communicator used for parallel communication + virtual label comm() const + { + return comm_; + } + //- Return Lower addressing virtual const labelUList& lowerAddr() const { diff --git a/src/OpenFOAM/meshes/polyMesh/polyMesh.C b/src/OpenFOAM/meshes/polyMesh/polyMesh.C index 8201254e0b..23409f946f 100644 --- a/src/OpenFOAM/meshes/polyMesh/polyMesh.C +++ b/src/OpenFOAM/meshes/polyMesh/polyMesh.C @@ -2,7 +2,7 @@ ========= | \\ / F ield | OpenFOAM: The Open Source CFD Toolbox \\ / O peration | - \\ / A nd | Copyright (C) 2011-2012 OpenFOAM Foundation + \\ / A nd | Copyright (C) 2011-2013 OpenFOAM Foundation \\/ M anipulation | ------------------------------------------------------------------------------- License @@ -210,6 +210,7 @@ Foam::polyMesh::polyMesh(const IOobject& io) *this ), bounds_(points_), + comm_(UPstream::worldComm), geometricD_(Vector