zhyang-dev/openfoam
1
0
Fork 0
mirror of https://develop.openfoam.com/Development/openfoam.git synced 2025-11-28 03:28:01 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

ENH: comunicators: have solver with user-defined communicator

Browse Source 
(originating from polyMesh::comm() and processorPolyPatch::comm())
This commit is contained in:
mattijs
2013-02-06 14:26:31 +00:00
parent ea8d290191
commit 33d6ea3e62
37 changed files with 613 additions and 154 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
src/OpenFOAM/db/IOstreams/Pstreams/PstreamReduceOps.H
View File

@ -2,7 +2,7 @@
========= |
\\ / F ield | OpenFOAM: The Open Source CFD Toolbox
\\ / O peration |
\\ / A nd | Copyright (C) 2011-2012 OpenFOAM Foundation
\\ / A nd | Copyright (C) 2011-2013 OpenFOAM Foundation
\\/ M anipulation |
-------------------------------------------------------------------------------
License
@ -48,6 +48,12 @@ void reduce
const label comm
)
{
if (UPstream::warnComm != -1 && comm != UPstream::warnComm)
{
Pout<< "** reducing:" << Value << " with comm:" << comm
<< endl;
error::printStack(Pout);
}
Pstream::gather(comms, Value, bop, tag, comm);
Pstream::scatter(comms, Value, tag, comm);
}

23
src/OpenFOAM/db/IOstreams/Pstreams/UPstream.C
View File

@ -2,7 +2,7 @@
========= |
\\ / F ield | OpenFOAM: The Open Source CFD Toolbox
\\ / O peration |
\\ / A nd | Copyright (C) 2012 OpenFOAM Foundation
\\ / A nd | Copyright (C) 2012-2013 OpenFOAM Foundation
\\/ M anipulation |
-------------------------------------------------------------------------------
License
@ -270,6 +270,19 @@ Foam::label Foam::UPstream::allocateCommunicator
forAll(procIDs_[index], i)
{
procIDs_[index][i] = subRanks[i];
// Enforce incremental order (so index is rank in next communicator)
if (i >= 1 && subRanks[i] <= subRanks[i-1])
{
FatalErrorIn
(
"UPstream::allocateCommunicator"
"(const label, const labelList&, const bool)"
) << "subranks not sorted : " << subRanks
<< " when allocating subcommunicator from parent "
<< parentIndex
<< Foam::abort(FatalError);
}
}
parentCommunicator_[index] = parentIndex;
@ -277,7 +290,7 @@ Foam::label Foam::UPstream::allocateCommunicator
treeCommunication_[index] = calcTreeComm(procIDs_[index].size());
if (doPstream)
if (doPstream && parRun())
{
allocatePstreamCommunicator(parentIndex, index);
}
@ -297,7 +310,7 @@ void Foam::UPstream::freeCommunicator
<< " myProcNo : " << myProcNo_[communicator] << endl
<< endl;
if (doPstream)
if (doPstream && parRun())
{
freePstreamCommunicator(communicator);
}
@ -437,6 +450,10 @@ addcommsTypeToOpt addcommsTypeToOpt_("commsType");
Foam::label Foam::UPstream::worldComm(0);
// Warn for use of any communicator
Foam::label Foam::UPstream::warnComm(-1);
// Number of polling cycles in processor updates
int Foam::UPstream::nPollProcInterfaces
(

47
src/OpenFOAM/db/IOstreams/Pstreams/UPstream.H
View File

@ -2,7 +2,7 @@
========= |
\\ / F ield | OpenFOAM: The Open Source CFD Toolbox
\\ / O peration |
\\ / A nd | Copyright (C) 2012 OpenFOAM Foundation
\\ / A nd | Copyright (C) 2012-2013 OpenFOAM Foundation
\\/ M anipulation |
-------------------------------------------------------------------------------
License
@ -269,6 +269,10 @@ public:
//- Default communicator (all processors)
static label worldComm;
//- Debugging: warn for use of any communicator differing from warnComm
static label warnComm;
// Constructors
//- Construct given optional buffer size
@ -319,17 +323,17 @@ public:
comm_(allocateCommunicator(parent, subRanks, doPstream))
{}
communicator(const label parent)
:
comm_
(
allocateCommunicator
(
parent,
identity(UPstream::nProcs(parent))
)
)
{}
// communicator(const label parent)
// :
// comm_
// (
// allocateCommunicator
// (
// parent,
// identity(UPstream::nProcs(parent))
// )
// )
// {}
~communicator()
{
@ -407,17 +411,24 @@ public:
return myProcNo_[communicator];
}
static label parent(const label communicator)
{
return parentCommunicator_(communicator);
}
// //- Process IDs
// static const List<int>& procIDs()
// {
// return procIDs_;
// }
//
// //- Process ID of given process index
// static int procID(int procNo)
// {
// return procIDs_[procNo];
// }
//- Process ID of given process index
static List<int>& procID(int communicator)
{
return procIDs_[communicator];
}
//- Process index of first slave
static int firstSlave()

70
src/OpenFOAM/db/error/messageStream.C
View File

@ -2,7 +2,7 @@
========= |
\\ / F ield | OpenFOAM: The Open Source CFD Toolbox
\\ / O peration |
\\ / A nd | Copyright (C) 2011 OpenFOAM Foundation
\\ / A nd | Copyright (C) 2011-2013 OpenFOAM Foundation
\\/ M anipulation |
-------------------------------------------------------------------------------
License
@ -166,6 +166,74 @@ Foam::OSstream& Foam::messageStream::operator()
}
Foam::OSstream& Foam::messageStream::operator()(const label communicator)
{
if (UPstream::warnComm != -1 && communicator != UPstream::warnComm)
{
Pout<< "** messageStream with comm:" << communicator
<< endl;
error::printStack(Pout);
}
if (communicator == UPstream::worldComm)
{
return operator()();
}
else
{
bool master = UPstream::master(communicator);
if (level)
{
bool collect = (severity_ == INFO || severity_ == WARNING);
// Report the error
if (!master && collect)
{
return Snull;
}
else
{
if (title().size())
{
if (Pstream::parRun() && !collect)
{
Pout<< title().c_str();
}
else
{
Sout<< title().c_str();
}
}
if (maxErrors_)
{
errorCount_++;
if (errorCount_ >= maxErrors_)
{
FatalErrorIn("messageStream::operator OSstream&()")
<< "Too many errors"
<< abort(FatalError);
}
}
if (Pstream::parRun() && !collect)
{
return Pout;
}
else
{
return Sout;
}
}
}
}
return Snull;
}
Foam::messageStream::operator Foam::OSstream&()
{
if (level)

5
src/OpenFOAM/db/error/messageStream.H
View File

@ -184,6 +184,11 @@ public:
const dictionary&
);
//- Convert to OSstream
// Use Info for default communicator, use Pout
// on master for non-default one.
OSstream& operator()(const label communicator);
//- Convert to OSstream for << operations
operator OSstream&();

32
src/OpenFOAM/fields/Fields/Field/FieldFunctions.C
View File

@ -2,7 +2,7 @@
========= |
\\ / F ield | OpenFOAM: The Open Source CFD Toolbox
\\ / O peration |
\\ / A nd | Copyright (C) 2011-2012 OpenFOAM Foundation
\\ / A nd | Copyright (C) 2011-2013 OpenFOAM Foundation
\\/ M anipulation |
-------------------------------------------------------------------------------
License
@ -477,10 +477,10 @@ TMP_UNARY_FUNCTION(Type, average)
#define G_UNARY_FUNCTION(ReturnType, gFunc, Func, rFunc) \
\
template<class Type> \
ReturnType gFunc(const UList<Type>& f) \
ReturnType gFunc(const UList<Type>& f, const int comm) \
{ \
ReturnType res = Func(f); \
reduce(res, rFunc##Op<Type>()); \
reduce(res, rFunc##Op<Type>(), Pstream::msgType(), comm); \
return res; \
} \
TMP_UNARY_FUNCTION(ReturnType, gFunc)
@ -495,27 +495,41 @@ G_UNARY_FUNCTION(Type, gSumCmptMag, sumCmptMag, sum)
#undef G_UNARY_FUNCTION
template<class Type>
scalar gSumProd(const UList<Type>& f1, const UList<Type>& f2)
scalar gSumProd
(
const UList<Type>& f1,
const UList<Type>& f2,
const int comm
)
{
scalar SumProd = sumProd(f1, f2);
reduce(SumProd, sumOp<scalar>());
reduce(SumProd, sumOp<scalar>(), Pstream::msgType(), comm);
return SumProd;
}
template<class Type>
Type gSumCmptProd(const UList<Type>& f1, const UList<Type>& f2)
Type gSumCmptProd
(
const UList<Type>& f1,
const UList<Type>& f2,
const int comm
)
{
Type SumProd = sumCmptProd(f1, f2);
reduce(SumProd, sumOp<Type>());
reduce(SumProd, sumOp<Type>(), Pstream::msgType(), comm);
return SumProd;
}
template<class Type>
Type gAverage(const UList<Type>& f)
Type gAverage
(
const UList<Type>& f,
const int comm
)
{
label n = f.size();
Type s = sum(f);
sumReduce(s, n);
sumReduce(s, n, Pstream::msgType(), comm);
if (n > 0)
{

25
src/OpenFOAM/fields/Fields/Field/FieldFunctions.H
View File

@ -2,7 +2,7 @@
========= |
\\ / F ield | OpenFOAM: The Open Source CFD Toolbox
\\ / O peration |
\\ / A nd | Copyright (C) 2011-2012 OpenFOAM Foundation
\\ / A nd | Copyright (C) 2011-2013 OpenFOAM Foundation
\\/ M anipulation |
-------------------------------------------------------------------------------
License
@ -25,6 +25,7 @@ License
#define TEMPLATE template<class Type>
#include "FieldFunctionsM.H"
#include "UPstream.H"
// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
@ -201,7 +202,7 @@ TMP_UNARY_FUNCTION(Type, average)
#define G_UNARY_FUNCTION(ReturnType, gFunc, Func, rFunc) \
\
template<class Type> \
ReturnType gFunc(const UList<Type>& f); \
ReturnType gFunc(const UList<Type>& f, const int comm = UPstream::worldComm); \
TMP_UNARY_FUNCTION(ReturnType, gFunc)
G_UNARY_FUNCTION(Type, gMax, max, max)
@ -214,13 +215,27 @@ G_UNARY_FUNCTION(Type, gSumCmptMag, sumCmptMag, sum)
#undef G_UNARY_FUNCTION
template<class Type>
scalar gSumProd(const UList<Type>& f1, const UList<Type>& f2);
scalar gSumProd
(
const UList<Type>& f1,
const UList<Type>& f2,
const int comm = UPstream::worldComm
);
template<class Type>
Type gSumCmptProd(const UList<Type>& f1, const UList<Type>& f2);
Type gSumCmptProd
(
const UList<Type>& f1,
const UList<Type>& f2,
const int comm = UPstream::worldComm
);
template<class Type>
Type gAverage(const UList<Type>& f);
Type gAverage
(
const UList<Type>& f,
const int comm = UPstream::worldComm
);
TMP_UNARY_FUNCTION(Type, gAverage)

6
src/OpenFOAM/matrices/LduMatrix/LduMatrix/SolverPerformance.C
View File

@ -2,7 +2,7 @@
========= |
\\ / F ield | OpenFOAM: The Open Source CFD Toolbox
\\ / O peration |
\\ / A nd | Copyright (C) 2011-2012 OpenFOAM Foundation
\\ / A nd | Copyright (C) 2011-2013 OpenFOAM Foundation
\\/ M anipulation |
-------------------------------------------------------------------------------
License
@ -24,6 +24,7 @@ License
\*---------------------------------------------------------------------------*/
#include "SolverPerformance.H"
#include "IOstreams.H"
// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
@ -64,7 +65,8 @@ bool Foam::SolverPerformance<Type>::checkConvergence
{
if (debug >= 2)
{
Info<< solverName_
//Info<< solverName_
Pout<< solverName_
<< ": Iteration " << noIterations_
<< " residual = " << finalResidual_
<< endl;

4
src/OpenFOAM/matrices/lduMatrix/lduAddressing/lduInterface/processorLduInterface.H
View File

@ -2,7 +2,7 @@
========= |
\\ / F ield | OpenFOAM: The Open Source CFD Toolbox
\\ / O peration |
\\ / A nd | Copyright (C) 2011 OpenFOAM Foundation
\\ / A nd | Copyright (C) 2011-2013 OpenFOAM Foundation
\\/ M anipulation |
-------------------------------------------------------------------------------
License
@ -84,7 +84,7 @@ public:
// Access
//- Return communicator used for sending
//- Return communicator used for parallel communication
virtual int comm() const = 0;
//- Return processor number (rank in communicator)

11
src/OpenFOAM/matrices/lduMatrix/lduMatrix/lduMatrixSolver.C
View File

@ -2,7 +2,7 @@
========= |
\\ / F ield | OpenFOAM: The Open Source CFD Toolbox
\\ / O peration |
\\ / A nd | Copyright (C) 2011-2012 OpenFOAM Foundation
\\ / A nd | Copyright (C) 2011-2013 OpenFOAM Foundation
\\/ M anipulation |
-------------------------------------------------------------------------------
License
@ -186,10 +186,15 @@ Foam::scalar Foam::lduMatrix::solver::normFactor
{
// --- Calculate A dot reference value of psi
matrix_.sumA(tmpField, interfaceBouCoeffs_, interfaces_);
tmpField *= gAverage(psi);
tmpField *= gAverage(psi, matrix_.lduMesh_.comm());
return
gSum(mag(Apsi - tmpField) + mag(source - tmpField))
gSum
(
(mag(Apsi - tmpField) + mag(source - tmpField))(),
matrix_.lduMesh_.comm()
)
+ solverPerformance::small_;
// At convergence this simpler method is equivalent to the above

3
src/OpenFOAM/matrices/lduMatrix/solvers/GAMG/GAMGAgglomerations/GAMGAgglomeration/GAMGAgglomerateLduAddressing.C
View File

@ -2,7 +2,7 @@
========= |
\\ / F ield | OpenFOAM: The Open Source CFD Toolbox
\\ / O peration |
\\ / A nd | Copyright (C) 2011 OpenFOAM Foundation
\\ / A nd | Copyright (C) 2011-2013 OpenFOAM Foundation
\\/ M anipulation |
-------------------------------------------------------------------------------
License
@ -280,6 +280,7 @@ void Foam::GAMGAgglomeration::agglomerateLduAddressing
coarseInterfaceAddr,
coarseInterfaces,
fineMeshAddr.patchSchedule(),
fineMesh.comm(),
true
)
);

4
src/OpenFOAM/matrices/lduMatrix/solvers/GAMG/GAMGAgglomerations/GAMGAgglomeration/GAMGAgglomeration.C
View File

@ -2,7 +2,7 @@
========= |
\\ / F ield | OpenFOAM: The Open Source CFD Toolbox
\\ / O peration |
\\ / A nd | Copyright (C) 2011 OpenFOAM Foundation
\\ / A nd | Copyright (C) 2011-2013 OpenFOAM Foundation
\\/ M anipulation |
-------------------------------------------------------------------------------
License
@ -57,7 +57,7 @@ bool Foam::GAMGAgglomeration::continueAgglomerating
{
// Check the need for further agglomeration on all processors
bool contAgg = nCoarseCells >= nCellsInCoarsestLevel_;
reduce(contAgg, andOp<bool>());
mesh().reduce(contAgg, andOp<bool>());
return contAgg;
}

9
src/OpenFOAM/matrices/lduMatrix/solvers/GAMG/GAMGAgglomerations/GAMGAgglomeration/GAMGAgglomeration.H
View File

@ -2,7 +2,7 @@
========= |
\\ / F ield | OpenFOAM: The Open Source CFD Toolbox
\\ / O peration |
\\ / A nd | Copyright (C) 2011 OpenFOAM Foundation
\\ / A nd | Copyright (C) 2011-2013 OpenFOAM Foundation
\\/ M anipulation |
-------------------------------------------------------------------------------
License
@ -51,6 +51,7 @@ namespace Foam
class lduMesh;
class lduMatrix;
class mapDistribute;
/*---------------------------------------------------------------------------*\
Class GAMGAgglomeration Declaration
@ -77,6 +78,9 @@ protected:
// Maps from the finer to the coarser level.
PtrList<labelField> restrictAddressing_;
//- For cross-processor operation
autoPtr<mapDistribute> restrictDistributeMapPtr_;
//- Face restriction addressing array.
// Maps from the finer to the coarser level.
// Positive indices map the finer faces which form part of the boundary
@ -85,6 +89,9 @@ protected:
// coarser cells to minus the corresponding coarser cell index minus 1.
PtrList<labelList> faceRestrictAddressing_;
//- For cross-processor operation
autoPtr<mapDistribute> faceRestrictDistributeMapPtr_;
//- Hierarchy of mesh addressing
PtrList<lduPrimitiveMesh> meshLevels_;

123
src/OpenFOAM/matrices/lduMatrix/solvers/GAMG/GAMGAgglomerations/GAMGAgglomeration/GAMGAgglomerationTemplates.C
View File

@ -2,7 +2,7 @@
========= |
\\ / F ield | OpenFOAM: The Open Source CFD Toolbox
\\ / O peration |
\\ / A nd | Copyright (C) 2011 OpenFOAM Foundation
\\ / A nd | Copyright (C) 2011-2013 OpenFOAM Foundation
\\/ M anipulation |
-------------------------------------------------------------------------------
License
@ -24,6 +24,7 @@ License
\*---------------------------------------------------------------------------*/
#include "GAMGAgglomeration.H"
#include "mapDistribute.H"
// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
@ -37,24 +38,57 @@ void Foam::GAMGAgglomeration::restrictField
{
const labelList& fineToCoarse = restrictAddressing_[fineLevelIndex];
if (ff.size() != fineToCoarse.size())
if (restrictDistributeMapPtr_.valid())
{
FatalErrorIn
(
"void GAMGAgglomeration::restrictField"
"(Field<Type>& cf, const Field<Type>& ff, "
"const label fineLevelIndex) const"
) << "field does not correspond to level " << fineLevelIndex
<< " sizes: field = " << ff.size()
<< " level = " << fineToCoarse.size()
<< abort(FatalError);
const mapDistribute& map = restrictDistributeMapPtr_();
Field<Type> combinedFf(map.constructSize());
forAll(ff, i)
{
combinedFf[i] = ff[i];
}
map.distribute(combinedFf);
if (combinedFf.size() != fineToCoarse.size())
{
FatalErrorIn
(
"void GAMGAgglomeration::restrictField"
"(Field<Type>& cf, const Field<Type>& ff, "
"const label fineLevelIndex) const"
) << "field does not correspond to level " << fineLevelIndex
<< " sizes: field = " << combinedFf.size()
<< " level = " << fineToCoarse.size()
<< abort(FatalError);
}
cf = pTraits<Type>::zero;
forAll(combinedFf, i)
{
cf[fineToCoarse[i]] += combinedFf[i];
}
}
cf = pTraits<Type>::zero;
forAll(ff, i)
else
{
cf[fineToCoarse[i]] += ff[i];
if (ff.size() != fineToCoarse.size())
{
FatalErrorIn
(
"void GAMGAgglomeration::restrictField"
"(Field<Type>& cf, const Field<Type>& ff, "
"const label fineLevelIndex) const"
) << "field does not correspond to level " << fineLevelIndex
<< " sizes: field = " << ff.size()
<< " level = " << fineToCoarse.size()
<< abort(FatalError);
}
cf = pTraits<Type>::zero;
forAll(ff, i)
{
cf[fineToCoarse[i]] += ff[i];
}
}
}
@ -69,15 +103,40 @@ void Foam::GAMGAgglomeration::restrictFaceField
{
const labelList& fineToCoarse = faceRestrictAddressing_[fineLevelIndex];
cf = pTraits<Type>::zero;
forAll(fineToCoarse, ffacei)
if (faceRestrictDistributeMapPtr_.valid())
{
label cFace = fineToCoarse[ffacei];
if (cFace >= 0)
const mapDistribute& map = faceRestrictDistributeMapPtr_();
Field<Type> combinedFf(map.constructSize());
forAll(ff, i)
{
cf[cFace] += ff[ffacei];
combinedFf[i] = ff[i];
}
map.distribute(combinedFf);
cf = pTraits<Type>::zero;
forAll(fineToCoarse, ffacei)
{
label cFace = fineToCoarse[ffacei];
if (cFace >= 0)
{
cf[cFace] += combinedFf[ffacei];
}
}
}
else
{
cf = pTraits<Type>::zero;
forAll(fineToCoarse, ffacei)
{
label cFace = fineToCoarse[ffacei];
if (cFace >= 0)
{
cf[cFace] += ff[ffacei];
}
}
}
}
@ -93,9 +152,23 @@ void Foam::GAMGAgglomeration::prolongField
{
const labelList& fineToCoarse = restrictAddressing_[coarseLevelIndex];
forAll(fineToCoarse, i)
if (restrictDistributeMapPtr_.valid())
{
ff[i] = cf[fineToCoarse[i]];
const mapDistribute& map = restrictDistributeMapPtr_();
Field<Type> combinedCf(cf);
map.reverseDistribute(fineToCoarse.size(), combinedCf);
forAll(fineToCoarse, i)
{
ff[i] = combinedCf[fineToCoarse[i]];
}
}
else
{
forAll(fineToCoarse, i)
{
ff[i] = cf[fineToCoarse[i]];
}
}
}

4
src/OpenFOAM/matrices/lduMatrix/solvers/GAMG/GAMGSolver.H
View File

@ -43,9 +43,7 @@ Description
SourceFiles
GAMGSolver.C
GAMGSolverCalcAgglomeration.C
GAMGSolverMakeCoarseMatrix.C
GAMGSolverOperations.C
GAMGSolverAgglomerateMatrix.C
GAMGSolverInterpolate.C
GAMGSolverScale.C
GAMGSolverSolve.C

4
src/OpenFOAM/matrices/lduMatrix/solvers/GAMG/GAMGSolverAgglomerateMatrix.C
View File

@ -2,7 +2,7 @@
========= |
\\ / F ield | OpenFOAM: The Open Source CFD Toolbox
\\ / O peration |
\\ / A nd | Copyright (C) 2011 OpenFOAM Foundation
\\ / A nd | Copyright (C) 2011-2013 OpenFOAM Foundation
\\/ M anipulation |
-------------------------------------------------------------------------------
License
@ -126,7 +126,7 @@ void Foam::GAMGSolver::agglomerateMatrix(const label fineLevelIndex)
}
// Check if matrix is assymetric and if so agglomerate both upper and lower
// Check if matrix is asymetric and if so agglomerate both upper and lower
// coefficients ...
if (fineMatrix.hasLower())
{

6
src/OpenFOAM/matrices/lduMatrix/solvers/GAMG/GAMGSolverScale.C
View File

@ -58,7 +58,11 @@ void Foam::GAMGSolver::scale
}
vector2D scalingVector(scalingFactorNum, scalingFactorDenom);
reduce(scalingVector, sumOp<vector2D>());
matrix().mesh().reduce
(
scalingVector,
sumOp<vector2D>()
);
scalar sf = scalingVector.x()/stabilise(scalingVector.y(), VSMALL);
if (debug >= 2)

16
src/OpenFOAM/matrices/lduMatrix/solvers/GAMG/GAMGSolverSolve.C
View File

@ -61,7 +61,11 @@ Foam::solverPerformance Foam::GAMGSolver::solve
scalarField finestResidual(source - Apsi);
// Calculate normalised residual for convergence test
solverPerf.initialResidual() = gSumMag(finestResidual)/normFactor;
solverPerf.initialResidual() = gSumMag
(
finestResidual,
matrix().mesh().comm()
)/normFactor;
solverPerf.finalResidual() = solverPerf.initialResidual();
@ -100,11 +104,15 @@ Foam::solverPerformance Foam::GAMGSolver::solve
finestResidual = source;
finestResidual -= Apsi;
solverPerf.finalResidual() = gSumMag(finestResidual)/normFactor;
solverPerf.finalResidual() = gSumMag
(
finestResidual,
matrix().mesh().comm()
)/normFactor;
if (debug >= 2)
{
solverPerf.print(Info);
solverPerf.print(Info(matrix().mesh().comm()));
}
} while
(
@ -481,7 +489,7 @@ void Foam::GAMGSolver::solveCoarsestLevel
if (debug >= 2)
{
coarseSolverPerf.print(Info);
coarseSolverPerf.print(Info(matrix().mesh().comm()));
}
}
}

8
src/OpenFOAM/matrices/lduMatrix/solvers/GAMG/interfaceFields/processorGAMGInterfaceField/processorGAMGInterfaceField.C
View File

@ -2,7 +2,7 @@
========= |
\\ / F ield | OpenFOAM: The Open Source CFD Toolbox
\\ / O peration |
\\ / A nd | Copyright (C) 2012 OpenFOAM Foundation
\\ / A nd | Copyright (C) 2012-2013 OpenFOAM Foundation
\\/ M anipulation |
-------------------------------------------------------------------------------
License
@ -92,7 +92,8 @@ void Foam::processorGAMGInterfaceField::initInterfaceMatrixUpdate
procInterface_.neighbProcNo(),
reinterpret_cast<char*>(scalarReceiveBuf_.begin()),
scalarReceiveBuf_.byteSize(),
procInterface_.tag()
procInterface_.tag(),
procInterface_.comm()
);
outstandingSendRequest_ = UPstream::nRequests();
@ -102,7 +103,8 @@ void Foam::processorGAMGInterfaceField::initInterfaceMatrixUpdate
procInterface_.neighbProcNo(),
reinterpret_cast<const char*>(scalarSendBuf_.begin()),
scalarSendBuf_.byteSize(),
procInterface_.tag()
procInterface_.tag(),
procInterface_.comm()
);
}
else

15
src/OpenFOAM/matrices/lduMatrix/solvers/PBiCG/PBiCG.C
View File

@ -2,7 +2,7 @@
========= |
\\ / F ield | OpenFOAM: The Open Source CFD Toolbox
\\ / O peration |
\\ / A nd | Copyright (C) 2011-2012 OpenFOAM Foundation
\\ / A nd | Copyright (C) 2011-2013 OpenFOAM Foundation
\\/ M anipulation |
-------------------------------------------------------------------------------
License
@ -114,7 +114,9 @@ Foam::solverPerformance Foam::PBiCG::solve
}
// --- Calculate normalised residual norm
solverPerf.initialResidual() = gSumMag(rA)/normFactor;
solverPerf.initialResidual() =
gSumMag(rA, matrix().mesh().comm())
/normFactor;
solverPerf.finalResidual() = solverPerf.initialResidual();
// --- Check convergence, solve if not converged
@ -139,7 +141,7 @@ Foam::solverPerformance Foam::PBiCG::solve
preconPtr->preconditionT(wT, rT, cmpt);
// --- Update search directions:
wArT = gSumProd(wA, rT);
wArT = gSumProd(wA, rT, matrix().mesh().comm());
if (solverPerf.nIterations() == 0)
{
@ -165,7 +167,7 @@ Foam::solverPerformance Foam::PBiCG::solve
matrix_.Amul(wA, pA, interfaceBouCoeffs_, interfaces_, cmpt);
matrix_.Tmul(wT, pT, interfaceIntCoeffs_, interfaces_, cmpt);
scalar wApT = gSumProd(wA, pT);
scalar wApT = gSumProd(wA, pT, matrix().mesh().comm());
// --- Test for singularity
if (solverPerf.checkSingularity(mag(wApT)/normFactor))
@ -185,8 +187,9 @@ Foam::solverPerformance Foam::PBiCG::solve
rTPtr[cell] -= alpha*wTPtr[cell];
}
solverPerf.finalResidual() = gSumMag(rA)/normFactor;
solverPerf.finalResidual() =
gSumMag(rA, matrix().mesh().comm())
/normFactor;
} while
(
solverPerf.nIterations()++ < maxIter_

14
src/OpenFOAM/matrices/lduMatrix/solvers/PCG/PCG.C
View File

@ -2,7 +2,7 @@
========= |
\\ / F ield | OpenFOAM: The Open Source CFD Toolbox
\\ / O peration |
\\ / A nd | Copyright (C) 2011-2012 OpenFOAM Foundation
\\ / A nd | Copyright (C) 2011-2013 OpenFOAM Foundation
\\/ M anipulation |
-------------------------------------------------------------------------------
License
@ -105,7 +105,9 @@ Foam::solverPerformance Foam::PCG::solve
}
// --- Calculate normalised residual norm
solverPerf.initialResidual() = gSumMag(rA)/normFactor;
solverPerf.initialResidual() =
gSumMag(rA, matrix().mesh().comm())
/normFactor;
solverPerf.finalResidual() = solverPerf.initialResidual();
// --- Check convergence, solve if not converged
@ -129,7 +131,7 @@ Foam::solverPerformance Foam::PCG::solve
preconPtr->precondition(wA, rA, cmpt);
// --- Update search directions:
wArA = gSumProd(wA, rA);
wArA = gSumProd(wA, rA, matrix().mesh().comm());
if (solverPerf.nIterations() == 0)
{
@ -152,7 +154,7 @@ Foam::solverPerformance Foam::PCG::solve
// --- Update preconditioned residual
matrix_.Amul(wA, pA, interfaceBouCoeffs_, interfaces_, cmpt);
scalar wApA = gSumProd(wA, pA);
scalar wApA = gSumProd(wA, pA, matrix().mesh().comm());
// --- Test for singularity
@ -169,7 +171,9 @@ Foam::solverPerformance Foam::PCG::solve
rAPtr[cell] -= alpha*wAPtr[cell];
}
solverPerf.finalResidual() = gSumMag(rA)/normFactor;
solverPerf.finalResidual() =
gSumMag(rA, matrix().mesh().comm())
/normFactor;
} while
(

14
src/OpenFOAM/matrices/lduMatrix/solvers/smoothSolver/smoothSolver.C
View File

@ -2,7 +2,7 @@
========= |
\\ / F ield | OpenFOAM: The Open Source CFD Toolbox
\\ / O peration |
\\ / A nd | Copyright (C) 2011-2012 OpenFOAM Foundation
\\ / A nd | Copyright (C) 2011-2013 OpenFOAM Foundation
\\/ M anipulation |
-------------------------------------------------------------------------------
License
@ -122,13 +122,18 @@ Foam::solverPerformance Foam::smoothSolver::solve
normFactor = this->normFactor(psi, source, Apsi, temp);
// Calculate residual magnitude
solverPerf.initialResidual() = gSumMag(source - Apsi)/normFactor;
solverPerf.initialResidual() = gSumMag
(
(source - Apsi)(),
matrix().mesh().comm()
)/normFactor;
solverPerf.finalResidual() = solverPerf.initialResidual();
}
if (lduMatrix::debug >= 2)
{
Info<< " Normalisation factor = " << normFactor << endl;
Info(matrix().mesh().comm())
<< " Normalisation factor = " << normFactor << endl;
}
@ -166,7 +171,8 @@ Foam::solverPerformance Foam::smoothSolver::solve
interfaceBouCoeffs_,
interfaces_,
cmpt
)
)(),
matrix().mesh().comm()
)/normFactor;
} while
(

4
src/OpenFOAM/matrices/lduMatrix/solvers/smoothSolver/smoothSolver.H
View File

@ -2,7 +2,7 @@
========= |
\\ / F ield | OpenFOAM: The Open Source CFD Toolbox
\\ / O peration |
\\ / A nd | Copyright (C) 2011-2012 OpenFOAM Foundation
\\ / A nd | Copyright (C) 2011-2013 OpenFOAM Foundation
\\/ M anipulation |
-------------------------------------------------------------------------------
License
@ -25,7 +25,7 @@ Class
Foam::smoothSolver
Description
Iterative solver for symmetric and assymetric matrices which uses a
Iterative solver for symmetric and asymetric matrices which uses a
run-time selected smoother e.g. GaussSeidel to converge the solution to
the required tolerance.

19
src/OpenFOAM/meshes/lduMesh/lduMesh.H
View File

@ -2,7 +2,7 @@
========= |
\\ / F ield | OpenFOAM: The Open Source CFD Toolbox
\\ / O peration |
\\ / A nd | Copyright (C) 2011 OpenFOAM Foundation
\\ / A nd | Copyright (C) 2011-2013 OpenFOAM Foundation
\\/ M anipulation |
-------------------------------------------------------------------------------
License
@ -77,6 +77,17 @@ public:
//- Return a list of pointers for each patch
// with only those pointing to interfaces being set
virtual lduInterfacePtrsList interfaces() const = 0;
//- Return communicator used for parallel communication
virtual int comm() const = 0;
//- Helper: reduce with current communicator
template<class T, class BinaryOp>
void reduce
(
T& Value,
const BinaryOp& bop
) const;
};
@ -86,6 +97,12 @@ public:
// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
#ifdef NoRepository
# include "lduMeshTemplates.C"
#endif
// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
#endif
// ************************************************************************* //

41
src/OpenFOAM/meshes/lduMesh/lduMeshTemplates.C Normal file
View File

@ -0,0 +1,41 @@
/*---------------------------------------------------------------------------*\
========= |
\\ / F ield | OpenFOAM: The Open Source CFD Toolbox
\\ / O peration |
\\ / A nd | Copyright (C) 2013 OpenFOAM Foundation
\\/ M anipulation |
-------------------------------------------------------------------------------
License
This file is part of OpenFOAM.
OpenFOAM is free software: you can redistribute it and/or modify it
under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
OpenFOAM is distributed in the hope that it will be useful, but WITHOUT
ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
for more details.
You should have received a copy of the GNU General Public License
along with OpenFOAM. If not, see <http://www.gnu.org/licenses/>.
\*---------------------------------------------------------------------------*/
#include "lduMesh.H"
// * * * * * * * * * * * * * * * Member Functions * * * * * * * * * * * * * //
template<class T, class BinaryOp>
void Foam::lduMesh::reduce
(
T& Value,
const BinaryOp& bop
) const
{
Foam::reduce(Value, bop, Pstream::msgType(), comm());
}
// ************************************************************************* //

42
src/OpenFOAM/meshes/lduMesh/lduPrimitiveMesh.H
View File

@ -2,7 +2,7 @@
========= |
\\ / F ield | OpenFOAM: The Open Source CFD Toolbox
\\ / O peration |
\\ / A nd | Copyright (C) 2011 OpenFOAM Foundation
\\ / A nd | Copyright (C) 2011-2013 OpenFOAM Foundation
\\/ M anipulation |
-------------------------------------------------------------------------------
License
@ -68,6 +68,8 @@ class lduPrimitiveMesh
// Note this does not need to be held as a copy because it is invariant
const lduSchedule& patchSchedule_;
//- Communicator to use for any parallel communication
const label comm_;
// Private Member Functions
@ -90,7 +92,8 @@ public:
const labelUList& u,
const labelListList& pa,
lduInterfacePtrsList interfaces,
const lduSchedule& ps
const lduSchedule& ps,
const label comm
)
:
lduAddressing(nCells),
@ -98,8 +101,18 @@ public:
upperAddr_(u),
patchAddr_(pa),
interfaces_(interfaces),
patchSchedule_(ps)
{}
patchSchedule_(ps),
comm_(comm)
{
Pout<< "lduPrimitiveMesh :"
<< " nCells:" << nCells
<< " l:" << lowerAddr_.size()
<< " u::" << upperAddr_.size()
<< " pa:" << patchAddr_.size()
<< " interfaces:" << interfaces_.size()
<< " comm:" << comm_
<< endl;
}
//- Construct from components and re-use storage as specified.
@ -111,6 +124,7 @@ public:
labelListList& pa,
lduInterfacePtrsList interfaces,
const lduSchedule& ps,
const label comm,
bool reUse
)
:
@ -119,8 +133,18 @@ public:
upperAddr_(u, reUse),
patchAddr_(pa, reUse),
interfaces_(interfaces, reUse),
patchSchedule_(ps)
{}
patchSchedule_(ps),
comm_(comm)
{
Pout<< "lduPrimitiveMesh :"
<< " nCells:" << nCells
<< " l:" << lowerAddr_.size()
<< " u::" << upperAddr_.size()
<< " pa:" << patchAddr_.size()
<< " interfaces:" << interfaces_.size()
<< " comm:" << comm_
<< endl;
}
//- Destructor
@ -145,6 +169,12 @@ public:
return interfaces_;
}
//- Return communicator used for parallel communication
virtual label comm() const
{
return comm_;
}
//- Return Lower addressing
virtual const labelUList& lowerAddr() const
{

17
src/OpenFOAM/meshes/polyMesh/polyMesh.C
View File

@ -2,7 +2,7 @@
========= |
\\ / F ield | OpenFOAM: The Open Source CFD Toolbox
\\ / O peration |
\\ / A nd | Copyright (C) 2011-2012 OpenFOAM Foundation
\\ / A nd | Copyright (C) 2011-2013 OpenFOAM Foundation
\\/ M anipulation |
-------------------------------------------------------------------------------
License
@ -210,6 +210,7 @@ Foam::polyMesh::polyMesh(const IOobject& io)
*this
),
bounds_(points_),
comm_(UPstream::worldComm),
geometricD_(Vector<label>::zero),
solutionD_(Vector<label>::zero),
tetBasePtIsPtr_(NULL),
@ -403,6 +404,7 @@ Foam::polyMesh::polyMesh
polyPatchList()
),
bounds_(points_, syncPar),
comm_(UPstream::worldComm),
geometricD_(Vector<label>::zero),
solutionD_(Vector<label>::zero),
tetBasePtIsPtr_(NULL),
@ -561,6 +563,7 @@ Foam::polyMesh::polyMesh
0
),
bounds_(points_, syncPar),
comm_(UPstream::worldComm),
geometricD_(Vector<label>::zero),
solutionD_(Vector<label>::zero),
tetBasePtIsPtr_(NULL),
@ -1212,6 +1215,18 @@ const Foam::globalMeshData& Foam::polyMesh::globalData() const
}
Foam::label Foam::polyMesh::comm() const
{
return comm_;
}
Foam::label& Foam::polyMesh::comm()
{
return comm_;
}
// Remove all files and some subdirs (eg, sets)
void Foam::polyMesh::removeFiles(const fileName& instanceDir) const
{

11
src/OpenFOAM/meshes/polyMesh/polyMesh.H
View File

@ -2,7 +2,7 @@
========= |
\\ / F ield | OpenFOAM: The Open Source CFD Toolbox
\\ / O peration |
\\ / A nd | Copyright (C) 2011-2012 OpenFOAM Foundation
\\ / A nd | Copyright (C) 2011-2013 OpenFOAM Foundation
\\/ M anipulation |
-------------------------------------------------------------------------------
License
@ -133,6 +133,9 @@ private:
// Created from points on construction, updated when the mesh moves
boundBox bounds_;
//- Communicator used for parallel communication
label comm_;
//- vector of non-constrained directions in mesh
// defined according to the presence of empty and wedge patches
mutable Vector<label> geometricD_;
@ -442,6 +445,12 @@ public:
//- Return parallel info
const globalMeshData& globalData() const;
//- Return communicator used for parallel communication
label comm() const;
//- Return communicator used for parallel communication
label& comm();
//- Return the object registry
const objectRegistry& thisDb() const
{

4
src/OpenFOAM/meshes/polyMesh/polyMeshFromShapeMesh.C
View File

@ -2,7 +2,7 @@
========= |
\\ / F ield | OpenFOAM: The Open Source CFD Toolbox
\\ / O peration |
\\ / A nd | Copyright (C) 2011-2012 OpenFOAM Foundation
\\ / A nd | Copyright (C) 2011-2013 OpenFOAM Foundation
\\/ M anipulation |
-------------------------------------------------------------------------------
License
@ -504,6 +504,7 @@ Foam::polyMesh::polyMesh
boundaryFaces.size() + 1 // add room for a default patch
),
bounds_(points_, syncPar),
comm_(UPstream::worldComm),
geometricD_(Vector<label>::zero),
solutionD_(Vector<label>::zero),
tetBasePtIsPtr_(NULL),
@ -787,6 +788,7 @@ Foam::polyMesh::polyMesh
boundaryFaces.size() + 1 // add room for a default patch
),
bounds_(points_, syncPar),
comm_(UPstream::worldComm),
geometricD_(Vector<label>::zero),
solutionD_(Vector<label>::zero),
tetBasePtIsPtr_(NULL),

26
src/OpenFOAM/meshes/polyMesh/polyPatches/constraint/processor/processorPolyPatch.H
View File

@ -2,7 +2,7 @@
========= |
\\ / F ield | OpenFOAM: The Open Source CFD Toolbox
\\ / O peration |
\\ / A nd | Copyright (C) 2011-2012 OpenFOAM Foundation
\\ / A nd | Copyright (C) 2011-2013 OpenFOAM Foundation
\\/ M anipulation |
-------------------------------------------------------------------------------
License
@ -267,6 +267,30 @@ public:
return neighbProcNo_;
}
// For testing
//- Return communicator used for communication
label& comm()
{
return comm_;
}
//- Return processor number
int& myProcNo()
{
return myProcNo_;
}
//- Return neigbour processor number
int& neighbProcNo()
{
return neighbProcNo_;
}
//- Does the processor own the patch ?
virtual bool owner() const
{

27
src/Pstream/mpi/UIPread.C
View File

@ -2,7 +2,7 @@
========= |
\\ / F ield | OpenFOAM: The Open Source CFD Toolbox
\\ / O peration |
\\ / A nd | Copyright (C) 2011-2012 OpenFOAM Foundation
\\ / A nd | Copyright (C) 2011-2013 OpenFOAM Foundation
\\/ M anipulation |
-------------------------------------------------------------------------------
License
@ -70,7 +70,7 @@ Foam::UIPstream::UIPstream
label wantedSize = externalBuf_.capacity();
if (debug)
if (debug&1)
{
Pout<< "UIPstream::UIPstream : read from:" << fromProcNo
<< " tag:" << tag << " wanted size:" << wantedSize
@ -88,7 +88,7 @@ Foam::UIPstream::UIPstream
externalBuf_.setCapacity(messageSize_);
wantedSize = messageSize_;
if (debug)
if (debug&1)
{
Pout<< "UIPstream::UIPstream : probed size:" << wantedSize
<< Foam::endl;
@ -145,7 +145,7 @@ Foam::UIPstream::UIPstream(const int fromProcNo, PstreamBuffers& buffers)
// Message is already received into externalBuf
messageSize_ = buffers.recvBuf_[fromProcNo].size();
if (debug)
if (debug&1)
{
Pout<< "UIPstream::UIPstream PstreamBuffers :"
<< " fromProcNo:" << fromProcNo
@ -159,7 +159,7 @@ Foam::UIPstream::UIPstream(const int fromProcNo, PstreamBuffers& buffers)
label wantedSize = externalBuf_.capacity();
if (debug)
if (debug&1)
{
Pout<< "UIPstream::UIPstream PstreamBuffers :"
<< " read from:" << fromProcNo
@ -177,7 +177,7 @@ Foam::UIPstream::UIPstream(const int fromProcNo, PstreamBuffers& buffers)
externalBuf_.setCapacity(messageSize_);
wantedSize = messageSize_;
if (debug)
if (debug&1)
{
Pout<< "UIPstream::UIPstream PstreamBuffers : probed size:"
<< wantedSize << Foam::endl;
@ -217,7 +217,7 @@ Foam::label Foam::UIPstream::read
const label communicator
)
{
if (debug)
if (debug&1)
{
Pout<< "UIPstream::read : starting read from:" << fromProcNo
<< " tag:" << tag << " comm:" << communicator
@ -225,6 +225,15 @@ Foam::label Foam::UIPstream::read
<< " commsType:" << UPstream::commsTypeNames[commsType]
<< Foam::endl;
}
if (UPstream::warnComm != -1 && communicator != UPstream::warnComm)
{
Pout<< "UIPstream::read : starting read from:" << fromProcNo
<< " tag:" << tag << " comm:" << communicator
<< " wanted size:" << label(bufSize)
<< " commsType:" << UPstream::commsTypeNames[commsType]
<< Foam::endl;
error::printStack(Pout);
}
if (commsType == blocking || commsType == scheduled)
{
@ -260,7 +269,7 @@ Foam::label Foam::UIPstream::read
int messageSize;
MPI_Get_count(&status, MPI_BYTE, &messageSize);
if (debug)
if (debug&1)
{
Pout<< "UIPstream::read : finished read from:" << fromProcNo
<< " tag:" << tag << " read size:" << label(bufSize)
@ -310,7 +319,7 @@ Foam::label Foam::UIPstream::read
return 0;
}
if (debug)
if (debug&1)
{
Pout<< "UIPstream::read : started read from:" << fromProcNo
<< " tag:" << tag << " read size:" << label(bufSize)

20
src/Pstream/mpi/UOPwrite.C
View File

@ -2,7 +2,7 @@
========= |
\\ / F ield | OpenFOAM: The Open Source CFD Toolbox
\\ / O peration |
\\ / A nd | Copyright (C) 2011 OpenFOAM Foundation
\\ / A nd | Copyright (C) 2011-2013 OpenFOAM Foundation
\\/ M anipulation |
-------------------------------------------------------------------------------
License
@ -43,7 +43,7 @@ bool Foam::UOPstream::write
const label communicator
)
{
if (debug)
if (debug&1)
{
Pout<< "UOPstream::write : starting write to:" << toProcNo
<< " tag:" << tag
@ -51,6 +51,16 @@ bool Foam::UOPstream::write
<< " commsType:" << UPstream::commsTypeNames[commsType]
<< Foam::endl;
}
if (UPstream::warnComm != -1 && communicator != UPstream::warnComm)
{
Pout<< "UOPstream::write : starting write to:" << toProcNo
<< " tag:" << tag
<< " comm:" << communicator << " size:" << label(bufSize)
<< " commsType:" << UPstream::commsTypeNames[commsType]
<< Foam::endl;
error::printStack(Pout);
}
PstreamGlobals::checkCommunicator(communicator, toProcNo);
@ -69,7 +79,7 @@ bool Foam::UOPstream::write
PstreamGlobals::MPICommunicators_[communicator] //MPI_COMM_WORLD
);
if (debug)
if (debug&1)
{
Pout<< "UOPstream::write : finished write to:" << toProcNo
<< " tag:" << tag << " size:" << label(bufSize)
@ -89,7 +99,7 @@ bool Foam::UOPstream::write
PstreamGlobals::MPICommunicators_[communicator] //MPI_COMM_WORLD
);
if (debug)
if (debug&1)
{
Pout<< "UOPstream::write : finished write to:" << toProcNo
<< " tag:" << tag << " size:" << label(bufSize)
@ -112,7 +122,7 @@ bool Foam::UOPstream::write
&request
);
if (debug)
if (debug&1)
{
Pout<< "UOPstream::write : started write to:" << toProcNo
<< " tag:" << tag << " size:" << label(bufSize)

53
src/Pstream/mpi/UPstream.C
View File

@ -68,7 +68,7 @@ bool Foam::UPstream::init(int& argc, char**& argv)
int myRank;
MPI_Comm_rank(MPI_COMM_WORLD, &myRank);
if (debug)
if (debug&1)
{
Pout<< "UPstream::init : initialised with numProcs:" << numprocs
<< " myRank:" << myRank << endl;
@ -107,13 +107,12 @@ bool Foam::UPstream::init(int& argc, char**& argv)
}
# endif
int processorNameLen;
char processorName[MPI_MAX_PROCESSOR_NAME];
MPI_Get_processor_name(processorName, &processorNameLen);
processorName[processorNameLen] = '\0';
Pout<< "Processor name:" << processorName << endl;
//int processorNameLen;
//char processorName[MPI_MAX_PROCESSOR_NAME];
//
//MPI_Get_processor_name(processorName, &processorNameLen);
//processorName[processorNameLen] = '\0';
//Pout<< "Processor name:" << processorName << endl;
return true;
}
@ -121,7 +120,7 @@ bool Foam::UPstream::init(int& argc, char**& argv)
void Foam::UPstream::exit(int errnum)
{
if (debug)
if (debug&1)
{
Pout<< "UPstream::exit." << endl;
}
@ -181,6 +180,12 @@ void Foam::reduce
const label communicator
)
{
if (UPstream::warnComm != -1 && communicator != UPstream::warnComm)
{
Pout<< "** reducing:" << Value << " with comm:" << communicator
<< endl;
error::printStack(Pout);
}
allReduce(Value, 1, MPI_SCALAR, MPI_SUM, bop, tag, communicator);
}
@ -193,6 +198,12 @@ void Foam::reduce
const label communicator
)
{
if (UPstream::warnComm != -1 && communicator != UPstream::warnComm)
{
Pout<< "** reducing:" << Value << " with comm:" << communicator
<< endl;
error::printStack(Pout);
}
allReduce(Value, 1, MPI_SCALAR, MPI_MIN, bop, tag, communicator);
}
@ -205,6 +216,12 @@ void Foam::reduce
const label communicator
)
{
if (UPstream::warnComm != -1 && communicator != UPstream::warnComm)
{
Pout<< "** reducing:" << Value << " with comm:" << communicator
<< endl;
error::printStack(Pout);
}
allReduce(Value, 2, MPI_SCALAR, MPI_SUM, bop, tag, communicator);
}
@ -217,6 +234,12 @@ void Foam::sumReduce
const label communicator
)
{
if (UPstream::warnComm != -1 && communicator != UPstream::warnComm)
{
Pout<< "** reducing:" << Value << " with comm:" << communicator
<< endl;
error::printStack(Pout);
}
vector2D twoScalars(Value, scalar(Count));
reduce(twoScalars, sumOp<vector2D>(), tag, communicator);
@ -407,7 +430,7 @@ void Foam::UPstream::resetRequests(const label i)
void Foam::UPstream::waitRequests(const label start)
{
if (debug)
if (debug&1)
{
Pout<< "UPstream::waitRequests : starting wait for "
<< PstreamGlobals::outstandingRequests_.size()-start
@ -442,7 +465,7 @@ void Foam::UPstream::waitRequests(const label start)
resetRequests(start);
}
if (debug)
if (debug&1)
{
Pout<< "UPstream::waitRequests : finished wait." << endl;
}
@ -451,7 +474,7 @@ void Foam::UPstream::waitRequests(const label start)
void Foam::UPstream::waitRequest(const label i)
{
if (debug)
if (debug&1)
{
Pout<< "UPstream::waitRequest : starting wait for request:" << i
<< endl;
@ -484,7 +507,7 @@ void Foam::UPstream::waitRequest(const label i)
) << "MPI_Wait returned with error" << Foam::endl;
}
if (debug)
if (debug&1)
{
Pout<< "UPstream::waitRequest : finished wait for request:" << i
<< endl;
@ -494,7 +517,7 @@ void Foam::UPstream::waitRequest(const label i)
bool Foam::UPstream::finishedRequest(const label i)
{
if (debug)
if (debug&1)
{
Pout<< "UPstream::waitRequests : checking finishedRequest:" << i
<< endl;
@ -520,7 +543,7 @@ bool Foam::UPstream::finishedRequest(const label i)
MPI_STATUS_IGNORE
);
if (debug)
if (debug&1)
{
Pout<< "UPstream::waitRequests : finished finishedRequest:" << i
<< endl;

4
src/Pstream/mpi/allReduce.H
View File

@ -2,7 +2,7 @@
========= |
\\ / F ield | OpenFOAM: The Open Source CFD Toolbox
\\ / O peration |
\\ / A nd | Copyright (C) 2012 OpenFOAM Foundation
\\ / A nd | Copyright (C) 2012-2013 OpenFOAM Foundation
\\/ M anipulation |
-------------------------------------------------------------------------------
License
@ -53,7 +53,7 @@ void allReduce
MPI_Op op,
const BinaryOp& bop,
const int tag,
const label communicator
const int communicator
);
// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //

28
src/finiteVolume/fvMatrices/fvMatrix/fvMatrix.C
View File

@ -2,7 +2,7 @@
========= |
\\ / F ield | OpenFOAM: The Open Source CFD Toolbox
\\ / O peration |
\\ / A nd | Copyright (C) 2011-2012 OpenFOAM Foundation
\\ / A nd | Copyright (C) 2011-2013 OpenFOAM Foundation
\\/ M anipulation |
-------------------------------------------------------------------------------
License
@ -599,10 +599,28 @@ void Foam::fvMatrix<Type>::relax(const scalar alpha)
}
}
reduce(nNon, sumOp<label>());
reduce(maxNon, maxOp<scalar>());
reduce(sumNon, sumOp<scalar>());
sumNon /= returnReduce(D.size(), sumOp<label>());
reduce(nNon, sumOp<label>(), UPstream::msgType(), psi_.mesh().comm());
reduce
(
maxNon,
maxOp<scalar>(),
UPstream::msgType(),
psi_.mesh().comm()
);
reduce
(
sumNon,
sumOp<scalar>(),
UPstream::msgType(),
psi_.mesh().comm()
);
sumNon /= returnReduce
(
D.size(),
sumOp<label>(),
UPstream::msgType(),
psi_.mesh().comm()
);
InfoIn("fvMatrix<Type>::relax(const scalar alpha)")
<< "Matrix dominance test for " << psi_.name() << nl

12
src/finiteVolume/fvMatrices/fvMatrix/fvMatrixSolve.C
View File

@ -2,7 +2,7 @@
========= |
\\ / F ield | OpenFOAM: The Open Source CFD Toolbox
\\ / O peration |
\\ / A nd | Copyright (C) 2011-2012 OpenFOAM Foundation
\\ / A nd | Copyright (C) 2011-2013 OpenFOAM Foundation
\\/ M anipulation |
-------------------------------------------------------------------------------
License
@ -198,7 +198,10 @@ Foam::solverPerformance Foam::fvMatrix<Type>::solveSegregated
solverControls
)->solve(psiCmpt, sourceCmpt, cmpt);
solverPerf.print(Info);
//solverPerf.print(Info);
Pout<< "** calling solverPerformance::print with comm:" << this->mesh().comm()
<< endl;
solverPerf.print(Info(this->mesh().comm()));
solverPerfVec = max(solverPerfVec, solverPerf);
solverPerfVec.solverName() = solverPerf.solverName();
@ -261,7 +264,10 @@ Foam::solverPerformance Foam::fvMatrix<Type>::solveCoupled
coupledMatrixSolver->solve(psi)
);
solverPerf.print(Info);
//solverPerf.print(Info);
Pout<< "** calling solverPerformance::print with comm:" << this->mesh().comm()
<< endl;
solverPerf.print(Info(this->mesh().comm()));
psi.correctBoundaryConditions();

8
src/finiteVolume/fvMesh/fvMesh.H
View File

@ -2,7 +2,7 @@
========= |
\\ / F ield | OpenFOAM: The Open Source CFD Toolbox
\\ / O peration |
\\ / A nd | Copyright (C) 2012 OpenFOAM Foundation
\\ / A nd | Copyright (C) 2012-2013 OpenFOAM Foundation
\\/ M anipulation |
-------------------------------------------------------------------------------
License
@ -270,6 +270,12 @@ public:
return boundary().interfaces();
}
//- Return communicator used for parallel communication
virtual int comm() const
{
return polyMesh::comm();
}
//- Internal face owner
const labelUList& owner() const
{