From 2516d984b4e0a581e99d0dec6d60e77a3d142fe3 Mon Sep 17 00:00:00 2001 From: mattijs Date: Mon, 16 Jan 2012 15:59:03 +0000 Subject: [PATCH] ENH: fvPatchField: support for polling. Support for zero-copy transfer. --- src/OpenFOAM/db/IOstreams/Pstreams/UPstream.C | 8 ++- src/OpenFOAM/db/IOstreams/Pstreams/UPstream.H | 7 ++- src/OpenFOAM/global/argList/argList.C | 9 ++-- .../lduInterfaceField/lduInterfaceField.H | 29 +++++++++-- src/Pstream/dummy/UPstream.C | 6 ++- src/Pstream/mpi/UPstream.C | 52 +++++++++++++++++-- .../basic/sliced/slicedFvPatchField.C | 13 ++++- .../basic/sliced/slicedFvPatchField.H | 5 +- .../processor/processorFvPatchField.C | 9 +++- .../processor/processorFvPatchField.H | 6 ++- .../fvPatchFields/fvPatchField/fvPatchField.C | 9 +++- .../fvPatchFields/fvPatchField/fvPatchField.H | 5 +- .../fvMesh/fvPatches/fvPatch/fvPatch.H | 6 ++- .../fvPatches/fvPatch/fvPatchTemplates.C | 20 ++++++- 14 files changed, 161 insertions(+), 23 deletions(-) diff --git a/src/OpenFOAM/db/IOstreams/Pstreams/UPstream.C b/src/OpenFOAM/db/IOstreams/Pstreams/UPstream.C index a45d385894..d0a3d87276 100644 --- a/src/OpenFOAM/db/IOstreams/Pstreams/UPstream.C +++ b/src/OpenFOAM/db/IOstreams/Pstreams/UPstream.C @@ -2,7 +2,7 @@ ========= | \\ / F ield | OpenFOAM: The Open Source CFD Toolbox \\ / O peration | - \\ / A nd | Copyright (C) 2011 OpenFOAM Foundation + \\ / A nd | Copyright (C) 2012 OpenFOAM Foundation \\/ M anipulation | ------------------------------------------------------------------------------- License @@ -254,5 +254,11 @@ Foam::UPstream::commsTypes Foam::UPstream::defaultCommsType commsTypeNames.read(debug::optimisationSwitches().lookup("commsType")) ); +// Number of polling cycles in processor updates +int Foam::UPstream::nPollProcInterfaces +( + debug::optimisationSwitch("nPollProcInterfaces", 0) +); + // ************************************************************************* // diff --git a/src/OpenFOAM/db/IOstreams/Pstreams/UPstream.H b/src/OpenFOAM/db/IOstreams/Pstreams/UPstream.H index 6ae1dfb368..1ad4aa99cc 100644 --- a/src/OpenFOAM/db/IOstreams/Pstreams/UPstream.H +++ b/src/OpenFOAM/db/IOstreams/Pstreams/UPstream.H @@ -2,7 +2,7 @@ ========= | \\ / F ield | OpenFOAM: The Open Source CFD Toolbox \\ / O peration | - \\ / A nd | Copyright (C) 2011 OpenFOAM Foundation + \\ / A nd | Copyright (C) 2012 OpenFOAM Foundation \\/ M anipulation | ------------------------------------------------------------------------------- License @@ -242,6 +242,8 @@ public: //- Default commsType static commsTypes defaultCommsType; + //- Number of polling cycles in processor updates + static int nPollProcInterfaces; // Constructors @@ -273,6 +275,9 @@ public: //- Wait until all requests (from start onwards) have finished. static void waitRequests(const label start = 0); + //- Wait until request i has finished. + static void waitRequest(const label i); + //- Non-blocking comms: has request i finished? static bool finishedRequest(const label i); diff --git a/src/OpenFOAM/global/argList/argList.C b/src/OpenFOAM/global/argList/argList.C index 5d8758e2d2..fa2b1a57af 100644 --- a/src/OpenFOAM/global/argList/argList.C +++ b/src/OpenFOAM/global/argList/argList.C @@ -2,7 +2,7 @@ ========= | \\ / F ield | OpenFOAM: The Open Source CFD Toolbox \\ / O peration | - \\ / A nd | Copyright (C) 2011 OpenFOAM Foundation + \\ / A nd | Copyright (C) 2012 OpenFOAM Foundation \\/ M anipulation | ------------------------------------------------------------------------------- License @@ -772,10 +772,11 @@ Foam::argList::argList Info<< "Roots : " << roots << nl; } Info<< "Pstream initialized with:" << nl - << " floatTransfer : " << Pstream::floatTransfer << nl - << " nProcsSimpleSum : " << Pstream::nProcsSimpleSum << nl - << " commsType : " + << " floatTransfer : " << Pstream::floatTransfer << nl + << " nProcsSimpleSum : " << Pstream::nProcsSimpleSum << nl + << " commsType : " << Pstream::commsTypeNames[Pstream::defaultCommsType] + << " polling iterations : " << Pstream::nPollProcInterfaces << endl; } } diff --git a/src/OpenFOAM/matrices/lduMatrix/lduAddressing/lduInterfaceFields/lduInterfaceField/lduInterfaceField.H b/src/OpenFOAM/matrices/lduMatrix/lduAddressing/lduInterfaceFields/lduInterfaceField/lduInterfaceField.H index 040c7f8da8..6da2953826 100644 --- a/src/OpenFOAM/matrices/lduMatrix/lduAddressing/lduInterfaceFields/lduInterfaceField/lduInterfaceField.H +++ b/src/OpenFOAM/matrices/lduMatrix/lduAddressing/lduInterfaceFields/lduInterfaceField/lduInterfaceField.H @@ -2,7 +2,7 @@ ========= | \\ / F ield | OpenFOAM: The Open Source CFD Toolbox \\ / O peration | - \\ / A nd | Copyright (C) 2011 OpenFOAM Foundation + \\ / A nd | Copyright (C) 2012 OpenFOAM Foundation \\/ M anipulation | ------------------------------------------------------------------------------- License @@ -60,7 +60,6 @@ class lduInterfaceField //- Reference to the coupled patch this field is defined for const lduInterface& interface_; - // Private Member Functions //- Disallow default bitwise copy construct @@ -69,6 +68,11 @@ class lduInterfaceField //- Disallow default bitwise assignment void operator=(const lduInterfaceField&); +protected: + + //- Update index used so that updateInterfaceMatrix is called only once + // during the construction of the matrix + bool updatedMatrix_; public: @@ -81,7 +85,8 @@ public: //- Construct given coupled patch lduInterfaceField(const lduInterface& patch) : - interface_(patch) + interface_(patch), + updatedMatrix_(false) {} @@ -120,6 +125,24 @@ public: ) const {} + //- Whether matrix has been updated + bool updatedMatrix() const + { + return updatedMatrix_; + } + + //- Whether matrix has been updated + bool& updatedMatrix() + { + return updatedMatrix_; + } + + //- Is all data available + virtual bool ready() const + { + return true; + } + //- Update result field based on interface functionality virtual void updateInterfaceMatrix ( diff --git a/src/Pstream/dummy/UPstream.C b/src/Pstream/dummy/UPstream.C index f208ac880e..5cd8bbcbcb 100644 --- a/src/Pstream/dummy/UPstream.C +++ b/src/Pstream/dummy/UPstream.C @@ -2,7 +2,7 @@ ========= | \\ / F ield | OpenFOAM: The Open Source CFD Toolbox \\ / O peration | - \\ / A nd | Copyright (C) 2011 OpenFOAM Foundation + \\ / A nd | Copyright (C) 2012 OpenFOAM Foundation \\/ M anipulation | ------------------------------------------------------------------------------- License @@ -74,6 +74,10 @@ void Foam::UPstream::waitRequests(const label start) {} +void Foam::UPstream::waitRequest(const label i) +{} + + bool Foam::UPstream::finishedRequest(const label i) { notImplemented("UPstream::finishedRequest()"); diff --git a/src/Pstream/mpi/UPstream.C b/src/Pstream/mpi/UPstream.C index c85f341d8f..1605eca8ce 100644 --- a/src/Pstream/mpi/UPstream.C +++ b/src/Pstream/mpi/UPstream.C @@ -2,7 +2,7 @@ ========= | \\ / F ield | OpenFOAM: The Open Source CFD Toolbox \\ / O peration | - \\ / A nd | Copyright (C) 2011 OpenFOAM Foundation + \\ / A nd | Copyright (C) 2012 OpenFOAM Foundation \\/ M anipulation | ------------------------------------------------------------------------------- License @@ -516,11 +516,55 @@ void Foam::UPstream::waitRequests(const label start) } +void Foam::UPstream::waitRequest(const label i) +{ + if (debug) + { + Pout<< "UPstream::waitRequest : starting wait for request:" << i + << endl; + } + + if (i >= PstreamGlobals::outstandingRequests_.size()) + { + FatalErrorIn + ( + "UPstream::waitRequest(const label)" + ) << "There are " << PstreamGlobals::outstandingRequests_.size() + << " outstanding send requests and you are asking for i=" << i + << nl + << "Maybe you are mixing blocking/non-blocking comms?" + << Foam::abort(FatalError); + } + + int flag; + if + ( + MPI_Wait + ( + &PstreamGlobals::outstandingRequests_[i], + MPI_STATUS_IGNORE + ) + ) + { + FatalErrorIn + ( + "UPstream::waitRequest()" + ) << "MPI_Wait returned with error" << Foam::endl; + } + + if (debug) + { + Pout<< "UPstream::waitRequest : finished wait for request:" << i + << endl; + } +} + + bool Foam::UPstream::finishedRequest(const label i) { if (debug) { - Pout<< "UPstream::waitRequests : starting wait for request:" << i + Pout<< "UPstream::waitRequests : checking finishedRequest:" << i << endl; } @@ -546,7 +590,7 @@ bool Foam::UPstream::finishedRequest(const label i) if (debug) { - Pout<< "UPstream::waitRequests : finished wait for request:" << i + Pout<< "UPstream::waitRequests : finished finishedRequest:" << i << endl; } @@ -554,6 +598,4 @@ bool Foam::UPstream::finishedRequest(const label i) } -// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * // - // ************************************************************************* // diff --git a/src/finiteVolume/fields/fvPatchFields/basic/sliced/slicedFvPatchField.C b/src/finiteVolume/fields/fvPatchFields/basic/sliced/slicedFvPatchField.C index acf50983ef..f530fd1ac9 100644 --- a/src/finiteVolume/fields/fvPatchFields/basic/sliced/slicedFvPatchField.C +++ b/src/finiteVolume/fields/fvPatchFields/basic/sliced/slicedFvPatchField.C @@ -2,7 +2,7 @@ ========= | \\ / F ield | OpenFOAM: The Open Source CFD Toolbox \\ / O peration | - \\ / A nd | Copyright (C) 2011 OpenFOAM Foundation + \\ / A nd | Copyright (C) 2012 OpenFOAM Foundation \\/ M anipulation | ------------------------------------------------------------------------------- License @@ -198,6 +198,17 @@ tmp > slicedFvPatchField::patchInternalField() const } +template +void slicedFvPatchField::patchInternalField(Field&) const +{ + notImplemented + ( + "slicedFvPatchField::" + "patchInternalField(Field&)" + ); +} + + template tmp > slicedFvPatchField::patchNeighbourField ( diff --git a/src/finiteVolume/fields/fvPatchFields/basic/sliced/slicedFvPatchField.H b/src/finiteVolume/fields/fvPatchFields/basic/sliced/slicedFvPatchField.H index 9784c51323..a40d212cd7 100644 --- a/src/finiteVolume/fields/fvPatchFields/basic/sliced/slicedFvPatchField.H +++ b/src/finiteVolume/fields/fvPatchFields/basic/sliced/slicedFvPatchField.H @@ -2,7 +2,7 @@ ========= | \\ / F ield | OpenFOAM: The Open Source CFD Toolbox \\ / O peration | - \\ / A nd | Copyright (C) 2011 OpenFOAM Foundation + \\ / A nd | Copyright (C) 2012 OpenFOAM Foundation \\/ M anipulation | ------------------------------------------------------------------------------- License @@ -148,6 +148,9 @@ public: //- Return internal field next to patch as patch field virtual tmp > patchInternalField() const; + //- Return internal field next to patch as patch field + virtual void patchInternalField(Field&) const; + //- Return neighbour coupled given internal cell data virtual tmp > patchNeighbourField ( diff --git a/src/finiteVolume/fields/fvPatchFields/constraint/processor/processorFvPatchField.C b/src/finiteVolume/fields/fvPatchFields/constraint/processor/processorFvPatchField.C index 4bf8e67cd7..70bba49de9 100644 --- a/src/finiteVolume/fields/fvPatchFields/constraint/processor/processorFvPatchField.C +++ b/src/finiteVolume/fields/fvPatchFields/constraint/processor/processorFvPatchField.C @@ -2,7 +2,7 @@ ========= | \\ / F ield | OpenFOAM: The Open Source CFD Toolbox \\ / O peration | - \\ / A nd | Copyright (C) 2011 OpenFOAM Foundation + \\ / A nd | Copyright (C) 2012 OpenFOAM Foundation \\/ M anipulation | ------------------------------------------------------------------------------- License @@ -223,6 +223,13 @@ void processorFvPatchField::initInterfaceMatrixUpdate } +template +bool processorFvPatchField::ready() const +{ + return true; +} + + template void processorFvPatchField::updateInterfaceMatrix ( diff --git a/src/finiteVolume/fields/fvPatchFields/constraint/processor/processorFvPatchField.H b/src/finiteVolume/fields/fvPatchFields/constraint/processor/processorFvPatchField.H index 0130909d04..1ed07e4238 100644 --- a/src/finiteVolume/fields/fvPatchFields/constraint/processor/processorFvPatchField.H +++ b/src/finiteVolume/fields/fvPatchFields/constraint/processor/processorFvPatchField.H @@ -2,7 +2,7 @@ ========= | \\ / F ield | OpenFOAM: The Open Source CFD Toolbox \\ / O peration | - \\ / A nd | Copyright (C) 2011 OpenFOAM Foundation + \\ / A nd | Copyright (C) 2012 OpenFOAM Foundation \\/ M anipulation | ------------------------------------------------------------------------------- License @@ -179,6 +179,9 @@ public: const Pstream::commsTypes commsType ) const; + //- Is all data available + virtual bool ready() const; + //- Update result field based on interface functionality virtual void updateInterfaceMatrix ( @@ -221,6 +224,7 @@ public: { return pTraits::rank; } + }; diff --git a/src/finiteVolume/fields/fvPatchFields/fvPatchField/fvPatchField.C b/src/finiteVolume/fields/fvPatchFields/fvPatchField/fvPatchField.C index 63038d3dd3..9266ab2749 100644 --- a/src/finiteVolume/fields/fvPatchFields/fvPatchField/fvPatchField.C +++ b/src/finiteVolume/fields/fvPatchFields/fvPatchField/fvPatchField.C @@ -2,7 +2,7 @@ ========= | \\ / F ield | OpenFOAM: The Open Source CFD Toolbox \\ / O peration | - \\ / A nd | Copyright (C) 2011 OpenFOAM Foundation + \\ / A nd | Copyright (C) 2012 OpenFOAM Foundation \\/ M anipulation | ------------------------------------------------------------------------------- License @@ -190,6 +190,13 @@ Foam::fvPatchField::patchInternalField() const } +template +void Foam::fvPatchField::patchInternalField(Field& pif) const +{ + patch_.patchInternalField(internalField_, pif); +} + + template void Foam::fvPatchField::autoMap ( diff --git a/src/finiteVolume/fields/fvPatchFields/fvPatchField/fvPatchField.H b/src/finiteVolume/fields/fvPatchFields/fvPatchField/fvPatchField.H index 4d9d2e0f88..369246d3b0 100644 --- a/src/finiteVolume/fields/fvPatchFields/fvPatchField/fvPatchField.H +++ b/src/finiteVolume/fields/fvPatchFields/fvPatchField/fvPatchField.H @@ -2,7 +2,7 @@ ========= | \\ / F ield | OpenFOAM: The Open Source CFD Toolbox \\ / O peration | - \\ / A nd | Copyright (C) 2011 OpenFOAM Foundation + \\ / A nd | Copyright (C) 2012 OpenFOAM Foundation \\/ M anipulation | ------------------------------------------------------------------------------- License @@ -352,6 +352,9 @@ public: //- Return internal field next to patch as patch field virtual tmp > patchInternalField() const; + //- Return internal field next to patch as patch field + virtual void patchInternalField(Field&) const; + //- Return patchField on the opposite patch of a coupled patch virtual tmp > patchNeighbourField() const { diff --git a/src/finiteVolume/fvMesh/fvPatches/fvPatch/fvPatch.H b/src/finiteVolume/fvMesh/fvPatches/fvPatch/fvPatch.H index be7ee1bda4..28a07343e5 100644 --- a/src/finiteVolume/fvMesh/fvPatches/fvPatch/fvPatch.H +++ b/src/finiteVolume/fvMesh/fvPatches/fvPatch/fvPatch.H @@ -2,7 +2,7 @@ ========= | \\ / F ield | OpenFOAM: The Open Source CFD Toolbox \\ / O peration | - \\ / A nd | Copyright (C) 2011 OpenFOAM Foundation + \\ / A nd | Copyright (C) 2012 OpenFOAM Foundation \\/ M anipulation | ------------------------------------------------------------------------------- License @@ -239,6 +239,10 @@ public: template tmp > patchInternalField(const UList&) const; + //- Return given internal field next to patch as patch field + template + void patchInternalField(const UList&, Field&) const; + //- Return the corresponding patchField of the named field template const typename GeometricField::PatchFieldType& patchField diff --git a/src/finiteVolume/fvMesh/fvPatches/fvPatch/fvPatchTemplates.C b/src/finiteVolume/fvMesh/fvPatches/fvPatch/fvPatchTemplates.C index 5d47411fa0..11aef36b05 100644 --- a/src/finiteVolume/fvMesh/fvPatches/fvPatch/fvPatchTemplates.C +++ b/src/finiteVolume/fvMesh/fvPatches/fvPatch/fvPatchTemplates.C @@ -2,7 +2,7 @@ ========= | \\ / F ield | OpenFOAM: The Open Source CFD Toolbox \\ / O peration | - \\ / A nd | Copyright (C) 2011 OpenFOAM Foundation + \\ / A nd | Copyright (C) 2012 OpenFOAM Foundation \\/ M anipulation | ------------------------------------------------------------------------------- License @@ -47,6 +47,24 @@ Foam::tmp > Foam::fvPatch::patchInternalField } +template +void Foam::fvPatch::patchInternalField +( + const UList& f, + Field& pif +) const +{ + pif.setSize(size()); + + const labelUList& faceCells = this->faceCells(); + + forAll(pif, facei) + { + pif[facei] = f[faceCells[facei]]; + } +} + + template const typename GeometricField::PatchFieldType& Foam::fvPatch::patchField (