zhyang-dev/openfoam
1
0
Fork 0
mirror of https://develop.openfoam.com/Development/openfoam.git synced 2025-11-28 03:28:01 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

ENH: direct, non-blocking transfer for finiteArea processor patch

Browse Source 
STYLE: pTraits::rank instead of std::is_arithmetic to suppress transform

- more consistent with doTransform() coding, potentially useful for
  complex
This commit is contained in:
Mark Olesen
2023-01-12 20:21:07 +01:00
parent b15638a2d2
commit 2641dc4c3a
3 changed files with 261 additions and 37 deletions
Download Patch File Download Diff File Expand all files Collapse all files

270
src/finiteArea/fields/faPatchFields/constraint/processor/processorFaPatchField.C
View File

@ -6,7 +6,7 @@
\\/ M anipulation |
-------------------------------------------------------------------------------
Copyright (C) 2016-2017 Wikki Ltd
Copyright (C) 2019 OpenCFD Ltd.
Copyright (C) 2019-2023 OpenCFD Ltd.
-------------------------------------------------------------------------------
License
This file is part of OpenFOAM.
@ -27,7 +27,6 @@ License
\*---------------------------------------------------------------------------*/
#include "processorFaPatchField.H"
#include "processorFaPatch.H"
#include "transformField.H"
// * * * * * * * * * * * * * * * * Constructors * * * * * * * * * * * * * * //
@ -40,7 +39,9 @@ Foam::processorFaPatchField<Type>::processorFaPatchField
)
:
coupledFaPatchField<Type>(p, iF),
procPatch_(refCast<const processorFaPatch>(p))
procPatch_(refCast<const processorFaPatch>(p)),
sendRequest_(-1),
recvRequest_(-1)
{}
@ -53,7 +54,9 @@ Foam::processorFaPatchField<Type>::processorFaPatchField
)
:
coupledFaPatchField<Type>(p, iF, f),
procPatch_(refCast<const processorFaPatch>(p))
procPatch_(refCast<const processorFaPatch>(p)),
sendRequest_(-1),
recvRequest_(-1)
{}
@ -67,7 +70,9 @@ Foam::processorFaPatchField<Type>::processorFaPatchField
)
:
coupledFaPatchField<Type>(ptf, p, iF, mapper),
procPatch_(refCast<const processorFaPatch>(p))
procPatch_(refCast<const processorFaPatch>(p)),
sendRequest_(-1),
recvRequest_(-1)
{
if (!isType<processorFaPatch>(this->patch()))
{
@ -79,6 +84,12 @@ Foam::processorFaPatchField<Type>::processorFaPatchField
<< " in file " << this->internalField().objectPath()
<< exit(FatalError);
}
if (debug && !ptf.ready())
{
FatalErrorInFunction
<< "On patch " << procPatch_.name() << " outstanding request."
<< abort(FatalError);
}
}
@ -91,7 +102,9 @@ Foam::processorFaPatchField<Type>::processorFaPatchField
)
:
coupledFaPatchField<Type>(p, iF, dict),
procPatch_(refCast<const processorFaPatch>(p, dict))
procPatch_(refCast<const processorFaPatch>(p, dict)),
sendRequest_(-1),
recvRequest_(-1)
{
if (!isType<processorFaPatch>(p))
{
@ -114,8 +127,21 @@ Foam::processorFaPatchField<Type>::processorFaPatchField
:
processorLduInterfaceField(),
coupledFaPatchField<Type>(ptf),
procPatch_(refCast<const processorFaPatch>(ptf.patch()))
{}
procPatch_(refCast<const processorFaPatch>(ptf.patch())),
sendRequest_(-1),
recvRequest_(-1),
sendBuf_(std::move(ptf.sendBuf_)),
recvBuf_(std::move(ptf.recvBuf_)),
scalarSendBuf_(std::move(ptf.scalarSendBuf_)),
scalarRecvBuf_(std::move(ptf.scalarRecvBuf_))
{
if (debug && !ptf.ready())
{
FatalErrorInFunction
<< "On patch " << procPatch_.name() << " outstanding request."
<< abort(FatalError);
}
}
template<class Type>
@ -126,16 +152,45 @@ Foam::processorFaPatchField<Type>::processorFaPatchField
)
:
coupledFaPatchField<Type>(ptf, iF),
procPatch_(refCast<const processorFaPatch>(ptf.patch()))
{}
procPatch_(refCast<const processorFaPatch>(ptf.patch())),
sendRequest_(-1),
recvRequest_(-1)
{
if (debug && !ptf.ready())
{
FatalErrorInFunction
<< "On patch " << procPatch_.name() << " outstanding request."
<< abort(FatalError);
}
}
// * * * * * * * * * * * * * * * Member Functions * * * * * * * * * * * * * //
template<class Type>
bool Foam::processorFaPatchField<Type>::ready() const
{
if (!UPstream::finishedRequest(sendRequest_)) return false;
sendRequest_ = -1;
if (!UPstream::finishedRequest(recvRequest_)) return false;
recvRequest_ = -1;
return true;
}
template<class Type>
Foam::tmp<Foam::Field<Type>>
Foam::processorFaPatchField<Type>::patchNeighbourField() const
{
if (debug && !this->ready())
{
FatalErrorInFunction
<< "On patch " << procPatch_.name()
<< " outstanding request."
<< abort(FatalError);
}
return *this;
}
@ -148,7 +203,46 @@ void Foam::processorFaPatchField<Type>::initEvaluate
{
if (Pstream::parRun())
{
procPatch_.send(commsType, this->patchInternalField()());
this->patchInternalField(sendBuf_);
if (commsType == UPstream::commsTypes::nonBlocking)
{
if (!is_contiguous<Type>::value)
{
FatalErrorInFunction
<< "Invalid for non-contiguous data types"
<< abort(FatalError);
}
// Receive straight into *this
this->resize_nocopy(sendBuf_.size());
recvRequest_ = UPstream::nRequests();
UIPstream::read
(
UPstream::commsTypes::nonBlocking,
procPatch_.neighbProcNo(),
this->data_bytes(),
this->size_bytes(),
procPatch_.tag(),
procPatch_.comm()
);
sendRequest_ = UPstream::nRequests();
UOPstream::write
(
UPstream::commsTypes::nonBlocking,
procPatch_.neighbProcNo(),
sendBuf_.cdata_bytes(),
sendBuf_.size_bytes(),
procPatch_.tag(),
procPatch_.comm()
);
}
else
{
procPatch_.send(commsType, sendBuf_);
}
}
}
@ -161,7 +255,19 @@ void Foam::processorFaPatchField<Type>::evaluate
{
if (Pstream::parRun())
{
procPatch_.receive<Type>(commsType, *this);
if (commsType == UPstream::commsTypes::nonBlocking)
{
// Fast path. Received into *this
// Treat send as finished when recv is done
UPstream::waitRequest(recvRequest_);
recvRequest_ = -1;
sendRequest_ = -1;
}
else
{
procPatch_.receive<Type>(commsType, *this);
}
if (doTransform())
{
@ -191,11 +297,47 @@ void Foam::processorFaPatchField<Type>::initInterfaceMatrixUpdate
const Pstream::commsTypes commsType
) const
{
procPatch_.send
(
commsType,
this->patch().patchInternalField(psiInternal)()
);
this->patch().patchInternalField(psiInternal, scalarSendBuf_);
if (commsType == UPstream::commsTypes::nonBlocking)
{
// Fast path.
if (debug && !this->ready())
{
FatalErrorInFunction
<< "On patch " << procPatch_.name()
<< " outstanding request."
<< abort(FatalError);
}
scalarRecvBuf_.resize_nocopy(scalarSendBuf_.size());
recvRequest_ = UPstream::nRequests();
UIPstream::read
(
UPstream::commsTypes::nonBlocking,
procPatch_.neighbProcNo(),
scalarRecvBuf_.data_bytes(),
scalarRecvBuf_.size_bytes(),
procPatch_.tag(),
procPatch_.comm()
);
sendRequest_ = UPstream::nRequests();
UOPstream::write
(
UPstream::commsTypes::nonBlocking,
procPatch_.neighbProcNo(),
scalarSendBuf_.cdata_bytes(),
scalarSendBuf_.size_bytes(),
procPatch_.tag(),
procPatch_.comm()
);
}
else
{
procPatch_.send(commsType, scalarSendBuf_);
}
this->updatedMatrix(false);
}
@ -221,19 +363,30 @@ void Foam::processorFaPatchField<Type>::updateInterfaceMatrix
const labelUList& faceCells = this->patch().edgeFaces();
solveScalarField pnf
(
procPatch_.receive<solveScalar>(commsType, this->size())
);
if (commsType == UPstream::commsTypes::nonBlocking)
{
// Fast path: consume straight from receive buffer
if (!std::is_arithmetic<Type>::value)
// Treat send as finished when recv is done
UPstream::waitRequest(recvRequest_);
recvRequest_ = -1;
sendRequest_ = -1;
}
else
{
scalarRecvBuf_.resize_nocopy(this->size());
procPatch_.receive(commsType, scalarRecvBuf_);
}
if (pTraits<Type>::rank)
{
// Transform non-scalar data according to the transformation tensor
transformCoupleField(pnf, cmpt);
transformCoupleField(scalarRecvBuf_, cmpt);
}
// Multiply the field by coefficients and add into the result
this->addToInternalField(result, !add, faceCells, coeffs, pnf);
this->addToInternalField(result, !add, faceCells, coeffs, scalarRecvBuf_);
this->updatedMatrix(true);
}
@ -251,11 +404,47 @@ void Foam::processorFaPatchField<Type>::initInterfaceMatrixUpdate
const Pstream::commsTypes commsType
) const
{
procPatch_.send
(
commsType,
this->patch().patchInternalField(psiInternal)()
);
this->patch().patchInternalField(psiInternal, sendBuf_);
if (commsType == UPstream::commsTypes::nonBlocking)
{
// Fast path.
if (debug && !this->ready())
{
FatalErrorInFunction
<< "On patch " << procPatch_.name()
<< " outstanding request."
<< abort(FatalError);
}
recvBuf_.resize_nocopy(sendBuf_.size());
recvRequest_ = UPstream::nRequests();
UIPstream::read
(
UPstream::commsTypes::nonBlocking,
procPatch_.neighbProcNo(),
recvBuf_.data_bytes(),
recvBuf_.size_bytes(),
procPatch_.tag(),
procPatch_.comm()
);
sendRequest_ = UPstream::nRequests();
UOPstream::write
(
UPstream::commsTypes::nonBlocking,
procPatch_.neighbProcNo(),
sendBuf_.cdata_bytes(),
sendBuf_.size_bytes(),
procPatch_.tag(),
procPatch_.comm()
);
}
else
{
procPatch_.send(commsType, sendBuf_);
}
this->updatedMatrix(false);
}
@ -280,16 +469,27 @@ void Foam::processorFaPatchField<Type>::updateInterfaceMatrix
const labelUList& faceCells = this->patch().edgeFaces();
Field<Type> pnf
(
procPatch_.receive<Type>(commsType, this->size())
);
if (commsType == UPstream::commsTypes::nonBlocking)
{
// Fast path: consume straight from receive buffer
// Treat send as finished when recv is done
UPstream::waitRequest(recvRequest_);
recvRequest_ = -1;
sendRequest_ = -1;
}
else
{
recvBuf_.resize_nocopy(this->size());
procPatch_.receive(commsType, recvBuf_);
}
// Transform according to the transformation tensor
transformCoupleField(pnf);
transformCoupleField(recvBuf_);
// Multiply the field by coefficients and add into the result
this->addToInternalField(result, !add, faceCells, coeffs, pnf);
this->addToInternalField(result, !add, faceCells, coeffs, recvBuf_);
this->updatedMatrix(true);
}

26
src/finiteArea/fields/faPatchFields/constraint/processor/processorFaPatchField.H
View File

@ -61,12 +61,33 @@ class processorFaPatchField
public processorLduInterfaceField,
public coupledFaPatchField<Type>
{
// Private data
// Private Data
//- Local reference cast into the processor patch
const processorFaPatch& procPatch_;
// Sending and receiving
//- Current (non-blocking) send request
mutable label sendRequest_;
//- Current (non-blocking) recv request
mutable label recvRequest_;
//- Send buffer.
mutable Field<Type> sendBuf_;
//- Receive buffer.
mutable Field<Type> recvBuf_;
//- Scalar send buffer
mutable solveScalarField scalarSendBuf_;
//- Scalar recv buffer
mutable solveScalarField scalarRecvBuf_;
public:
//- Runtime type information
@ -168,6 +189,9 @@ public:
//- Return patch-normal gradient
virtual tmp<Field<Type>> snGrad() const;
//- Is all data available
virtual bool ready() const;
// Coupled interface functionality

2
src/finiteVolume/fields/fvPatchFields/constraint/processor/processorFvPatchField.C
View File

@ -414,7 +414,7 @@ void Foam::processorFvPatchField<Type>::updateInterfaceMatrix
}
if (!std::is_arithmetic<Type>::value)
if (pTraits<Type>::rank)
{
// Transform non-scalar data according to the transformation tensor
transformCoupleField(scalarRecvBuf_, cmpt);