ENH: add non-blocking handling for cyclicAMI (#2963)

Co-authored-by: Mark Olesen <>

src/finiteVolume/fields/fvPatchFields/constraint/cyclicAMI/cyclicAMIFvPatchField.C

@@ -26,9 +26,11 @@ License
 
 \*---------------------------------------------------------------------------*/
 
+#include "cyclicAMIPolyPatch.H"
+#include "mapDistributeBase.H"
+#include "AMIInterpolation.H"
 #include "fvMatrix.H"
 #include "volFields.H"
-//#include "cylicFvPatchField.H"
 
 // * * * * * * * * * * * * * * * * Constructors  * * * * * * * * * * * * * * //
 
@@ -41,7 +43,10 @@ Foam::cyclicAMIFvPatchField<Type>::cyclicAMIFvPatchField
 :
     cyclicAMILduInterfaceField(),
     coupledFvPatchField<Type>(p, iF),
-    cyclicAMIPatch_(refCast<const cyclicAMIFvPatch>(p))
+    cyclicAMIPatch_(refCast<const cyclicAMIFvPatch>(p)),
+    sendRequests_(0),
+    recvRequests_(0),
+    patchNeighbourFieldPtr_(nullptr)
 {}
 
 
@@ -55,12 +60,15 @@ Foam::cyclicAMIFvPatchField<Type>::cyclicAMIFvPatchField
 :
     cyclicAMILduInterfaceField(),
     coupledFvPatchField<Type>(p, iF, dict, IOobjectOption::NO_READ),
-    cyclicAMIPatch_(refCast<const cyclicAMIFvPatch>(p, dict))
+    cyclicAMIPatch_(refCast<const cyclicAMIFvPatch>(p, dict)),
+    sendRequests_(0),
+    recvRequests_(0),
+    patchNeighbourFieldPtr_(nullptr)
 {
     if (!isA<cyclicAMIFvPatch>(p))
     {
         FatalIOErrorInFunction(dict)
-            << "    patch type '" << p.type()
+            << "\n    patch type '" << p.type()
             << "' not constraint type '" << typeName << "'"
             << "\n    for patch " << p.name()
             << " of field " << this->internalField().name()
@@ -68,12 +76,24 @@ Foam::cyclicAMIFvPatchField<Type>::cyclicAMIFvPatchField
             << exit(FatalIOError);
     }
 
-    // Use 'value' supplied, or set to coupled or internal field
+    // Handle neighbour value first, before any evaluate()
+    const auto* hasNeighbValue =
+        dict.findEntry("neighbourValue", keyType::LITERAL);
+
+    if (hasNeighbValue)
+    {
+        patchNeighbourFieldPtr_.reset
+        (
+            new Field<Type>(*hasNeighbValue, p.size())
+        );
+    }
+
+    // Use 'value' supplied, or evaluate (if coupled) or set to internal field
     if (!this->readValueEntry(dict))
     {
         if (this->coupled())
         {
-            this->evaluate(Pstream::commsTypes::blocking);
+            this->evaluate(UPstream::commsTypes::nonBlocking);
         }
         else
         {
@@ -94,8 +114,19 @@ Foam::cyclicAMIFvPatchField<Type>::cyclicAMIFvPatchField
 :
     cyclicAMILduInterfaceField(),
     coupledFvPatchField<Type>(ptf, p, iF, mapper),
-    cyclicAMIPatch_(refCast<const cyclicAMIFvPatch>(p))
+    cyclicAMIPatch_(refCast<const cyclicAMIFvPatch>(p)),
+    sendRequests_(0),
+    recvRequests_(0),
+    patchNeighbourFieldPtr_(nullptr)
 {
+    if (ptf.patchNeighbourFieldPtr_)
+    {
+        patchNeighbourFieldPtr_.reset
+        (
+            new Field<Type>(ptf.patchNeighbourFieldPtr_(), mapper)
+        );
+    }
+
     if (!isA<cyclicAMIFvPatch>(this->patch()))
     {
         FatalErrorInFunction
@@ -106,6 +137,12 @@ Foam::cyclicAMIFvPatchField<Type>::cyclicAMIFvPatchField
             << " in file " << this->internalField().objectPath()
             << exit(FatalError);
     }
+    if (debug && !ptf.all_ready())
+    {
+        FatalErrorInFunction
+            << "Outstanding request(s) on patch " << cyclicAMIPatch_.name()
+            << abort(FatalError);
+    }
 }
 
 
@@ -117,8 +154,18 @@ Foam::cyclicAMIFvPatchField<Type>::cyclicAMIFvPatchField
 :
     cyclicAMILduInterfaceField(),
     coupledFvPatchField<Type>(ptf),
-    cyclicAMIPatch_(ptf.cyclicAMIPatch_)
+    cyclicAMIPatch_(ptf.cyclicAMIPatch_),
-{}
+    sendRequests_(0),
+    recvRequests_(0),
+    patchNeighbourFieldPtr_(ptf.patchNeighbourFieldPtr_.clone())
+{
+    if (debug && !ptf.all_ready())
+    {
+        FatalErrorInFunction
+            << "Outstanding request(s) on patch " << cyclicAMIPatch_.name()
+            << abort(FatalError);
+    }
+}
 
 
 template<class Type>
@@ -130,43 +177,111 @@ Foam::cyclicAMIFvPatchField<Type>::cyclicAMIFvPatchField
 :
     cyclicAMILduInterfaceField(),
     coupledFvPatchField<Type>(ptf, iF),
-    cyclicAMIPatch_(ptf.cyclicAMIPatch_)
+    cyclicAMIPatch_(ptf.cyclicAMIPatch_),
-{}
+    sendRequests_(0),
+    recvRequests_(0),
+    patchNeighbourFieldPtr_(ptf.patchNeighbourFieldPtr_.clone())
+{
+    if (debug && !ptf.all_ready())
+    {
+        FatalErrorInFunction
+            << "Outstanding request(s) on patch " << cyclicAMIPatch_.name()
+            << abort(FatalError);
+    }
+}
 
 
 // * * * * * * * * * * * * * * * Member Functions  * * * * * * * * * * * * * //
 
 template<class Type>
-bool Foam::cyclicAMIFvPatchField<Type>::coupled() const
+bool Foam::cyclicAMIFvPatchField<Type>::all_ready() const
 {
-    return cyclicAMIPatch_.coupled();
+    int done = 0;
+
+    if
+    (
+        UPstream::finishedRequests
+        (
+            recvRequests_.start(),
+            recvRequests_.size()
+        )
+    )
+    {
+        recvRequests_.clear();
+        ++done;
+    }
+
+    if
+    (
+        UPstream::finishedRequests
+        (
+            sendRequests_.start(),
+            sendRequests_.size()
+        )
+    )
+    {
+        sendRequests_.clear();
+        ++done;
+    }
+
+    return (done == 2);
+}
+
+
+template<class Type>
+bool Foam::cyclicAMIFvPatchField<Type>::ready() const
+{
+    if
+    (
+        UPstream::finishedRequests
+        (
+            recvRequests_.start(),
+            recvRequests_.size()
+        )
+    )
+    {
+        recvRequests_.clear();
+
+        if
+        (
+            UPstream::finishedRequests
+            (
+                sendRequests_.start(),
+                sendRequests_.size()
+            )
+        )
+        {
+            sendRequests_.clear();
+        }
+
+        return true;
+    }
+
+    return false;
 }
 
 
 template<class Type>
 Foam::tmp<Foam::Field<Type>>
-Foam::cyclicAMIFvPatchField<Type>::patchNeighbourField() const
+Foam::cyclicAMIFvPatchField<Type>::patchNeighbourField
+(
+    const Field<Type>& iField
+) const
 {
-    const Field<Type>& iField = this->primitiveField();
+    // Bypass polyPatch to get nbrId. Instead use cyclicAMIFvPatch virtual
-
-    // By pass polyPatch to get nbrId. Instead use cyclicAMIFvPatch virtual
     // neighbPatch()
     const cyclicAMIFvPatch& neighbPatch = cyclicAMIPatch_.neighbPatch();
     const labelUList& nbrFaceCells = neighbPatch.faceCells();
 
     Field<Type> pnf(iField, nbrFaceCells);
+    Field<Type> defaultValues;
 
-    tmp<Field<Type>> tpnf;
     if (cyclicAMIPatch_.applyLowWeightCorrection())
     {
-        Field<Type> pnfInternal(iField, cyclicAMIPatch_.faceCells());
+        defaultValues = Field<Type>(iField, cyclicAMIPatch_.faceCells());
+    }
 
-        tpnf = cyclicAMIPatch_.interpolate(pnf, pnfInternal);
+    tmp<Field<Type>> tpnf = cyclicAMIPatch_.interpolate(pnf, defaultValues);
-    }
-    else
-    {
-        tpnf = cyclicAMIPatch_.interpolate(pnf);
-    }
 
     if (doTransform())
     {
@@ -177,6 +292,40 @@ Foam::cyclicAMIFvPatchField<Type>::patchNeighbourField() const
 }
 
 
+template<class Type>
+Foam::tmp<Foam::Field<Type>>
+Foam::cyclicAMIFvPatchField<Type>::patchNeighbourField() const
+{
+    if (this->ownerAMI().distributed())
+    {
+        if (!this->ready())
+        {
+            FatalErrorInFunction
+                << "Outstanding recv request(s) on patch "
+                << cyclicAMIPatch_.name()
+                << " field " << this->internalField().name()
+                << abort(FatalError);
+        }
+
+        // Initialise if not done in construct-from-dictionary
+        if (!patchNeighbourFieldPtr_)
+        {
+            // Do interpolation and store result
+            patchNeighbourFieldPtr_.reset
+            (
+                patchNeighbourField(this->primitiveField()).ptr()
+            );
+        }
+        return patchNeighbourFieldPtr_();
+    }
+    else
+    {
+        // Do interpolation
+        return patchNeighbourField(this->primitiveField());
+    }
+}
+
+
 template<class Type>
 const Foam::cyclicAMIFvPatchField<Type>&
 Foam::cyclicAMIFvPatchField<Type>::neighbourPatchField() const
@@ -194,6 +343,150 @@ Foam::cyclicAMIFvPatchField<Type>::neighbourPatchField() const
 }
 
 
+template<class Type>
+void Foam::cyclicAMIFvPatchField<Type>::initEvaluate
+(
+    const Pstream::commsTypes commsType
+)
+{
+    if (!this->updated())
+    {
+        this->updateCoeffs();
+    }
+
+    if (this->ownerAMI().distributed())
+    {
+        if (commsType != UPstream::commsTypes::nonBlocking)
+        {
+            // Invalidate old field - or flag as fatal?
+            patchNeighbourFieldPtr_.reset(nullptr);
+            return;
+        }
+
+        // Start sending
+
+        // By-pass polyPatch to get nbrId. Instead use cyclicAMIFvPatch virtual
+        // neighbPatch()
+        const cyclicAMIFvPatch& neighbPatch = cyclicAMIPatch_.neighbPatch();
+        const labelUList& nbrFaceCells = neighbPatch.faceCells();
+        const Field<Type> pnf(this->primitiveField(), nbrFaceCells);
+
+        const cyclicAMIPolyPatch& cpp = cyclicAMIPatch_.cyclicAMIPatch();
+
+        cpp.initInterpolate
+        (
+            pnf,
+            sendRequests_,
+            sendBufs_,
+            recvRequests_,
+            recvBufs_
+        );
+    }
+}
+
+
+template<class Type>
+void Foam::cyclicAMIFvPatchField<Type>::evaluate
+(
+    const Pstream::commsTypes commsType
+)
+{
+    if (!this->updated())
+    {
+        this->updateCoeffs();
+    }
+
+    const auto& AMI = this->ownerAMI();
+
+    if (AMI.distributed())
+    {
+        // Calculate patchNeighbourField
+        if (commsType != UPstream::commsTypes::nonBlocking)
+        {
+            FatalErrorInFunction
+                << "Can only evaluate distributed AMI with nonBlocking"
+                << exit(FatalError);
+        }
+
+        patchNeighbourFieldPtr_.reset(nullptr);
+
+        const cyclicAMIPolyPatch& cpp = cyclicAMIPatch_.cyclicAMIPatch();
+
+        Field<Type> defaultValues;
+        if (AMI.applyLowWeightCorrection())
+        {
+            defaultValues = this->patchInternalField();
+        }
+
+        patchNeighbourFieldPtr_.reset
+        (
+            cpp.interpolate
+            (
+                Field<Type>::null(),  // Not used for distributed
+                recvRequests_,
+                recvBufs_,
+                defaultValues
+            ).ptr()
+        );
+        auto& patchNeighbourField = patchNeighbourFieldPtr_.ref();
+
+        if (doTransform())
+        {
+            // In-place transform
+            transform(patchNeighbourField, forwardT(), patchNeighbourField);
+        }
+    }
+
+    // Use patchNeighbourField() and patchInternalField() to obtain face value
+    coupledFvPatchField<Type>::evaluate(commsType);
+}
+
+
+template<class Type>
+void Foam::cyclicAMIFvPatchField<Type>::initInterfaceMatrixUpdate
+(
+    solveScalarField& result,
+    const bool add,
+    const lduAddressing& lduAddr,
+    const label patchId,
+    const solveScalarField& psiInternal,
+    const scalarField& coeffs,
+    const direction cmpt,
+    const Pstream::commsTypes commsType
+) const
+{
+    if (this->ownerAMI().distributed())
+    {
+        // Start sending
+        if (commsType != UPstream::commsTypes::nonBlocking)
+        {
+            FatalErrorInFunction
+                << "Can only evaluate distributed AMI with nonBlocking"
+                << exit(FatalError);
+        }
+
+        const labelUList& nbrFaceCells =
+            lduAddr.patchAddr(cyclicAMIPatch_.neighbPatchID());
+
+        solveScalarField pnf(psiInternal, nbrFaceCells);
+
+        // Transform according to the transformation tensors
+        transformCoupleField(pnf, cmpt);
+
+        const cyclicAMIPolyPatch& cpp = cyclicAMIPatch_.cyclicAMIPatch();
+
+        cpp.initInterpolate
+        (
+            pnf,
+            sendRequests_,
+            scalarSendBufs_,
+            recvRequests_,
+            scalarRecvBufs_
+        );
+    }
+}
+
+
 template<class Type>
 void Foam::cyclicAMIFvPatchField<Type>::updateInterfaceMatrix
 (
@@ -204,27 +497,63 @@ void Foam::cyclicAMIFvPatchField<Type>::updateInterfaceMatrix
     const solveScalarField& psiInternal,
     const scalarField& coeffs,
     const direction cmpt,
-    const Pstream::commsTypes
+    const Pstream::commsTypes commsType
 ) const
 {
+    const labelUList& faceCells = lduAddr.patchAddr(patchId);
+
+    const auto& AMI =
+    (
+        cyclicAMIPatch_.owner()
+      ? cyclicAMIPatch_.AMI()
+      : cyclicAMIPatch_.neighbPatch().AMI()
+    );
+
+    solveScalarField pnf;
+
+    if (this->ownerAMI().distributed())
+    {
+        if (commsType != UPstream::commsTypes::nonBlocking)
+        {
+            FatalErrorInFunction
+                << "Can only evaluate distributed AMI with nonBlocking"
+                << exit(FatalError);
+        }
+
+        solveScalarField defaultValues;
+        if (AMI.applyLowWeightCorrection())
+        {
+            defaultValues = solveScalarField(psiInternal, faceCells);
+        }
+
+        const cyclicAMIPolyPatch& cpp = cyclicAMIPatch_.cyclicAMIPatch();
+
+        pnf =
+            cpp.interpolate
+            (
+                solveScalarField::null(),  // Not used for distributed
+                recvRequests_,
+                scalarRecvBufs_,
+                defaultValues
+            );
+    }
+    else
+    {
+        solveScalarField defaultValues;
+        if (cyclicAMIPatch_.applyLowWeightCorrection())
+        {
+            defaultValues = solveScalarField(psiInternal, faceCells);
+        }
+
         const labelUList& nbrFaceCells =
             lduAddr.patchAddr(cyclicAMIPatch_.neighbPatchID());
 
-    solveScalarField pnf(psiInternal, nbrFaceCells);
+        pnf = solveScalarField(psiInternal, nbrFaceCells);
-
-    const labelUList& faceCells = lduAddr.patchAddr(patchId);
 
         // Transform according to the transformation tensors
         transformCoupleField(pnf, cmpt);
 
-    if (cyclicAMIPatch_.applyLowWeightCorrection())
+        pnf = cyclicAMIPatch_.interpolate(pnf, defaultValues);
-    {
-        solveScalarField pif(psiInternal, faceCells);
-        pnf = cyclicAMIPatch_.interpolate(pnf, pif);
-    }
-    else
-    {
-        pnf = cyclicAMIPatch_.interpolate(pnf);
     }
 
     // Multiply the field by coefficients and add into the result
@@ -232,6 +561,51 @@ void Foam::cyclicAMIFvPatchField<Type>::updateInterfaceMatrix
 }
 
 
+template<class Type>
+void Foam::cyclicAMIFvPatchField<Type>::initInterfaceMatrixUpdate
+(
+    Field<Type>& result,
+    const bool add,
+    const lduAddressing& lduAddr,
+    const label patchId,
+    const Field<Type>& psiInternal,
+    const scalarField& coeffs,
+    const Pstream::commsTypes commsType
+) const
+{
+    const auto& AMI = this->ownerAMI();
+
+    if (AMI.distributed())
+    {
+        if (commsType != UPstream::commsTypes::nonBlocking)
+        {
+            FatalErrorInFunction
+                << "Can only evaluate distributed AMI with nonBlocking"
+                << exit(FatalError);
+        }
+
+        const labelUList& nbrFaceCells =
+            lduAddr.patchAddr(cyclicAMIPatch_.neighbPatchID());
+
+        Field<Type> pnf(psiInternal, nbrFaceCells);
+
+        // Transform according to the transformation tensors
+        transformCoupleField(pnf);
+
+        const cyclicAMIPolyPatch& cpp = cyclicAMIPatch_.cyclicAMIPatch();
+
+        cpp.initInterpolate
+        (
+            pnf,
+            sendRequests_,
+            sendBufs_,
+            recvRequests_,
+            recvBufs_
+        );
+    }
+}
+
+
 template<class Type>
 void Foam::cyclicAMIFvPatchField<Type>::updateInterfaceMatrix
 (
@@ -241,9 +615,43 @@ void Foam::cyclicAMIFvPatchField<Type>::updateInterfaceMatrix
     const label patchId,
     const Field<Type>& psiInternal,
     const scalarField& coeffs,
-    const Pstream::commsTypes
+    const Pstream::commsTypes commsType
 ) const
 {
+    const labelUList& faceCells = lduAddr.patchAddr(patchId);
+
+    const auto& AMI = this->ownerAMI();
+
+    Field<Type> pnf;
+
+    if (AMI.distributed())
+    {
+        if (commsType != UPstream::commsTypes::nonBlocking)
+        {
+            FatalErrorInFunction
+                << "Can only evaluate distributed AMI with nonBlocking"
+                << exit(FatalError);
+        }
+
+        const cyclicAMIPolyPatch& cpp = cyclicAMIPatch_.cyclicAMIPatch();
+
+        Field<Type> defaultValues;
+        if (AMI.applyLowWeightCorrection())
+        {
+            defaultValues = Field<Type>(psiInternal, faceCells);
+        }
+
+        pnf =
+            cpp.interpolate
+            (
+                Field<Type>::null(),  // Not used for distributed
+                recvRequests_,
+                recvBufs_,
+                defaultValues
+            );
+    }
+    else
+    {
         const labelUList& nbrFaceCells =
             lduAddr.patchAddr(cyclicAMIPatch_.neighbPatchID());
 
@@ -252,17 +660,14 @@ void Foam::cyclicAMIFvPatchField<Type>::updateInterfaceMatrix
         // Transform according to the transformation tensors
         transformCoupleField(pnf);
 
+        Field<Type> defaultValues;
         if (cyclicAMIPatch_.applyLowWeightCorrection())
         {
-        Field<Type> pif(psiInternal, cyclicAMIPatch_.faceCells());
+            defaultValues = Field<Type>(psiInternal, faceCells);
-        pnf = cyclicAMIPatch_.interpolate(pnf, pif);
-    }
-    else
-    {
-        pnf = cyclicAMIPatch_.interpolate(pnf);
         }
 
-    const labelUList& faceCells = lduAddr.patchAddr(patchId);
+        pnf = cyclicAMIPatch_.interpolate(pnf, defaultValues);
+    }
 
     // Multiply the field by coefficients and add into the result
     this->addToInternalField(result, !add, faceCells, coeffs, pnf);
@@ -277,10 +682,9 @@ void Foam::cyclicAMIFvPatchField<Type>::manipulateMatrix
     const direction cmpt
 )
 {
-
     if (this->cyclicAMIPatch().owner())
     {
-        label index = this->patch().index();
+        const label index = this->patch().index();
 
         const label globalPatchID =
             matrix.lduMeshAssembly().patchLocalToGlobalMap()[mat][index];
@@ -376,7 +780,8 @@ Foam::cyclicAMIFvPatchField<Type>::coeffs
         matrix.lduMeshAssembly().cellBoundMap()[mat][index].size()
     );
 
-    Field<scalar> mapCoeffs(nSubFaces, Zero);
+    auto tmapCoeffs = tmp<Field<scalar>>::New(nSubFaces, Zero);
+    auto& mapCoeffs = tmapCoeffs.ref();
 
     const scalarListList& srcWeight =
         cyclicAMIPatch_.cyclicAMIPatch().AMI().srcWeights();
@@ -394,7 +799,7 @@ Foam::cyclicAMIFvPatchField<Type>::coeffs
         }
     }
 
-    return tmp<Field<scalar>>(new Field<scalar>(mapCoeffs));
+    return tmapCoeffs;
 }
 
 
@@ -408,7 +813,7 @@ void Foam::cyclicAMIFvPatchField<Type>::collectStencilData
     List<Type2>& expandedData
 )
 {
-    expandedData.setSize(stencil.size());
+    expandedData.resize_nocopy(stencil.size());
     if (mapPtr)
     {
         Type2 work(data);
@@ -417,7 +822,7 @@ void Foam::cyclicAMIFvPatchField<Type>::collectStencilData
         forAll(stencil, facei)
         {
             const labelList& slots = stencil[facei];
-            expandedData[facei].append
+            expandedData[facei].push_back
             (
                 UIndirectList<typename Type2::value_type>(work, slots)
             );
@@ -428,7 +833,7 @@ void Foam::cyclicAMIFvPatchField<Type>::collectStencilData
         forAll(stencil, facei)
         {
             const labelList& slots = stencil[facei];
-            expandedData[facei].append
+            expandedData[facei].push_back
             (
                 UIndirectList<typename Type2::value_type>(data, slots)
             );
@@ -442,8 +847,12 @@ void Foam::cyclicAMIFvPatchField<Type>::write(Ostream& os) const
 {
     fvPatchField<Type>::write(os);
     fvPatchField<Type>::writeValueEntry(os);
+
+    if (patchNeighbourFieldPtr_)
+    {
+        patchNeighbourFieldPtr_->writeEntry("neighbourValue", os);
+    }
 }
 
 
 // ************************************************************************* //
-

src/finiteVolume/fields/fvPatchFields/constraint/cyclicAMI/cyclicAMIFvPatchField.H

@@ -6,7 +6,7 @@
      \\/     M anipulation  |
 -------------------------------------------------------------------------------
     Copyright (C) 2011-2016 OpenFOAM Foundation
-    Copyright (C) 2019 OpenCFD Ltd.
+    Copyright (C) 2019-2023 OpenCFD Ltd.
 -------------------------------------------------------------------------------
 License
     This file is part of OpenFOAM.
@@ -41,16 +41,24 @@ Usage
     <patchName>
     {
         type            cyclicAMI;
+        value           <initial value>;
+        neighbourValue  <initial value of neighbour patch cells>;
     }
     \endverbatim
 
 Note
     The outer boundary of the patch pairs must be similar, i.e. if the owner
-    patch is transformed to the neighbour patch, the outer perimiter of each
+    patch is transformed to the neighbour patch, the outer perimeter of each
     patch should be identical (or very similar).
 
+    The \c neighbourValue is only needed when running distributed,
+    i.e. the neighbour cells are on a different processor from the owner cells.
+    It guarantees consistent restart e.g. when doing a snGrad and avoids
+    additional communication.
+
 See also
     Foam::AMIInterpolation
+    Foam::processorFvPatchField
 
 SourceFiles
     cyclicAMIFvPatchField.C
@@ -79,14 +87,55 @@ class cyclicAMIFvPatchField
     virtual public cyclicAMILduInterfaceField,
     public coupledFvPatchField<Type>
 {
-    // Private data
+    // Private Data
 
         //- Local reference cast into the cyclic patch
         const cyclicAMIFvPatch& cyclicAMIPatch_;
 
 
+        // Sending and receiving (distributed AMI)
+
+            //- Current range of send requests (non-blocking)
+            mutable labelRange sendRequests_;
+
+            //- Current range of recv requests (non-blocking)
+            mutable labelRange recvRequests_;
+
+            //- Send buffers
+            mutable PtrList<List<Type>> sendBufs_;
+
+            //- Receive buffers_
+            mutable PtrList<List<Type>> recvBufs_;
+
+            //- Scalar send buffers
+            mutable PtrList<List<solveScalar>> scalarSendBufs_;
+
+            //- Scalar receive buffers
+            mutable PtrList<List<solveScalar>> scalarRecvBufs_;
+
+            //- Neighbour coupled internal cell data
+            mutable autoPtr<Field<Type>> patchNeighbourFieldPtr_;
+
+
     // Private Member Functions
 
+        //- Return the AMI corresponding to the owner side
+        const AMIPatchToPatchInterpolation& ownerAMI() const
+        {
+            return
+            (
+                cyclicAMIPatch_.owner()
+              ? cyclicAMIPatch_.AMI()
+              : cyclicAMIPatch_.neighbPatch().AMI()
+            );
+        }
+
+        //- All receive/send requests have completed
+        virtual bool all_ready() const;
+
+        //- Return neighbour coupled internal cell data
+        tmp<Field<Type>> patchNeighbourField(const Field<Type>&) const;
+
         //- Return neighbour side field given internal fields
         template<class Type2>
         tmp<Field<Type2>> neighbourSideField
@@ -103,7 +152,6 @@ class cyclicAMIFvPatchField
             const label
         ) const;
 
-
         template<class Type2>
         void collectStencilData
         (
@@ -178,22 +226,23 @@ public:
         }
 
 
-    // Member functions
+    // Member Functions
-
-        // Access
 
         //- Return local reference cast into the cyclic AMI patch
-            const cyclicAMIFvPatch& cyclicAMIPatch() const
+        const cyclicAMIFvPatch& cyclicAMIPatch() const noexcept
         {
             return cyclicAMIPatch_;
         }
 
 
-        // Evaluation functions
+    // Coupling
 
         //- Return true if coupled. Note that the underlying patch
         //  is not coupled() - the points don't align.
-            virtual bool coupled() const;
+        virtual bool coupled() const { return cyclicAMIPatch_.coupled(); }
+
+        //- Are all (receive) data available?
+        virtual bool ready() const;
 
         //- Return neighbour coupled internal cell data
         virtual tmp<Field<Type>> patchNeighbourField() const;
@@ -201,6 +250,31 @@ public:
         //- Return reference to neighbour patchField
         const cyclicAMIFvPatchField<Type>& neighbourPatchField() const;
 
+
+    // Evaluation
+
+        //- Initialise the evaluation of the patch field
+        virtual void initEvaluate(const Pstream::commsTypes commsType);
+
+        //- Evaluate the patch field
+        virtual void evaluate(const Pstream::commsTypes commsType);
+
+
+        // Coupled interface functionality
+
+            //- Initialise neighbour matrix update
+            virtual void initInterfaceMatrixUpdate
+            (
+                solveScalarField& result,
+                const bool add,
+                const lduAddressing& lduAddr,
+                const label patchId,
+                const solveScalarField& psiInternal,
+                const scalarField& coeffs,
+                const direction cmpt,
+                const Pstream::commsTypes commsType
+            ) const;
+
             //- Update result field based on interface functionality
             virtual void updateInterfaceMatrix
             (
@@ -214,6 +288,18 @@ public:
                 const Pstream::commsTypes commsType
             ) const;
 
+            //- Initialise neighbour matrix update
+            virtual void initInterfaceMatrixUpdate
+            (
+                Field<Type>& result,
+                const bool add,
+                const lduAddressing& lduAddr,
+                const label patchId,
+                const Field<Type>& psiInternal,
+                const scalarField& coeffs,
+                const Pstream::commsTypes commsType
+            ) const;
+
             //- Update result field based on interface functionality
             virtual void updateInterfaceMatrix
             (
@@ -236,7 +322,7 @@ public:
             );
 
 
-        // Cyclic AMI coupled interface functions
+        // Coupled interface functions
 
             //- Does the patch field perform the transformation
             virtual bool doTransform() const

src/finiteVolume/fields/fvPatchFields/constraint/cyclicAMI/cyclicAMIFvPatchFields.C

@@ -26,8 +26,11 @@ License
 \*---------------------------------------------------------------------------*/
 
 #include "cyclicAMIFvPatchFields.H"
-#include "addToRunTimeSelectionTable.H"
+#include "cyclicAMIPolyPatch.H"
+#include "mapDistributeBase.H"
+#include "AMIInterpolation.H"
 #include "volFields.H"
+#include "addToRunTimeSelectionTable.H"
 
 // * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
 

src/finiteVolume/fields/fvPatchFields/constraint/cyclicAMI/cyclicAMIFvPatchFields.H

@@ -25,8 +25,8 @@ License
 
 \*---------------------------------------------------------------------------*/
 
-#ifndef cyclicAMIFvPatchFields_H
+#ifndef Foam_cyclicAMIFvPatchFields_H
-#define cyclicAMIFvPatchFields_H
+#define Foam_cyclicAMIFvPatchFields_H
 
 #include "cyclicAMIFvPatchField.H"
 #include "fieldTypes.H"

src/meshTools/AMIInterpolation/AMIInterpolation/AMIInterpolation.C

@@ -762,25 +762,27 @@ bool Foam::AMIInterpolation::calculate
 
     if (srcTotalSize == 0)
     {
-        DebugInfo<< "AMI: no source faces present - no addressing constructed"
+        DebugInfo
+            << "AMI: no source faces present - no addressing constructed"
             << endl;
 
         return false;
     }
 
-    Info<< indent
-        << "AMI: Patch source faces: " << srcTotalSize << nl
-        << "AMI: Patch target faces: " << tgtTotalSize << endl;
-
     singlePatchProc_ = calcDistribution(srcPatch, tgtPatch);
 
-    if (debug)
+    Info<< indent << "AMI: Patch source faces: " << srcTotalSize << nl
+        << indent << "AMI: Patch target faces: " << tgtTotalSize << nl;
+
+    if (distributed())
     {
-        Info<< "AMIInterpolation:" << nl
+        Info<< indent << "AMI: distributed" << endl;
-            << "    singlePatchProc:" << singlePatchProc_ << nl
-            << endl;
     }
 
+    DebugInfo
+        << "AMI: patch proc:" << singlePatchProc_
+        << endl;
+
     return true;
 }
 

src/meshTools/AMIInterpolation/AMIInterpolation/AMIInterpolation.H

@@ -506,6 +506,30 @@ public:
 
             // Low-level
 
+                //- Weighted sum of contributions
+                template<class Type, class CombineOp>
+                static void weightedSum
+                (
+                    const scalar lowWeightCorrection,
+                    const labelListList& allSlots,
+                    const scalarListList& allWeights,
+                    const scalarField& weightsSum,
+                    const UList<Type>& fld,
+                    const CombineOp& cop,
+                    List<Type>& result,
+                    const UList<Type>& defaultValues
+                );
+
+                //- Weighted sum of contributions
+                template<class Type>
+                void weightedSum
+                (
+                    const bool interpolateToSource,
+                    const UList<Type>& fld,
+                    List<Type>& result,
+                    const UList<Type>& defaultValues
+                ) const;
+
                 //- Interpolate from target to source with supplied op
                 //- to combine existing value with remote value and weight
                 template<class Type, class CombineOp>

src/meshTools/AMIInterpolation/AMIInterpolation/AMIInterpolationTemplates.C

@@ -6,7 +6,7 @@
      \\/     M anipulation  |
 -------------------------------------------------------------------------------
     Copyright (C) 2011-2017 OpenFOAM Foundation
-    Copyright (C) 2015-2018 OpenCFD Ltd.
+    Copyright (C) 2015-2023 OpenCFD Ltd.
 -------------------------------------------------------------------------------
 License
     This file is part of OpenFOAM.
@@ -31,6 +31,78 @@ License
 
 // * * * * * * * * * * * * * Private Member Functions  * * * * * * * * * * * //
 
+template<class Type, class CombineOp>
+void Foam::AMIInterpolation::weightedSum
+(
+    const scalar lowWeightCorrection,
+    const labelListList& allSlots,
+    const scalarListList& allWeights,
+    const scalarField& weightsSum,
+    const UList<Type>& fld,
+    const CombineOp& cop,
+    List<Type>& result,
+    const UList<Type>& defaultValues
+)
+{
+    if (lowWeightCorrection > 0)
+    {
+        forAll(result, facei)
+        {
+            if (weightsSum[facei] < lowWeightCorrection)
+            {
+                result[facei] = defaultValues[facei];
+            }
+            else
+            {
+                const labelList& slots = allSlots[facei];
+                const scalarList& weights = allWeights[facei];
+
+                forAll(slots, i)
+                {
+                    cop(result[facei], facei, fld[slots[i]], weights[i]);
+                }
+            }
+        }
+    }
+    else
+    {
+        forAll(result, facei)
+        {
+            const labelList& slots = allSlots[facei];
+            const scalarList& weights = allWeights[facei];
+
+            forAll(slots, i)
+            {
+                cop(result[facei], facei, fld[slots[i]], weights[i]);
+            }
+        }
+    }
+}
+
+
+template<class Type>
+void Foam::AMIInterpolation::weightedSum
+(
+    const bool interpolateToSource,
+    const UList<Type>& fld,
+    List<Type>& result,
+    const UList<Type>& defaultValues
+) const
+{
+    weightedSum
+    (
+        lowWeightCorrection_,
+        (interpolateToSource ? srcAddress_ : tgtAddress_),
+        (interpolateToSource ? srcWeights_ : tgtWeights_),
+        (interpolateToSource ? srcWeightsSum_ : tgtWeightsSum_),
+        fld,
+        multiplyWeightedOp<Type, plusEqOp<Type>>(plusEqOp<Type>()),
+        result,
+        defaultValues
+    );
+}
+
+
 template<class Type, class CombineOp>
 void Foam::AMIInterpolation::interpolateToTarget
 (
@@ -51,10 +123,11 @@ void Foam::AMIInterpolation::interpolateToTarget
             << "    supplied field = " << fld.size()
             << abort(FatalError);
     }
-
+    else if
-    if (lowWeightCorrection_ > 0)
+    (
-    {
+        (lowWeightCorrection_ > 0)
-        if (defaultValues.size() != tgtAddress_.size())
+     && (defaultValues.size() != tgtAddress_.size())
+    )
     {
         FatalErrorInFunction
             << "Employing default values when sum of weights falls below "
@@ -65,55 +138,28 @@ void Foam::AMIInterpolation::interpolateToTarget
             << "    target patch   = " << tgtAddress_.size() << nl
             << abort(FatalError);
     }
-    }
 
     result.setSize(tgtAddress_.size());
+    List<Type> work;
 
     if (distributed())
     {
         const mapDistribute& map = srcMapPtr_();
-
+        work = fld;  // deep copy
-        List<Type> work(fld);
         map.distribute(work);
+    }
 
-        forAll(result, facei)
+    weightedSum
-        {
+    (
-            if (tgtWeightsSum_[facei] < lowWeightCorrection_)
+        lowWeightCorrection_,
-            {
+        tgtAddress_,
-                result[facei] = defaultValues[facei];
+        tgtWeights_,
-            }
+        tgtWeightsSum_,
-            else
+        (distributed() ? work : fld),
-            {
+        cop,
-                const labelList& faces = tgtAddress_[facei];
+        result,
-                const scalarList& weights = tgtWeights_[facei];
+        defaultValues
-
+    );
-                forAll(faces, i)
-                {
-                    cop(result[facei], facei, work[faces[i]], weights[i]);
-                }
-            }
-        }
-    }
-    else
-    {
-        forAll(result, facei)
-        {
-            if (tgtWeightsSum_[facei] < lowWeightCorrection_)
-            {
-                result[facei] = defaultValues[facei];
-            }
-            else
-            {
-                const labelList& faces = tgtAddress_[facei];
-                const scalarList& weights = tgtWeights_[facei];
-
-                forAll(faces, i)
-                {
-                    cop(result[facei], facei, fld[faces[i]], weights[i]);
-                }
-            }
-        }
-    }
 }
 
 
@@ -137,69 +183,43 @@ void Foam::AMIInterpolation::interpolateToSource
             << "    supplied field = " << fld.size()
             << abort(FatalError);
     }
-
+    else if
-    if (lowWeightCorrection_ > 0)
+    (
-    {
+        (lowWeightCorrection_ > 0)
-        if (defaultValues.size() != srcAddress_.size())
+     && (defaultValues.size() != srcAddress_.size())
+    )
     {
         FatalErrorInFunction
             << "Employing default values when sum of weights falls below "
             << lowWeightCorrection_
-                << " but supplied default field size is not equal to source "
+            << " but number of default values is not equal to source "
             << "patch size" << nl
             << "    default values = " << defaultValues.size() << nl
             << "    source patch   = " << srcAddress_.size() << nl
             << abort(FatalError);
     }
-    }
 
     result.setSize(srcAddress_.size());
+    List<Type> work;
 
     if (distributed())
     {
         const mapDistribute& map = tgtMapPtr_();
-
+        work = fld;  // deep copy
-        List<Type> work(fld);
         map.distribute(work);
+    }
 
-        forAll(result, facei)
+    weightedSum
-        {
+    (
-            if (srcWeightsSum_[facei] < lowWeightCorrection_)
+        lowWeightCorrection_,
-            {
+        srcAddress_,
-                result[facei] = defaultValues[facei];
+        srcWeights_,
-            }
+        srcWeightsSum_,
-            else
+        (distributed() ? work : fld),
-            {
+        cop,
-                const labelList& faces = srcAddress_[facei];
+        result,
-                const scalarList& weights = srcWeights_[facei];
+        defaultValues
-
+    );
-                forAll(faces, i)
-                {
-                    cop(result[facei], facei, work[faces[i]], weights[i]);
-                }
-            }
-        }
-    }
-    else
-    {
-        forAll(result, facei)
-        {
-            if (srcWeightsSum_[facei] < lowWeightCorrection_)
-            {
-                result[facei] = defaultValues[facei];
-            }
-            else
-            {
-                const labelList& faces = srcAddress_[facei];
-                const scalarList& weights = srcWeights_[facei];
-
-                forAll(faces, i)
-                {
-                    cop(result[facei], facei, fld[faces[i]], weights[i]);
-                }
-            }
-        }
-    }
 }
 
 

src/meshTools/AMIInterpolation/patches/cyclicAMI/cyclicAMIPolyPatch/cyclicAMIPolyPatch.C

@@ -6,7 +6,7 @@
      \\/     M anipulation  |
 -------------------------------------------------------------------------------
     Copyright (C) 2011-2016 OpenFOAM Foundation
-    Copyright (C) 2016-2021 OpenCFD Ltd.
+    Copyright (C) 2016-2023 OpenCFD Ltd.
 -------------------------------------------------------------------------------
 License
     This file is part of OpenFOAM.
@@ -294,30 +294,29 @@ Foam::cyclicAMIPolyPatch::cylindricalCS() const
     if (periodicID != -1)
     {
         // Get the periodic patch
-        const coupledPolyPatch& perPp
+        const coupledPolyPatch& perPp =
-        (
+            refCast<const coupledPolyPatch>(boundaryMesh()[periodicID]);
-            refCast<const coupledPolyPatch>
+
-            (
-                boundaryMesh()[periodicID]
-            )
-        );
         if (!perPp.parallel())
         {
             vector axis(Zero);
             point axisPoint(Zero);
-            if (isA<cyclicPolyPatch>(perPp))
+
+            if
+            (
+                const cyclicPolyPatch* cpp = isA<cyclicPolyPatch>(perPp)
+            )
             {
-                axis =
+                axis = cpp->rotationAxis();
-                    refCast<const cyclicPolyPatch>(perPp).rotationAxis();
+                axisPoint = cpp->rotationCentre();
-                axisPoint =
-                    refCast<const cyclicPolyPatch>(perPp).rotationCentre();
             }
-            else if (isA<cyclicAMIPolyPatch>(perPp))
+            else if
+            (
+                const cyclicAMIPolyPatch* cpp = isA<cyclicAMIPolyPatch>(perPp)
+            )
             {
-                axis =
+                axis = cpp->rotationAxis();
-                    refCast<const cyclicAMIPolyPatch>(perPp).rotationAxis();
+                axisPoint = cpp->rotationCentre();
-                axisPoint =
-                    refCast<const cyclicAMIPolyPatch>(perPp).rotationCentre();
             }
             else
             {

src/meshTools/AMIInterpolation/patches/cyclicAMI/cyclicAMIPolyPatch/cyclicAMIPolyPatch.H

@@ -6,7 +6,7 @@
      \\/     M anipulation  |
 -------------------------------------------------------------------------------
     Copyright (C) 2011-2016 OpenFOAM Foundation
-    Copyright (C) 2018-2021 OpenCFD Ltd.
+    Copyright (C) 2018-2023 OpenCFD Ltd.
 -------------------------------------------------------------------------------
 License
     This file is part of OpenFOAM.
@@ -47,9 +47,10 @@ SourceFiles
 
 \*---------------------------------------------------------------------------*/
 
-#ifndef cyclicAMIPolyPatch_H
+#ifndef Foam_cyclicAMIPolyPatch_H
-#define cyclicAMIPolyPatch_H
+#define Foam_cyclicAMIPolyPatch_H
 
+#include "AMIInterpolation.H"
 #include "coupledPolyPatch.H"
 #include "AMIPatchToPatchInterpolation.H"
 #include "polyBoundaryMesh.H"
@@ -87,7 +88,7 @@ class cyclicAMIPolyPatch
 
 protected:
 
-    // Protected data
+    // Protected Data
 
         //- Name of other half
         mutable word nbrPatchName_;
@@ -286,7 +287,7 @@ public:
         }
 
         //- Construct and return a clone, resetting the face list
-        //  and boundary mesh
+        //- and boundary mesh
         virtual autoPtr<polyPatch> clone
         (
             const polyBoundaryMesh& bm,
@@ -467,7 +468,7 @@ public:
                 ) const;
 
                 //- Transform a patch-based direction from this side to
-                //  nbr side
+                //- nbr side
                 virtual void reverseTransformDirection
                 (
                     vector& d,
@@ -512,6 +513,38 @@ public:
                 ) const;
 
 
+            // Interpolations (non-blocking). Subject to change
+
+                template<class Type>
+                void initInterpolateUntransformed
+                (
+                    const Field<Type>& fld,
+                    labelRange& sendRequests,
+                    PtrList<List<Type>>& sendBuffers,
+                    labelRange& recvRequests,
+                    PtrList<List<Type>>& recvBuffers
+                ) const;
+
+                template<class Type>
+                void initInterpolate
+                (
+                    const Field<Type>& fld,
+                    labelRange& sendRequests,
+                    PtrList<List<Type>>& sendBuffers,
+                    labelRange& recvRequests,
+                    PtrList<List<Type>>& recvBuffers
+                ) const;
+
+                template<class Type>
+                tmp<Field<Type>> interpolate
+                (
+                    const Field<Type>& localFld,
+                    const labelRange& requests,  // The receive requests
+                    const PtrList<List<Type>>& recvBuffers,
+                    const UList<Type>& defaultValues
+                ) const;
+
+
         //- Calculate the patch geometry
         virtual void calcGeometry
         (

src/meshTools/AMIInterpolation/patches/cyclicAMI/cyclicAMIPolyPatch/cyclicAMIPolyPatchTemplates.C

@@ -6,7 +6,7 @@
      \\/     M anipulation  |
 -------------------------------------------------------------------------------
     Copyright (C) 2011-2016 OpenFOAM Foundation
-    Copyright (C) 2021 OpenCFD Ltd.
+    Copyright (C) 2021-2023 OpenCFD Ltd.
 -------------------------------------------------------------------------------
 License
     This file is part of OpenFOAM.
@@ -53,13 +53,16 @@ Foam::tmp<Foam::Field<Type>> Foam::cyclicAMIPolyPatch::interpolate
     const UList<Type>& defaultValues
 ) const
 {
-    if (pTraits<Type>::rank == 0)
+    autoPtr<coordSystem::cylindrical> cs;
+
+    // Similar to doTransform.
+    // - could also check if !std::is_same<sphericalTensor, Type>:value
+
+    if (is_vectorspace<Type>::value)
     {
-        return interpolateUntransformed(fld, defaultValues);
+        cs.reset(cylindricalCS());
     }
-    else
+
-    {
-        autoPtr<coordSystem::cylindrical> cs(cylindricalCS());
     if (!cs)
     {
         return interpolateUntransformed(fld, defaultValues);
@@ -90,13 +93,12 @@ Foam::tmp<Foam::Field<Type>> Foam::cyclicAMIPolyPatch::interpolate
         }
 
 
-            auto tlocalFld(tmp<Field<Type>>::New(fld.size()));
+        Field<Type> localFld(fld.size());
-            Field<Type>& localFld = tlocalFld.ref();
 
         // Transform to cylindrical coords
         {
-                tmp<tensorField> nbrT(cs().R(nbrPp.faceCentres()));
+            const tensorField nbrT(cs().R(nbrPp.faceCentres()));
-                localFld = Foam::invTransform(nbrT(), fld);
+            Foam::invTransform(localFld, nbrT, fld);
         }
 
         if (debug&2)
@@ -121,9 +123,9 @@ Foam::tmp<Foam::Field<Type>> Foam::cyclicAMIPolyPatch::interpolate
         }
 
 
-            const tmp<tensorField> T(cs().R(this->faceCentres()));
+        const tensorField ownT(cs().R(this->faceCentres()));
 
-            List<Type> localDeflt(defaultValues.size());
+        Field<Type> localDeflt(defaultValues.size());
         if (defaultValues.size() == size())
         {
             // Transform default values into cylindrical coords (using
@@ -132,18 +134,16 @@ Foam::tmp<Foam::Field<Type>> Foam::cyclicAMIPolyPatch::interpolate
             // Field so we can use transformField routines.
             const SubField<Type> defaultSubFld(defaultValues);
             const Field<Type>& defaultFld(defaultSubFld);
-                localDeflt = Foam::invTransform(T(), defaultFld);
+            Foam::invTransform(localDeflt, ownT, defaultFld);
         }
 
         // Do the actual interpolation and interpolate back to cartesian
-            // coords
         return Foam::transform
         (
-                T,
+            ownT,
             interpolateUntransformed(localFld, localDeflt)
         );
     }
-    }
 }
 
 
@@ -158,6 +158,160 @@ Foam::tmp<Foam::Field<Type>> Foam::cyclicAMIPolyPatch::interpolate
 }
 
 
+template<class Type>
+void Foam::cyclicAMIPolyPatch::initInterpolateUntransformed
+(
+    const Field<Type>& fld,
+    labelRange& sendRequests,
+    PtrList<List<Type>>& sendBuffers,
+    labelRange& recvRequests,
+    PtrList<List<Type>>& recvBuffers
+) const
+{
+    const auto& AMI = (owner() ? this->AMI() : neighbPatch().AMI());
+
+    if (AMI.distributed())
+    {
+        const auto& map = (owner() ? AMI.tgtMap() : AMI.srcMap());
+
+        // Insert send/receive requests (non-blocking)
+        map.send(fld, sendRequests, sendBuffers, recvRequests, recvBuffers);
+    }
+}
+
+
+template<class Type>
+void Foam::cyclicAMIPolyPatch::initInterpolate
+(
+    const Field<Type>& fld,
+    labelRange& sendRequests,
+    PtrList<List<Type>>& sendBuffers,
+    labelRange& recvRequests,
+    PtrList<List<Type>>& recvBuffers
+) const
+{
+    const auto& AMI = (owner() ? this->AMI() : neighbPatch().AMI());
+
+    if (!AMI.distributed())
+    {
+        return;
+    }
+
+    autoPtr<coordSystem::cylindrical> cs;
+
+    if (is_vectorspace<Type>::value)
+    {
+        cs.reset(cylindricalCS());
+    }
+
+    if (!cs)
+    {
+        initInterpolateUntransformed
+        (
+            fld,
+            sendRequests,
+            sendBuffers,
+            recvRequests,
+            recvBuffers
+        );
+    }
+    else
+    {
+        const cyclicAMIPolyPatch& nbrPp = this->neighbPatch();
+
+        Field<Type> localFld(fld.size());
+
+        // Transform to cylindrical coords
+        {
+            const tensorField nbrT(cs().R(nbrPp.faceCentres()));
+            Foam::invTransform(localFld, nbrT, fld);
+        }
+
+        initInterpolateUntransformed
+        (
+            localFld,
+            sendRequests,
+            sendBuffers,
+            recvRequests,
+            recvBuffers
+        );
+    }
+}
+
+
+template<class Type>
+Foam::tmp<Foam::Field<Type>> Foam::cyclicAMIPolyPatch::interpolate
+(
+    const Field<Type>& localFld,
+    const labelRange& requests,
+    const PtrList<List<Type>>& recvBuffers,
+    const UList<Type>& defaultValues
+) const
+{
+    const auto& AMI = (owner() ? this->AMI() : neighbPatch().AMI());
+    const auto& map = (owner() ? AMI.tgtMap() : AMI.srcMap());
+
+    Field<Type> work;
+    if (AMI.distributed())
+    {
+        // Receive (= copy) data from buffers into work. TBD: receive directly
+        // into slices of work.
+        map.receive(requests, recvBuffers, work);
+    }
+    const Field<Type>& fld = (AMI.distributed() ? work : localFld);
+
+    auto tresult = tmp<Field<Type>>::New(this->size(), Zero);
+
+    // Note: tresult is optionally in transformed coord system
+    autoPtr<coordSystem::cylindrical> cs;
+
+    if (is_vectorspace<Type>::value)
+    {
+        cs.reset(cylindricalCS());
+    }
+
+    if (!cs)
+    {
+        AMI.weightedSum
+        (
+            owner(),
+            fld,
+            tresult.ref(),
+            defaultValues
+        );
+    }
+    else
+    {
+        const tensorField ownT(cs().R(this->faceCentres()));
+
+        Field<Type> localDeflt(defaultValues.size());
+        if (defaultValues.size() == size())
+        {
+            // Transform default values into cylindrical coords (using
+            // *this faceCentres)
+            // We get in UList (why? Copied from cyclicAMI). Convert to
+            // Field so we can use transformField routines.
+            const SubField<Type> defaultSubFld(defaultValues);
+            const Field<Type>& defaultFld(defaultSubFld);
+            Foam::invTransform(localDeflt, ownT, defaultFld);
+        }
+
+        AMI.weightedSum
+        (
+            owner(),
+            fld,
+            tresult.ref(),
+            localDeflt
+        );
+
+        // Transform back
+        Foam::transform(tresult.ref(), ownT, tresult());
+    }
+
+    return tresult;
+}
+
+
 template<class Type, class CombineOp>
 void Foam::cyclicAMIPolyPatch::interpolate
 (
@@ -168,26 +322,37 @@ void Foam::cyclicAMIPolyPatch::interpolate
 ) const
 {
     //- Commented out for now since called with non-primitives (e.g. wallPoint
-    //  from FaceCellWave) - these are missing the pTraits<Type>::rank and
+    //  from FaceCellWave) - missing Foam::transform, Foam::invTransform
-    //  Foam::transform
     /*
-    autoPtr<coordSystem::cylindrical> cs(cylindricalCS());
+    autoPtr<coordSystem::cylindrical> cs;
 
-    if (cs && pTraits<Type>::rank > 0)
+    if (is_vectorspace<Type>::value)
+    {
+        cs.reset(cylindricalCS());
+    }
+
+    if (cs)
     {
         const cyclicAMIPolyPatch& nbrPp = this->neighbPatch();
 
-        tmp<tensorField> nbrT(cs().R(nbrPp.faceCentres()));
+        // Transform to cylindrical coords
-
-        result = Foam::invTransform(nbrT, result);
-        List<Type> localDeflt(defaultValues.size());
-        if (defaultValues.size() == nbrT().size())
         {
+            const tensorField nbrT(cs().R(nbrPp.faceCentres()));
+            Foam::invTransform(result, nbrT, result);
+        }
+
+        const tensorField ownT(cs().R(this->faceCentres()));
+
+        Field<Type> localDeflt(defaultValues.size());
+        if (defaultValues.size() == size())
+        {
+            // Transform default values into cylindrical coords (using
+            // *this faceCentres)
             // We get in UList (why? Copied from cyclicAMI). Convert to
             // Field so we can use transformField routines.
             const SubField<Type> defaultSubFld(defaultValues);
             const Field<Type>& defaultFld(defaultSubFld);
-            localDeflt = Foam::invTransform(nbrT, defaultFld);
+            Foam::invTransform(localDeflt, ownT, defaultFld);
         }
 
         // Do actual AMI interpolation
@@ -213,8 +378,7 @@ void Foam::cyclicAMIPolyPatch::interpolate
         }
 
         // Transform back. Result is now at *this
-        const vectorField::subField fc(this->faceCentres());
+        Foam::transform(result, ownT, result);
-        result = Foam::transform(cs().R(fc), result);
     }
     else
     */

tutorials/basic/laplacianFoam/implicitAMI-nonblocking/0/T

@@ -0,0 +1,93 @@
+/*--------------------------------*- C++ -*----------------------------------*\
+| =========                 |                                                 |
+| \\      /  F ield         | OpenFOAM: The Open Source CFD Toolbox           |
+|  \\    /   O peration     | Version:  v2312                                 |
+|   \\  /    A nd           | Website:  www.openfoam.com                      |
+|    \\/     M anipulation  |                                                 |
+\*---------------------------------------------------------------------------*/
+FoamFile
+{
+    version     2.0;
+    format      ascii;
+    class       volScalarField;
+    object      T;
+}
+// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
+
+dimensions      [0 0 0 1 0 0 0];
+
+internalField   uniform 0;
+
+boundaryField
+{
+    cyclicAMI
+    {
+        type            cyclicAMI;
+        value           $internalField;
+    }
+
+    // solid1
+    solid1_top
+    {
+        type            uniformFixedValue;
+        uniformValue    1;
+    }
+    solid1_left
+    {
+        type            zeroGradient;
+    }
+    solid1_right
+    {
+        type            zeroGradient;
+    }
+
+    // solid2
+    solid2_left
+    {
+        type            zeroGradient;
+    }
+    solid2_right
+    {
+        type            zeroGradient;
+    }
+    solid2_bottom
+    {
+        type            uniformFixedValue;
+        uniformValue    0;
+    }
+
+
+    // solid1
+    solid1_b_top
+    {
+        type            uniformFixedValue;
+        uniformValue    1;
+    }
+    solid1_b_left
+    {
+        type            zeroGradient;
+    }
+    solid1_b_right
+    {
+        type            zeroGradient;
+    }
+
+
+    // solid2
+    solid2_b_left
+    {
+        type            zeroGradient;
+    }
+    solid2_b_right
+    {
+        type            zeroGradient;
+    }
+    solid2_b_bottom
+    {
+        type            uniformFixedValue;
+        uniformValue    0;
+    }
+}
+
+
+// ************************************************************************* //

tutorials/basic/laplacianFoam/implicitAMI-nonblocking/Allrun

@@ -0,0 +1,14 @@
+#!/bin/sh
+cd "${0%/*}" || exit                                # Run from this directory
+. ${WM_PROJECT_DIR:?}/bin/tools/RunFunctions        # Tutorial run functions
+#------------------------------------------------------------------------------
+
+# restore0Dir
+
+runApplication blockMesh
+
+runApplication decomposePar
+
+runParallel $(getApplication)
+
+#------------------------------------------------------------------------------

tutorials/basic/laplacianFoam/implicitAMI-nonblocking/constant/transportProperties

@@ -0,0 +1,21 @@
+/*--------------------------------*- C++ -*----------------------------------*\
+| =========                 |                                                 |
+| \\      /  F ield         | OpenFOAM: The Open Source CFD Toolbox           |
+|  \\    /   O peration     | Version:  v2306                                 |
+|   \\  /    A nd           | Website:  www.openfoam.com                      |
+|    \\/     M anipulation  |                                                 |
+\*---------------------------------------------------------------------------*/
+FoamFile
+{
+    version     2.0;
+    format      ascii;
+    class       dictionary;
+    location    "constant";
+    object      transportProperties;
+}
+// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
+
+DT              4e-05;
+
+
+// ************************************************************************* //

tutorials/basic/laplacianFoam/implicitAMI-nonblocking/system/blockMeshDict

@@ -0,0 +1,269 @@
+/*--------------------------------*- C++ -*----------------------------------*\
+| =========                 |                                                 |
+| \\      /  F ield         | OpenFOAM: The Open Source CFD Toolbox           |
+|  \\    /   O peration     | Version:  v2312                                 |
+|   \\  /    A nd           | Website:  www.openfoam.com                      |
+|    \\/     M anipulation  |                                                 |
+\*---------------------------------------------------------------------------*/
+FoamFile
+{
+    version     2.0;
+    format      ascii;
+    class       dictionary;
+    object      blockMeshDict;
+}
+// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
+
+scale   1;
+
+vertices
+(
+    // solid1
+    (-0.1  0.00  -0.05)
+    ( 0.1  0.00  -0.05)
+    ( 0.1  0.04  -0.05)
+    (-0.1  0.04  -0.05)
+    (-0.1  0.00   0.05)
+    ( 0.1  0.00   0.05)
+    ( 0.1  0.04   0.05)
+    (-0.1  0.04   0.05)
+
+
+    // solid2
+    (-0.1 -0.04  -0.05)
+    ( 0.1 -0.04  -0.05)
+    ( 0.1  0.00  -0.05)
+    (-0.1  0.00  -0.05)
+    (-0.1 -0.04   0.05)
+    ( 0.1 -0.04   0.05)
+    ( 0.1  0.00   0.05)
+    (-0.1  0.00   0.05)
+
+    // solid1_b
+    ( 0.2  0.00  -0.05)
+    ( 0.4  0.00  -0.05)
+    ( 0.4  0.04  -0.05)
+    ( 0.2  0.04  -0.05)
+    ( 0.2  0.00   0.05)
+    ( 0.4  0.00   0.05)
+    ( 0.4  0.04   0.05)
+    ( 0.2  0.04   0.05)
+
+
+    // solid2_b
+    ( 0.2 -0.04  -0.05)
+    ( 0.4 -0.04  -0.05)
+    ( 0.4  0.00  -0.05)
+    ( 0.2  0.00  -0.05)
+    ( 0.2 -0.04   0.05)
+    ( 0.4 -0.04   0.05)
+    ( 0.4  0.00   0.05)
+    ( 0.2  0.00   0.05)
+);
+
+blocks
+(
+    hex (0 1 2 3 4 5 6 7) solid1 (5 10 1) simpleGrading (1 1 1)
+    hex (8 9 10 11 12 13 14 15) solid2 (4 10 1) simpleGrading (1 1 1)
+
+    hex (16 17 18 19 20 21 22 23) solid1_b (3 7 1) simpleGrading (1 1 1)
+    hex (24 25 26 27 28 29 30 31) solid2_b (7 7 1) simpleGrading (1 1 1)
+);
+
+edges
+(
+);
+
+boundary
+(
+    solid1_bottom
+    {
+        type            cyclicAMI;
+        inGroups        (coupleGroup);
+        coupleGroup     coupleGroup;
+        faces
+        (
+            (1 5 4 0)
+        );
+    }
+    solid1_top
+    {
+        type wall;
+        faces
+        (
+            (3 7 6 2)
+        );
+    }
+    solid1_left
+    {
+        type wall;
+        faces
+        (
+            (0 4 7 3)
+        );
+    }
+    solid1_right
+    {
+        type wall;
+        faces
+        (
+            (2 6 5 1)
+        );
+    }
+    solid1_frontAndBack
+    {
+        type empty;
+        faces
+        (
+            (0 3 2 1)
+            (4 5 6 7)
+        );
+    }
+
+    solid2_bottom
+    {
+        type wall;
+        faces
+        (
+            (9 13 12 8)
+        );
+    }
+    solid2_top
+    {
+        type            cyclicAMI;
+        inGroups        (coupleGroup);
+        coupleGroup     coupleGroup;
+
+        faces
+        (
+            (11 15 14 10)
+        );
+    }
+    solid2_left
+    {
+        type wall;
+        faces
+        (
+            (8 12 15 11)
+        );
+    }
+    solid2_right
+    {
+        type wall;
+        faces
+        (
+            (10 14 13 9)
+        );
+    }
+    
+    solid2_frontAndBack
+    {
+        type empty;
+        faces
+        (
+            (8 11 10 9)
+            (12 13 14 15)
+        );
+    }
+
+
+    // block b
+    // ~~~~~~~
+
+    solid1_b_bottom
+    {
+        type            cyclicAMI;
+        inGroups        (coupleGroup_b);
+        coupleGroup     coupleGroup_b;
+        faces
+        (
+            (17 21 20 16)
+        );
+    }
+    solid1_b_top
+    {
+        type wall;
+        faces
+        (
+            (19 23 22 18)
+        );
+    }
+    solid1_b_left
+    {
+        type wall;
+        faces
+        (
+            (16 20 23 19)
+        );
+    }
+    solid1_b_right
+    {
+        type wall;
+        faces
+        (
+            (18 22 21 17)
+        );
+    }
+    solid1_b_frontAndBack
+    {
+        type empty;
+        faces
+        (
+            (16 19 18 17)
+            (20 21 22 23)
+        );
+    }
+
+    solid2_b_bottom
+    {
+        type wall;
+        faces
+        (
+            //(9 13 12 8)
+            (25 29 28 24)
+        );
+    }
+    solid2_b_top
+    {
+        type            cyclicAMI;
+        inGroups        (coupleGroup_b);
+        coupleGroup     coupleGroup_b;
+
+        faces
+        (
+            //(11 15 14 10)
+            (27 31 30 26)
+        );
+    }
+    solid2_b_left
+    {
+        type wall;
+        faces
+        (
+            //(8 12 15 11)
+            (24 28 31 27)
+        );
+    }
+    solid2_b_right
+    {
+        type wall;
+        faces
+        (
+            //(10 14 13 9)
+            (26 30 29 25)
+        );
+    }
+    solid2_b_frontAndBack
+    {
+        type empty;
+        faces
+        (
+            //(8 11 10 9)
+            (24 27 26 25)
+            //(12 13 14 15)
+            (28 29 30 31)
+        );
+    }
+);
+
+// ************************************************************************* //

tutorials/basic/laplacianFoam/implicitAMI-nonblocking/system/controlDict

@@ -0,0 +1,50 @@
+/*--------------------------------*- C++ -*----------------------------------*\
+| =========                 |                                                 |
+| \\      /  F ield         | OpenFOAM: The Open Source CFD Toolbox           |
+|  \\    /   O peration     | Version:  v2312                                 |
+|   \\  /    A nd           | Website:  www.openfoam.com                      |
+|    \\/     M anipulation  |                                                 |
+\*---------------------------------------------------------------------------*/
+FoamFile
+{
+    version     2.0;
+    format      ascii;
+    class       dictionary;
+    location    "system";
+    object      controlDict;
+}
+// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
+
+application     laplacianFoam;
+
+startFrom       latestTime;
+
+startTime       0;
+
+stopAt          endTime;
+
+endTime         2;
+
+deltaT          1;
+
+//writeControl    runTime;
+//writeInterval   0.1;
+writeControl    timeStep;
+writeInterval   1;
+
+purgeWrite      0;
+
+writeFormat     ascii;
+
+writePrecision  6;
+
+writeCompression off;
+
+timeFormat      general;
+
+timePrecision   6;
+
+runTimeModifiable true;
+
+
+// ************************************************************************* //

tutorials/basic/laplacianFoam/implicitAMI-nonblocking/system/decomposeParDict

@@ -0,0 +1,26 @@
+/*--------------------------------*- C++ -*----------------------------------*\
+| =========                 |                                                 |
+| \\      /  F ield         | OpenFOAM: The Open Source CFD Toolbox           |
+|  \\    /   O peration     | Version:  v2312                                 |
+|   \\  /    A nd           | Website:  www.openfoam.com                      |
+|    \\/     M anipulation  |                                                 |
+\*---------------------------------------------------------------------------*/
+FoamFile
+{
+    version     2.0;
+    format      ascii;
+    class       dictionary;
+    object      decomposeParDict;
+}
+// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
+
+//- The total number of domains (mandatory)
+numberOfSubdomains  4;
+
+//- The decomposition method (mandatory)
+method          hierarchical;
+n               (4 1 1);
+
+// NOTE: no patch constraints!
+
+// ************************************************************************* //

tutorials/basic/laplacianFoam/implicitAMI-nonblocking/system/fvSchemes

@@ -0,0 +1,52 @@
+/*--------------------------------*- C++ -*----------------------------------*\
+| =========                 |                                                 |
+| \\      /  F ield         | OpenFOAM: The Open Source CFD Toolbox           |
+|  \\    /   O peration     | Version:  v2312                                 |
+|   \\  /    A nd           | Website:  www.openfoam.com                      |
+|    \\/     M anipulation  |                                                 |
+\*---------------------------------------------------------------------------*/
+FoamFile
+{
+    version     2.0;
+    format      ascii;
+    class       dictionary;
+    location    "system";
+    object      fvSchemes;
+}
+// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
+
+ddtSchemes
+{
+    default         steadyState;    //Euler;
+}
+
+gradSchemes
+{
+    default         Gauss linear;
+    grad(T)         Gauss linear;
+}
+
+divSchemes
+{
+    default         none;
+}
+
+laplacianSchemes
+{
+    default         none;
+    laplacian(DT,T) Gauss linear corrected;
+    laplacian(DTV,T) Gauss linear corrected;
+}
+
+interpolationSchemes
+{
+    default         linear;
+}
+
+snGradSchemes
+{
+    default         corrected;
+}
+
+
+// ************************************************************************* //

tutorials/basic/laplacianFoam/implicitAMI-nonblocking/system/fvSolution

@@ -0,0 +1,35 @@
+/*--------------------------------*- C++ -*----------------------------------*\
+| =========                 |                                                 |
+| \\      /  F ield         | OpenFOAM: The Open Source CFD Toolbox           |
+|  \\    /   O peration     | Version:  v2312                                 |
+|   \\  /    A nd           | Website:  www.openfoam.com                      |
+|    \\/     M anipulation  |                                                 |
+\*---------------------------------------------------------------------------*/
+FoamFile
+{
+    version     2.0;
+    format      ascii;
+    class       dictionary;
+    location    "system";
+    object      fvSolution;
+}
+// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
+
+solvers
+{
+    T
+    {
+        solver          PCG;
+        preconditioner  DIC;
+        tolerance       1e-06;
+        relTol          0;
+    }
+}
+
+SIMPLE
+{
+    nNonOrthogonalCorrectors 2;
+}
+
+
+// ************************************************************************* //