Merge branch 'issue-3238-nbx-failure' into 'develop'

ENH: more unique tags to avoid overlapping comms

See merge request Development/openfoam!704

src/OpenFOAM/db/IOstreams/Pstreams/PstreamBuffers.C

@@ -45,6 +45,11 @@ registerOptSwitch
     Foam::PstreamBuffers::algorithm
 );
 
+namespace Foam
+{
+    defineTypeNameAndDebug(PstreamBuffers, 0);
+}
+
 
 // * * * * * * * * * * * * * Private Member Functions  * * * * * * * * * * * //
 
@@ -231,6 +236,12 @@ void Foam::PstreamBuffers::finalExchange
     labelList& recvSizes
 )
 {
+    DebugPoutInFunction
+        << "tag:" << tag_
+        << " comm:" << comm_
+        << " nProcs:" << nProcs_
+        << endl;
+
     initFinalExchange();
 
     if (commsType_ == UPstream::commsTypes::nonBlocking)
@@ -303,13 +314,25 @@ Foam::PstreamBuffers::PstreamBuffers
     sendBuffers_(nProcs_),
     recvBuffers_(nProcs_),
     recvPositions_(nProcs_, Zero)
-{}
+{
+    DebugPoutInFunction
+        << "tag:" << tag_
+        << " comm:" << comm_
+        << " nProcs:" << nProcs_
+        << endl;
+}
 
 
 // * * * * * * * * * * * * * * * * Destructor  * * * * * * * * * * * * * * * //
 
 Foam::PstreamBuffers::~PstreamBuffers()
 {
+    DebugPoutInFunction
+        << "tag:" << tag_
+        << " comm:" << comm_
+        << " nProcs:" << nProcs_
+        << endl;
+
     // Check that all data has been consumed.
     forAll(recvBuffers_, proci)
     {
@@ -321,6 +344,7 @@ Foam::PstreamBuffers::~PstreamBuffers()
             FatalErrorInFunction
                 << "Message from processor " << proci
                 << " Only consumed " << pos << " of " << len << " bytes" << nl
+                << " comm " << comm_ << " tag " << tag_ << nl
                 << Foam::abort(FatalError);
         }
     }
@@ -623,6 +647,12 @@ Foam::PstreamBuffers::peekRecvData(const label proci) const
 
 void Foam::PstreamBuffers::finishedSends(const bool wait)
 {
+    DebugPoutInFunction
+        << "tag:" << tag_
+        << " comm:" << comm_
+        << " nProcs:" << nProcs_
+        << endl;
+
     labelList recvSizes;
     finalExchange(modeOption::DEFAULT, wait, recvSizes);
 }
@@ -630,6 +660,12 @@ void Foam::PstreamBuffers::finishedSends(const bool wait)
 
 void Foam::PstreamBuffers::finishedSendsNBX(const bool wait)
 {
+    DebugPoutInFunction
+        << "tag:" << tag_
+        << " comm:" << comm_
+        << " nProcs:" << nProcs_
+        << endl;
+
     labelList recvSizes;
     finalExchange(modeOption::NBX_PEX, wait, recvSizes);
 }
@@ -641,6 +677,12 @@ void Foam::PstreamBuffers::finishedSends
     const bool wait
 )
 {
+    DebugPoutInFunction
+        << "tag:" << tag_
+        << " comm:" << comm_
+        << " nProcs:" << nProcs_
+        << endl;
+
     // Resize for copying back
     recvSizes.resize_nocopy(sendBuffers_.size());
 

src/OpenFOAM/db/IOstreams/Pstreams/PstreamBuffers.H

@@ -248,6 +248,10 @@ class PstreamBuffers
 
 public:
 
+    // Declare name of the class and its debug switch
+    ClassName("PstreamBuffers");
+
+
     // Static Data
 
         //- Preferred exchange algorithm (may change or be removed in future)

src/OpenFOAM/meshes/polyMesh/globalMeshData/globalMeshData.C

@@ -88,6 +88,9 @@ void Foam::globalMeshData::initProcAddr()
 
     if (UPstream::parRun())
     {
+        // Allocate unique tag for all comms
+        const int oldTag = UPstream::incrMsgType();
+
         PstreamBuffers pBufs(mesh_.comm());
 
         // Send indices of my processor patches to my neighbours
@@ -120,6 +123,9 @@ void Foam::globalMeshData::initProcAddr()
 
             fromNeighbour >> processorPatchNeighbours_[patchi];
         }
+
+        // Reset tag
+        UPstream::msgType(oldTag);
     }
 }
 
@@ -137,6 +143,9 @@ void Foam::globalMeshData::calcSharedPoints() const
             << "Shared point addressing already done" << abort(FatalError);
     }
 
+    // Allocate unique tag for all comms
+    const int oldTag = UPstream::incrMsgType();
+
     // Calculate all shared points (exclude points that are only
     // on two coupled patches). This does all the hard work.
     const globalPoints parallelPoints(mesh_, false, true);
@@ -229,6 +238,9 @@ void Foam::globalMeshData::calcSharedPoints() const
         }
     }
 
+    // Reset tag
+    UPstream::msgType(oldTag);
+
     if (debug)
     {
         Pout<< "globalMeshData : nGlobalPoints_:" << nGlobalPoints_ << nl
@@ -356,6 +368,9 @@ void Foam::globalMeshData::calcSharedEdges() const
 
     EdgeMap<label> globalShared(2*nGlobalPoints());
 
+    // Allocate unique tag for all comms
+    const int oldTag = UPstream::incrMsgType();
+
     if (UPstream::master())
     {
         label sharedEdgeI = 0;
@@ -466,6 +481,9 @@ void Foam::globalMeshData::calcSharedEdges() const
         new labelList(std::move(dynSharedEdgeAddr))
     );
 
+    // Reset tag
+    UPstream::msgType(oldTag);
+
     if (debug)
     {
         Pout<< "globalMeshData : nGlobalEdges_:" << nGlobalEdges_ << nl
@@ -486,6 +504,9 @@ void Foam::globalMeshData::calcGlobalPointSlaves() const
             << endl;
     }
 
+    // Allocate unique tag for all comms
+    const int oldTag = UPstream::incrMsgType();
+
     // Calculate connected points for master points.
     globalPoints globalData(mesh_, coupledPatch(), true, true);
 
@@ -511,6 +532,9 @@ void Foam::globalMeshData::calcGlobalPointSlaves() const
             std::move(globalData.map())
         )
     );
+
+    // Reset tag
+    UPstream::msgType(oldTag);
 }
 
 
@@ -522,10 +546,14 @@ void Foam::globalMeshData::calcPointConnectivity
     const globalIndexAndTransform& transforms = globalTransforms();
     const labelListList& slaves = globalPointSlaves();
     const labelListList& transformedSlaves = globalPointTransformedSlaves();
+    const auto& slavesMap = globalPointSlavesMap();
 
 
+    // Allocate unique tag for all comms
+    const int oldTag = UPstream::incrMsgType();
+
     // Create field with my local data
-    labelPairList myData(globalPointSlavesMap().constructSize());
+    labelPairList myData(slavesMap.constructSize());
     forAll(slaves, pointi)
     {
         myData[pointi] = transforms.encode
@@ -536,11 +564,11 @@ void Foam::globalMeshData::calcPointConnectivity
         );
     }
     // Send to master
-    globalPointSlavesMap().distribute(myData);
+    slavesMap.distribute(myData, true, UPstream::msgType());
 
 
     // String of connected points with their transform
-    allPointConnectivity.setSize(globalPointSlavesMap().constructSize());
+    allPointConnectivity.setSize(slavesMap.constructSize());
     allPointConnectivity = labelPairList(0);
 
     // Pass1: do the master points since these also update local slaves
@@ -569,7 +597,7 @@ void Foam::globalMeshData::calcPointConnectivity
             forAll(pTransformSlaves, i)
             {
                 // Get transform from index
-                label transformI = globalPointSlavesMap().whichTransform
+                label transformI = slavesMap.whichTransform
                 (
                     pTransformSlaves[i]
                 );
@@ -610,11 +638,16 @@ void Foam::globalMeshData::calcPointConnectivity
     }
 
 
-    globalPointSlavesMap().reverseDistribute
+    slavesMap.reverseDistribute
     (
         slaves.size(),
-        allPointConnectivity
+        allPointConnectivity,
+        true,
+        UPstream::msgType()
     );
+
+    // Reset tag
+    UPstream::msgType(oldTag);
 }
 
 
@@ -655,8 +688,25 @@ void Foam::globalMeshData::calcGlobalPointEdges
     }
 
     // Pull slave data to master. Dummy transform.
-    globalPointSlavesMap().distribute(globalPointEdges);
+
-    globalPointSlavesMap().distribute(globalPointPoints);
+    // Allocate unique tag for all comms
+    const int oldTag = UPstream::incrMsgType();
+
+    globalPointSlavesMap().distribute
+    (
+        globalPointEdges,
+        true,
+        UPstream::msgType()
+    );
+    // Make sure second send uses 'far' away tags in case of NBX deciding on
+    // multi-pass spraying of messages with consecutive tags
+    globalPointSlavesMap().distribute
+    (
+        globalPointPoints,
+        true,
+        UPstream::msgType()+23456    // Unique, far enough away tag
+    );
+
     // Add all pointEdges
     forAll(slaves, pointi)
     {
@@ -764,14 +814,21 @@ void Foam::globalMeshData::calcGlobalPointEdges
     globalPointSlavesMap().reverseDistribute
     (
         slaves.size(),
-        globalPointEdges
+        globalPointEdges,
+        true,
+        UPstream::msgType()
     );
     // Push back
     globalPointSlavesMap().reverseDistribute
     (
         slaves.size(),
-        globalPointPoints
+        globalPointPoints,
+        true,
+        UPstream::msgType()+65432   // Unique, far enough away tag
     );
+
+    // Reset tag
+    UPstream::msgType(oldTag);
 }
 
 
@@ -852,6 +909,8 @@ void Foam::globalMeshData::calcGlobalEdgeSlaves() const
     // need to make sure that all points know about connectivity and
     // the transformations.
 
+    // Allocate unique tag for all comms
+    const int oldTag = UPstream::incrMsgType();
 
     // 1. collect point connectivity - basically recreating globalPoints output.
     // All points will now have a string of coupled points. The transforms are
@@ -1037,10 +1096,13 @@ void Foam::globalMeshData::calcGlobalEdgeSlaves() const
             transformedEdges,
             globalEdgeTransformedSlavesPtr_(),
 
-            compactMap
+            compactMap,
+            UPstream::msgType()
         )
     );
 
+    // Reset tag
+    UPstream::msgType(oldTag);
 
     if (debug)
     {
@@ -1061,6 +1123,9 @@ void Foam::globalMeshData::calcGlobalEdgeOrientation() const
             << " calculating edge orientation w.r.t. master edge." << endl;
     }
 
+    // Allocate unique tag for all comms
+    const int oldTag = UPstream::incrMsgType();
+
     const globalIndex& globalPoints = globalPointNumbering();
 
     // 1. Determine master point
@@ -1172,6 +1237,9 @@ void Foam::globalMeshData::calcGlobalEdgeOrientation() const
         }
     }
 
+    // Reset tag
+    UPstream::msgType(oldTag);
+
     if (debug)
     {
         Pout<< "globalMeshData::calcGlobalEdgeOrientation() :"
@@ -1262,6 +1330,9 @@ void Foam::globalMeshData::calcGlobalPointBoundaryFaces() const
 
     const label myProci = UPstream::myProcNo();
 
+    // Allocate unique tag for all comms
+    const int oldTag = UPstream::incrMsgType();
+
     // Construct local point to (uncoupled)boundaryfaces.
     labelListList pointBoundaryFaces;
     calcPointBoundaryFaces(pointBoundaryFaces);
@@ -1429,6 +1500,9 @@ void Foam::globalMeshData::calcGlobalPointBoundaryFaces() const
     globalPointBoundaryFaces.setSize(coupledPatch().nPoints());
     globalPointTransformedBoundaryFacesPtr_().setSize(coupledPatch().nPoints());
 
+    // Reset tag
+    UPstream::msgType(oldTag);
+
     if (debug)
     {
         Pout<< "globalMeshData::calcGlobalPointBoundaryFaces() :"
@@ -1498,6 +1572,10 @@ void Foam::globalMeshData::calcGlobalPointBoundaryCells() const
     // Convert point-cells to global (boundary)cell numbers
     // ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
+
+    // Allocate unique tag for all comms
+    const int oldTag = UPstream::incrMsgType();
+
     globalBoundaryCellNumberingPtr_.reset
     (
         new globalIndex(boundaryCells.size())
@@ -1654,6 +1732,9 @@ void Foam::globalMeshData::calcGlobalPointBoundaryCells() const
     globalPointBoundaryCells.setSize(coupledPatch().nPoints());
     globalPointTransformedBoundaryCellsPtr_().setSize(coupledPatch().nPoints());
 
+    // Reset tag
+    UPstream::msgType(oldTag);
+
     if (debug)
     {
         Pout<< "globalMeshData::calcGlobalPointBoundaryCells() :"
@@ -1676,6 +1757,9 @@ void Foam::globalMeshData::calcGlobalCoPointSlaves() const
             << " slave point addressing." << endl;
     }
 
+    // Allocate unique tag for all comms
+    const int oldTag = UPstream::incrMsgType();
+
     // Calculate connected points for master points.
     globalPoints globalData(mesh_, coupledPatch(), true, false);
 
@@ -1694,6 +1778,8 @@ void Foam::globalMeshData::calcGlobalCoPointSlaves() const
         )
     );
 
+    UPstream::msgType(oldTag);
+
     if (debug)
     {
         Pout<< "globalMeshData::calcGlobalCoPointSlaves() :"
@@ -1850,6 +1936,9 @@ Foam::pointField Foam::globalMeshData::sharedPoints() const
     // Get all processors to send their shared points to master.
     // (not very efficient)
 
+    // Allocate unique tag for all comms
+    const int oldTag = UPstream::incrMsgType();
+
     pointField sharedPoints(nGlobalPoints());
     const labelList& pointAddr = sharedPointAddr();
     const labelList& pointLabels = sharedPointLabels();
@@ -1901,6 +1990,8 @@ Foam::pointField Foam::globalMeshData::sharedPoints() const
     // Broadcast: sharedPoints to all
     Pstream::broadcast(sharedPoints);  // == worldComm
 
+    // Reset tag
+    UPstream::msgType(oldTag);
 
     return sharedPoints;
 }
@@ -1908,6 +1999,9 @@ Foam::pointField Foam::globalMeshData::sharedPoints() const
 
 Foam::pointField Foam::globalMeshData::geometricSharedPoints() const
 {
+    // Allocate unique tag for all comms
+    const int oldTag = UPstream::incrMsgType();
+
     // Get coords of my shared points
     pointField sharedPoints(mesh_.points(), sharedPointLabels());
 
@@ -1926,6 +2020,9 @@ Foam::pointField Foam::globalMeshData::geometricSharedPoints() const
         pointMap
     );
 
+    // Reset tag
+    UPstream::msgType(oldTag);
+
     return sharedPoints;
 }
 
@@ -2091,10 +2188,16 @@ const Foam::globalIndex& Foam::globalMeshData::globalPointNumbering() const
 {
     if (!globalPointNumberingPtr_)
     {
+        // Allocate unique tag for all comms
+        const int oldTag = UPstream::incrMsgType();
+
         globalPointNumberingPtr_.reset
         (
             new globalIndex(coupledPatch().nPoints())
         );
+
+        // Reset tag
+        UPstream::msgType(oldTag);
     }
     return *globalPointNumberingPtr_;
 }
@@ -2105,7 +2208,13 @@ Foam::globalMeshData::globalTransforms() const
 {
     if (!globalTransformsPtr_)
     {
+        // Allocate unique tag for all comms
+        const int oldTag = UPstream::incrMsgType();
+
         globalTransformsPtr_.reset(new globalIndexAndTransform(mesh_));
+
+        // Reset tag
+        UPstream::msgType(oldTag);
     }
     return *globalTransformsPtr_;
 }
@@ -2146,10 +2255,16 @@ const Foam::globalIndex& Foam::globalMeshData::globalEdgeNumbering() const
 {
     if (!globalEdgeNumberingPtr_)
     {
+        // Allocate unique tag for all comms
+        const int oldTag = UPstream::incrMsgType();
+
         globalEdgeNumberingPtr_.reset
         (
             new globalIndex(coupledPatch().nEdges())
         );
+
+        // Reset tag
+        UPstream::msgType(oldTag);
     }
     return *globalEdgeNumberingPtr_;
 }
@@ -2326,6 +2441,8 @@ Foam::autoPtr<Foam::globalIndex> Foam::globalMeshData::mergePoints
     const labelListList& pointSlaves = globalCoPointSlaves();
     const mapDistribute& pointSlavesMap = globalCoPointSlavesMap();
 
+    // Allocate unique tag for all comms
+    const int oldTag = UPstream::incrMsgType();
 
     // Points are either
     // - master with slaves
@@ -2444,6 +2561,9 @@ Foam::autoPtr<Foam::globalIndex> Foam::globalMeshData::mergePoints
         }
     }
 
+    // Restore tag
+    UPstream::msgType(oldTag);
+
     return globalPointsPtr;
 }
 
@@ -2471,6 +2591,9 @@ Foam::autoPtr<Foam::globalIndex> Foam::globalMeshData::mergePoints
     // patch point via coupled patches).
 
 
+    // Allocate unique tag for all comms
+    const int oldTag = UPstream::incrMsgType();
+
     // Determine mapping:
     // - from patch point to coupled point (or -1)
     // - from coupled point to global patch point
@@ -2659,6 +2782,9 @@ Foam::autoPtr<Foam::globalIndex> Foam::globalMeshData::mergePoints
         }
     }
 
+    // Restore tag
+    UPstream::msgType(oldTag);
+
     return globalPointsPtr;
 }
 

src/OpenFOAM/meshes/polyMesh/globalMeshData/globalMeshDataTemplates.C

@@ -6,7 +6,7 @@
      \\/     M anipulation  |
 -------------------------------------------------------------------------------
     Copyright (C) 2011-2015 OpenFOAM Foundation
-    Copyright (C) 2018-2019 OpenCFD Ltd.
+    Copyright (C) 2018-2024 OpenCFD Ltd.
 -------------------------------------------------------------------------------
 License
     This file is part of OpenFOAM.
@@ -44,6 +44,9 @@ void Foam::globalMeshData::syncData
     const TransformOp& top
 )
 {
+    // Allocate unique tag for all comms
+    const int oldTag = UPstream::incrMsgType();
+
     // Pull slave data onto master
     slavesMap.distribute(transforms, elems, top);
 
@@ -94,6 +97,9 @@ void Foam::globalMeshData::syncData
         elems,
         top
     );
+
+    // Reset tag
+    UPstream::msgType(oldTag);
 }
 
 
@@ -107,6 +113,9 @@ void Foam::globalMeshData::syncData
     const CombineOp& cop
 )
 {
+    // Allocate unique tag for all comms
+    const int oldTag = UPstream::incrMsgType();
+
     // Pull slave data onto master
     slavesMap.distribute(elems);
 
@@ -151,6 +160,9 @@ void Foam::globalMeshData::syncData
 
     // Push slave-slot data back to slaves
     slavesMap.reverseDistribute(elems.size(), elems);
+
+    // Reset tag
+    UPstream::msgType(oldTag);
 }
 
 

src/OpenFOAM/meshes/polyMesh/polyBoundaryMesh/polyBoundaryMesh.C

@@ -300,6 +300,9 @@ void Foam::polyBoundaryMesh::clearAddressing()
 
 void Foam::polyBoundaryMesh::calcGeometry()
 {
+    // Make sure messages don't interact by having unique tag
+    const int oldTag = UPstream::incrMsgType();
+
     PstreamBuffers pBufs(Pstream::defaultCommsType);
 
     if
@@ -341,6 +344,9 @@ void Foam::polyBoundaryMesh::calcGeometry()
             }
         }
     }
+
+    // Reset tag
+    UPstream::msgType(oldTag);
 }
 
 

src/OpenFOAM/meshes/polyMesh/syncTools/syncToolsTemplates.C

@@ -92,6 +92,9 @@ void Foam::syncTools::syncPointMap
 {
     const polyBoundaryMesh& patches = mesh.boundaryMesh();
 
+    // Make sure we use unique message tag
+    const int oldTag = UPstream::incrMsgType();
+
     // Synchronize multiple shared points.
     const globalMeshData& pd = mesh.globalData();
 
@@ -369,6 +372,9 @@ void Foam::syncTools::syncPointMap
             }
         }
     }
+
+    // Reset tag
+    UPstream::msgType(oldTag);
 }
 
 
@@ -387,6 +393,8 @@ void Foam::syncTools::syncEdgeMap
     // Do synchronisation without constructing globalEdge addressing
     // (since this constructs mesh edge addressing)
 
+    // Make sure we use unique message tag
+    const int oldTag = UPstream::incrMsgType();
 
     // Swap proc patch info
     // ~~~~~~~~~~~~~~~~~~~~
@@ -734,6 +742,9 @@ void Foam::syncTools::syncEdgeMap
             );
         }
     }
+
+    // Reset tag
+    UPstream::msgType(oldTag);
 }
 
 
@@ -1037,6 +1048,9 @@ void Foam::syncTools::syncBoundaryFaceList
 
     const polyBoundaryMesh& patches = mesh.boundaryMesh();
 
+    // Allocate unique tag for all comms
+    const int oldTag = UPstream::incrMsgType();
+
     if (parRun && UPstream::parRun())
     {
         // Avoid mesh.globalData() - possible race condition
@@ -1252,6 +1266,9 @@ void Foam::syncTools::syncBoundaryFaceList
             }
         }
     }
+
+    // Reset tag
+    UPstream::msgType(oldTag);
 }
 
 
@@ -1283,6 +1300,9 @@ void Foam::syncTools::syncFaceList
 
     const polyBoundaryMesh& patches = mesh.boundaryMesh();
 
+    // Allocate unique tag for all comms
+    const int oldTag = UPstream::incrMsgType();
+
     if (parRun && UPstream::parRun())
     {
         const label startRequest = UPstream::nRequests();
@@ -1403,6 +1423,9 @@ void Foam::syncTools::syncFaceList
             }
         }
     }
+
+    // Reset tag
+    UPstream::msgType(oldTag);
 }
 
 

src/finiteVolume/fields/fvPatchFields/constraint/cyclicACMI/cyclicACMIFvPatchField.C

@@ -611,6 +611,8 @@ void Foam::cyclicACMIFvPatchField<Type>::initInterfaceMatrixUpdate
             scalarRecvBufs_
         );
     }
+
+    this->updatedMatrix(false);
 }
 
 
@@ -679,6 +681,8 @@ void Foam::cyclicACMIFvPatchField<Type>::updateInterfaceMatrix
     }
 
     this->addToInternalField(result, !add, faceCells, coeffs, pnf);
+
+    this->updatedMatrix(true);
 }
 
 
@@ -735,6 +739,8 @@ void Foam::cyclicACMIFvPatchField<Type>::initInterfaceMatrixUpdate
             recvBufs_
         );
     }
+
+    this->updatedMatrix(false);
 }
 
 
@@ -792,6 +798,8 @@ void Foam::cyclicACMIFvPatchField<Type>::updateInterfaceMatrix
     }
 
     this->addToInternalField(result, !add, faceCells, coeffs, pnf);
+
+    this->updatedMatrix(true);
 }
 
 

src/finiteVolume/fields/fvPatchFields/constraint/cyclicAMI/cyclicAMIFvPatchField.C

@@ -616,6 +616,8 @@ void Foam::cyclicAMIFvPatchField<Type>::initInterfaceMatrixUpdate
             scalarRecvBufs_
         );
     }
+
+    this->updatedMatrix(false);
 }
 
 
@@ -693,6 +695,8 @@ void Foam::cyclicAMIFvPatchField<Type>::updateInterfaceMatrix
 
     // Multiply the field by coefficients and add into the result
     this->addToInternalField(result, !add, faceCells, coeffs, pnf);
+
+    this->updatedMatrix(true);
 }
 
 
@@ -751,6 +755,8 @@ void Foam::cyclicAMIFvPatchField<Type>::initInterfaceMatrixUpdate
             recvBufs_
         );
     }
+
+    this->updatedMatrix(false);
 }
 
 
@@ -827,6 +833,8 @@ void Foam::cyclicAMIFvPatchField<Type>::updateInterfaceMatrix
 
     // Multiply the field by coefficients and add into the result
     this->addToInternalField(result, !add, faceCells, coeffs, pnf);
+
+    this->updatedMatrix(true);
 }
 
 

src/meshTools/AMIInterpolation/AMIInterpolation/AMIInterpolationTemplates.C

@@ -145,7 +145,8 @@ void Foam::AMIInterpolation::interpolateToTarget
     if (distributed())
     {
         const mapDistribute& map = srcMapPtr_();
-        work = fld;  // deep copy
+        work.resize_nocopy(map.constructSize());
+        SubList<Type>(work, fld.size()) = fld;  // deep copy
         map.distribute(work);
     }
 
@@ -205,7 +206,8 @@ void Foam::AMIInterpolation::interpolateToSource
     if (distributed())
     {
         const mapDistribute& map = tgtMapPtr_();
-        work = fld;  // deep copy
+        work.resize_nocopy(map.constructSize());
+        SubList<Type>(work, fld.size()) = fld;  // deep copy
         map.distribute(work);
     }
 

src/meshTools/AMIInterpolation/AMIInterpolation/faceAreaWeightAMI/faceAreaWeightAMI.C

@@ -6,7 +6,7 @@
      \\/     M anipulation  |
 -------------------------------------------------------------------------------
     Copyright (C) 2013-2016 OpenFOAM Foundation
-    Copyright (C) 2018-2021 OpenCFD Ltd.
+    Copyright (C) 2018-2024 OpenCFD Ltd.
 -------------------------------------------------------------------------------
 License
     This file is part of OpenFOAM.
@@ -714,6 +714,8 @@ bool Foam::faceAreaWeightAMI::calculate
     if (distributed())
     {
         const label myRank = UPstream::myProcNo(comm_);
+        // Allocate unique tag for all comms
+        const int oldTag = UPstream::incrMsgType();
 
         const primitivePatch& srcPatch0 = this->srcPatch0();
         const primitivePatch& tgtPatch0 = this->tgtPatch0();
@@ -751,7 +753,7 @@ bool Foam::faceAreaWeightAMI::calculate
             labelList(),
             ListOps::appendEqOp<label>(),
             flipOp(),                   // flip operation
-            UPstream::msgType(),
+            UPstream::msgType()+77431,
             comm_
         );
 
@@ -768,7 +770,7 @@ bool Foam::faceAreaWeightAMI::calculate
             scalarList(),
             ListOps::appendEqOp<scalar>(),
             flipOp(),
-            UPstream::msgType(),
+            UPstream::msgType()+77432,
             comm_
         );
 
@@ -784,7 +786,7 @@ bool Foam::faceAreaWeightAMI::calculate
                 globalSrcFaces,
                 tgtAddress_,
                 cMapSrc,
-                UPstream::msgType(),
+                UPstream::msgType()+77433,
                 comm_
             )
         );
@@ -797,10 +799,13 @@ bool Foam::faceAreaWeightAMI::calculate
                 globalTgtFaces,
                 srcAddress_,
                 cMapTgt,
-                UPstream::msgType(),
+                UPstream::msgType()+77434,
                 comm_
             )
         );
+
+        // Reset tag
+        UPstream::msgType(oldTag);
     }
 
     // Convert the weights from areas to normalised values

src/meshTools/AMIInterpolation/AMIInterpolation/faceAreaWeightAMI2D/faceAreaWeightAMI2D.C

@@ -5,7 +5,7 @@
     \\  /    A nd           | www.openfoam.com
      \\/     M anipulation  |
 -------------------------------------------------------------------------------
-    Copyright (C) 2020,2022 OpenCFD Ltd.
+    Copyright (C) 2020,2022,2024 OpenCFD Ltd.
 -------------------------------------------------------------------------------
 License
     This file is part of OpenFOAM.
@@ -428,6 +428,8 @@ bool Foam::faceAreaWeightAMI2D::calculate
     if (distributed())
     {
         const label myRank = UPstream::myProcNo(comm_);
+        // Allocate unique tag for all comms
+        const int oldTag = UPstream::incrMsgType();
 
         const primitivePatch& srcPatch0 = this->srcPatch0();
         const primitivePatch& tgtPatch0 = this->tgtPatch0();
@@ -465,7 +467,7 @@ bool Foam::faceAreaWeightAMI2D::calculate
             labelList(),
             ListOps::appendEqOp<label>(),
             flipOp(),                   // flip operation
-            UPstream::msgType(),
+            UPstream::msgType()+77431,
             comm_
         );
 
@@ -482,7 +484,7 @@ bool Foam::faceAreaWeightAMI2D::calculate
             scalarList(),
             ListOps::appendEqOp<scalar>(),
             flipOp(),                   // flip operation
-            UPstream::msgType(),
+            UPstream::msgType()+77432,
             comm_
         );
 
@@ -498,7 +500,7 @@ bool Foam::faceAreaWeightAMI2D::calculate
                 globalSrcFaces,
                 tgtAddress_,
                 cMapSrc,
-                UPstream::msgType(),
+                UPstream::msgType()+77433,
                 comm_
             )
         );
@@ -511,10 +513,13 @@ bool Foam::faceAreaWeightAMI2D::calculate
                 globalTgtFaces,
                 srcAddress_,
                 cMapTgt,
-                UPstream::msgType(),
+                UPstream::msgType()+77434,
                 comm_
             )
         );
+
+        // Reset tag
+        UPstream::msgType(oldTag);
     }
 
     // Convert the weights from areas to normalised values

src/meshTools/AMIInterpolation/GAMG/interfaceFields/cyclicACMIGAMGInterfaceField/cyclicACMIGAMGInterfaceField.C

@@ -238,10 +238,13 @@ void Foam::cyclicACMIGAMGInterfaceField::initInterfaceMatrixUpdate
             sendRequests_,
             scalarSendBufs_,
             recvRequests_,
-            scalarRecvBufs_
+            scalarRecvBufs_,
+            19462+cyclicACMIInterface_.index()  // unique offset + patch index
         );
         UPstream::commWarn(oldWarnComm);
     }
+
+    this->updatedMatrix(false);
 }
 
 
@@ -286,7 +289,13 @@ void Foam::cyclicACMIGAMGInterfaceField::updateInterfaceMatrix
         // Receive (= copy) data from buffers into work. TBD: receive directly
         // into slices of work.
         solveScalarField work;
-        map.receive(recvRequests_, scalarRecvBufs_, work);
+        map.receive
+        (
+            recvRequests_,
+            scalarRecvBufs_,
+            work,
+            19462+cyclicACMIInterface_.index()  // unique offset + patch index
+        );
 
         // Receive requests all handled by last function call
         recvRequests_.clear();
@@ -327,6 +336,8 @@ void Foam::cyclicACMIGAMGInterfaceField::updateInterfaceMatrix
 
         this->addToInternalField(result, !add, faceCells, coeffs, pnf);
     }
+
+    this->updatedMatrix(true);
 }
 
 

src/meshTools/AMIInterpolation/GAMG/interfaceFields/cyclicAMIGAMGInterfaceField/cyclicAMIGAMGInterfaceField.C

@@ -239,10 +239,13 @@ void Foam::cyclicAMIGAMGInterfaceField::initInterfaceMatrixUpdate
             sendRequests_,
             scalarSendBufs_,
             recvRequests_,
-            scalarRecvBufs_
+            scalarRecvBufs_,
+            19462+cyclicAMIInterface_.index()   // unique offset + patch index
         );
         UPstream::commWarn(oldWarnComm);
     }
+
+    this->updatedMatrix(false);
 }
 
 
@@ -300,7 +303,13 @@ void Foam::cyclicAMIGAMGInterfaceField::updateInterfaceMatrix
         // Receive (= copy) data from buffers into work. TBD: receive directly
         // into slices of work.
         solveScalarField work;
-        map.receive(recvRequests_, scalarRecvBufs_, work);
+        map.receive
+        (
+            recvRequests_,
+            scalarRecvBufs_,
+            work,
+            19462+cyclicAMIInterface_.index()   // unique offset + patch index
+        );
 
         // Receive requests all handled by last function call
         recvRequests_.clear();
@@ -340,6 +349,8 @@ void Foam::cyclicAMIGAMGInterfaceField::updateInterfaceMatrix
         // Add result using coefficients
         this->addToInternalField(result, !add, faceCells, coeffs, pnf);
     }
+
+    this->updatedMatrix(true);
 }
 
 

src/meshTools/AMIInterpolation/patches/cyclicAMI/cyclicAMIPolyPatch/cyclicAMIPolyPatchTemplates.C

@@ -6,7 +6,7 @@
      \\/     M anipulation  |
 -------------------------------------------------------------------------------
     Copyright (C) 2011-2016 OpenFOAM Foundation
-    Copyright (C) 2021-2023 OpenCFD Ltd.
+    Copyright (C) 2021-2024 OpenCFD Ltd.
 -------------------------------------------------------------------------------
 License
     This file is part of OpenFOAM.
@@ -175,7 +175,15 @@ void Foam::cyclicAMIPolyPatch::initInterpolateUntransformed
         const auto& map = (owner() ? AMI.tgtMap() : AMI.srcMap());
 
         // Insert send/receive requests (non-blocking)
-        map.send(fld, sendRequests, sendBuffers, recvRequests, recvBuffers);
+        map.send
+        (
+            fld,
+            sendRequests,
+            sendBuffers,
+            recvRequests,
+            recvBuffers,
+            3894+this->index()  // unique offset + patch index
+        );
     }
 }
 
@@ -256,7 +264,13 @@ Foam::tmp<Foam::Field<Type>> Foam::cyclicAMIPolyPatch::interpolate
     {
         // Receive (= copy) data from buffers into work. TBD: receive directly
         // into slices of work.
-        map.receive(requests, recvBuffers, work);
+        map.receive
+        (
+            requests,
+            recvBuffers,
+            work,
+            3894+this->index()  // unique offset + patch index
+        );
     }
     const Field<Type>& fld = (AMI.distributed() ? work : localFld);
 

src/meshTools/polyTopoChange/polyTopoChange.C

@@ -2026,6 +2026,9 @@ void Foam::polyTopoChange::reorderCoupledFaces
     // Rotation on new faces.
     labelList rotation(faces_.size(), Zero);
 
+    // Allocate unique tag for all comms
+    const int oldTag = UPstream::incrMsgType();
+
     PstreamBuffers pBufs;
 
     // Send ordering
@@ -2131,6 +2134,9 @@ void Foam::polyTopoChange::reorderCoupledFaces
             }
         }
     }
+
+    // Reset tag
+    UPstream::msgType(oldTag);
 }