BUG: GAMG: processorAgglomeration not merging proc coeffs. Fixes #3172

- requires a field size check when copying into the flat boundary

src/OpenFOAM/matrices/lduMatrix/solvers/GAMG/GAMGSolverAgglomerateMatrix.C

@@ -6,7 +6,7 @@
      \\/     M anipulation  |
 -------------------------------------------------------------------------------
     Copyright (C) 2011-2017 OpenFOAM Foundation
-    Copyright (C) 2023 OpenCFD Ltd.
+    Copyright (C) 2023-2024 OpenCFD Ltd.
 -------------------------------------------------------------------------------
 License
     This file is part of OpenFOAM.
@@ -332,35 +332,47 @@ void Foam::GAMGSolver::gatherMatrices
 
         const label proci = UPstream::myProcNo(comm);
 
-        labelList validInterface(interfaces.size(), -1);
+        // All interfaceBouCoeffs need to be sent across
+        bitSet validCoeffs(interfaces.size());
+        forAll(interfaceBouCoeffs, intI)
+        {
+            if (interfaceBouCoeffs.set(intI))
+            {
+                validCoeffs.set(intI);
+            }
+        }
+        // Only preserved interfaces need to be sent across
+        bitSet validInterface(interfaces.size());
         forAll(interfaces, intI)
         {
             const label allIntI = boundaryMap[proci][intI];
             if (interfaces.set(intI) && allIntI != -1)
             {
-                validInterface[intI] = intI;
+                validInterface.set(intI);
             }
         }
 
         UOPstream toMaster(UPstream::masterNo(), pBufs);
 
-        toMaster<< mat << token::SPACE << validInterface;
+        toMaster<< mat
+            << token::SPACE << validCoeffs
+            << token::SPACE << validInterface;
 
-        forAll(validInterface, intI)
+        for (const label intI : validCoeffs)
         {
-            if (validInterface[intI] != -1)
-            {
-                const auto& interface = refCast<const GAMGInterfaceField>
-                (
-                    interfaces[intI]
-                );
+            toMaster
+                << interfaceBouCoeffs[intI]
+                << interfaceIntCoeffs[intI];
+        }
+        for (const label intI : validInterface)
+        {
+            const auto& interface = refCast<const GAMGInterfaceField>
+            (
+                interfaces[intI]
+            );
 
-                toMaster
-                    << interfaceBouCoeffs[intI]
-                    << interfaceIntCoeffs[intI]
-                    << interface.type();
-                interface.write(toMaster);
-            }
+            toMaster << interface.type();
+            interface.write(toMaster);
         }
     }
 
@@ -389,60 +401,41 @@ void Foam::GAMGSolver::gatherMatrices
 
             otherMats.set(otherI, new lduMatrix(destMesh, fromProc));
 
-            // Receive number of/valid interfaces
-            // >= 0 : remote interface index
-            // -1   : invalid interface
-            const labelList validInterface(fromProc);
+            // Receive bitSet of/valid interfaceCoeffs/interfaces
+            const bitSet validCoeffs(fromProc);
+            const bitSet validInterface(fromProc);
 
-            otherBouCoeffs.set
-            (
-                otherI,
-                new FieldField<Field, scalar>(validInterface.size())
-            );
-            otherIntCoeffs.set
-            (
-                otherI,
-                new FieldField<Field, scalar>(validInterface.size())
-            );
-            otherInterfaces.set
-            (
-                otherI,
-                new PtrList<lduInterfaceField>(validInterface.size())
-            );
+            otherBouCoeffs.emplace_set(otherI, validCoeffs.size());
+            otherIntCoeffs.emplace_set(otherI, validCoeffs.size());
+            otherInterfaces.emplace_set(otherI, validInterface.size());
 
-            forAll(validInterface, intI)
+            // Receive individual interface contributions
+            for (const label intI : validCoeffs)
             {
-                if (validInterface[intI] != -1)
-                {
-                    otherBouCoeffs[otherI].set
-                    (
-                        intI,
-                        new scalarField(fromProc)
-                    );
-                    otherIntCoeffs[otherI].set
-                    (
-                        intI,
-                        new scalarField(fromProc)
-                    );
+                otherBouCoeffs[otherI].emplace_set(intI, fromProc);
+                otherIntCoeffs[otherI].emplace_set(intI, fromProc);
+            }
 
-                    const word coupleType(fromProc);
+            // Receive individual interface contributions
+            for (const label intI : validInterface)
+            {
+                const word coupleType(fromProc);
 
-                    const label allIntI = boundaryMap[proci][intI];
+                const label allIntI = boundaryMap[proci][intI];
 
-                    otherInterfaces[otherI].set
+                otherInterfaces[otherI].set
+                (
+                    intI,
+                    GAMGInterfaceField::New
                     (
-                        intI,
-                        GAMGInterfaceField::New
+                        coupleType,
+                        refCast<const GAMGInterface>
                         (
-                            coupleType,
-                            refCast<const GAMGInterface>
-                            (
-                                destInterfaces[allIntI]
-                            ),
-                            fromProc
-                        ).release()
-                    );
-                }
+                            destInterfaces[allIntI]
+                        ),
+                        fromProc
+                    ).release()
+                );
             }
         }
     }

src/finiteVolume/fields/fvPatchFields/constraint/cyclicACMI/cyclicACMIFvPatchField.C

@@ -6,7 +6,7 @@
      \\/     M anipulation  |
 -------------------------------------------------------------------------------
     Copyright (C) 2013-2017 OpenFOAM Foundation
-    Copyright (C) 2019-2023 OpenCFD Ltd.
+    Copyright (C) 2019-2024 OpenCFD Ltd.
 -------------------------------------------------------------------------------
 License
     This file is part of OpenFOAM.
@@ -444,7 +444,14 @@ void Foam::cyclicACMIFvPatchField<Type>::initEvaluate
             << " starting send&receive"
             << endl;
 
-        if (!this->ready())
+        // Bypass polyPatch to get nbrId.
+        // - use cyclicACMIFvPatch::neighbPatch() virtual instead
+        const cyclicACMIFvPatch& neighbPatch = cyclicACMIPatch_.neighbPatch();
+        const labelUList& nbrFaceCells = neighbPatch.faceCells();
+        const Field<Type> pnf(this->primitiveField(), nbrFaceCells);
+
+        // Assert that all receives are known to have finished
+        if (!recvRequests_.empty())
         {
             FatalErrorInFunction
                 << "Outstanding recv request(s) on patch "
@@ -453,11 +460,8 @@ void Foam::cyclicACMIFvPatchField<Type>::initEvaluate
                 << abort(FatalError);
         }
 
-        // By-pass polyPatch to get nbrId. Instead use cyclicACMIFvPatch virtual
-        // neighbPatch()
-        const cyclicACMIFvPatch& neighbPatch = cyclicACMIPatch_.neighbPatch();
-        const labelUList& nbrFaceCells = neighbPatch.faceCells();
-        const Field<Type> pnf(this->primitiveField(), nbrFaceCells);
+        // Assume that sends are also OK
+        sendRequests_.clear();
 
         cyclicACMIPatch_.initInterpolate
         (
@@ -515,6 +519,10 @@ void Foam::cyclicACMIFvPatchField<Type>::evaluate
             ).ptr()
         );
 
+        // Receive requests all handled by last function call
+        recvRequests_.clear();
+
+
         auto& patchNeighbourField = patchNeighbourFieldPtr_.ref();
 
         if (doTransform())
@@ -559,7 +567,16 @@ void Foam::cyclicACMIFvPatchField<Type>::initInterfaceMatrixUpdate
             << " starting send&receive"
             << endl;
 
-        if (!this->ready())
+        const labelUList& nbrFaceCells =
+            lduAddr.patchAddr(cyclicACMIPatch_.neighbPatchID());
+
+        solveScalarField pnf(psiInternal, nbrFaceCells);
+
+        // Transform according to the transformation tensors
+        transformCoupleField(pnf, cmpt);
+
+        // Assert that all receives are known to have finished
+        if (!recvRequests_.empty())
         {
             FatalErrorInFunction
                 << "Outstanding recv request(s) on patch "
@@ -568,13 +585,8 @@ void Foam::cyclicACMIFvPatchField<Type>::initInterfaceMatrixUpdate
                 << abort(FatalError);
         }
 
-        const labelUList& nbrFaceCells =
-            lduAddr.patchAddr(cyclicACMIPatch_.neighbPatchID());
-
-        solveScalarField pnf(psiInternal, nbrFaceCells);
-
-        // Transform according to the transformation tensors
-        transformCoupleField(pnf, cmpt);
+        // Assume that sends are also OK
+        sendRequests_.clear();
 
         cyclicACMIPatch_.initInterpolate
         (
@@ -635,6 +647,9 @@ void Foam::cyclicACMIFvPatchField<Type>::updateInterfaceMatrix
                 recvRequests_,
                 scalarRecvBufs_
             );
+
+        // Receive requests all handled by last function call
+        recvRequests_.clear();
     }
     else
     {
@@ -676,7 +691,16 @@ void Foam::cyclicACMIFvPatchField<Type>::initInterfaceMatrixUpdate
                 << exit(FatalError);
         }
 
-        if (!this->ready())
+        const labelUList& nbrFaceCells =
+            lduAddr.patchAddr(cyclicACMIPatch_.neighbPatchID());
+
+        Field<Type> pnf(psiInternal, nbrFaceCells);
+
+        // Transform according to the transformation tensors
+        transformCoupleField(pnf);
+
+        // Assert that all receives are known to have finished
+        if (!recvRequests_.empty())
         {
             FatalErrorInFunction
                 << "Outstanding recv request(s) on patch "
@@ -685,13 +709,8 @@ void Foam::cyclicACMIFvPatchField<Type>::initInterfaceMatrixUpdate
                 << abort(FatalError);
         }
 
-        const labelUList& nbrFaceCells =
-            lduAddr.patchAddr(cyclicACMIPatch_.neighbPatchID());
-
-        Field<Type> pnf(psiInternal, nbrFaceCells);
-
-        // Transform according to the transformation tensors
-        transformCoupleField(pnf);
+        // Assume that sends are also OK
+        sendRequests_.clear();
 
         cyclicACMIPatch_.initInterpolate
         (
@@ -741,6 +760,9 @@ void Foam::cyclicACMIFvPatchField<Type>::updateInterfaceMatrix
                 recvRequests_,
                 recvBufs_
             );
+
+        // Receive requests all handled by last function call
+        recvRequests_.clear();
     }
     else
     {

src/finiteVolume/fields/fvPatchFields/constraint/cyclicAMI/cyclicAMIFvPatchField.C

@@ -6,7 +6,7 @@
      \\/     M anipulation  |
 -------------------------------------------------------------------------------
     Copyright (C) 2011-2017 OpenFOAM Foundation
-    Copyright (C) 2019-2023 OpenCFD Ltd.
+    Copyright (C) 2019-2024 OpenCFD Ltd.
 -------------------------------------------------------------------------------
 License
     This file is part of OpenFOAM.
@@ -446,14 +446,27 @@ void Foam::cyclicAMIFvPatchField<Type>::initEvaluate
 
         // Start sending
 
-        // By-pass polyPatch to get nbrId. Instead use cyclicAMIFvPatch virtual
-        // neighbPatch()
+        // Bypass polyPatch to get nbrId.
+        // - use cyclicACMIFvPatch::neighbPatch() virtual instead
         const cyclicAMIFvPatch& neighbPatch = cyclicAMIPatch_.neighbPatch();
         const labelUList& nbrFaceCells = neighbPatch.faceCells();
         const Field<Type> pnf(this->primitiveField(), nbrFaceCells);
 
         const cyclicAMIPolyPatch& cpp = cyclicAMIPatch_.cyclicAMIPatch();
 
+        // Assert that all receives are known to have finished
+        if (!recvRequests_.empty())
+        {
+            FatalErrorInFunction
+                << "Outstanding recv request(s) on patch "
+                << cyclicAMIPatch_.name()
+                << " field " << this->internalField().name()
+                << abort(FatalError);
+        }
+
+        // Assume that sends are also OK
+        sendRequests_.clear();
+
         cpp.initInterpolate
         (
             pnf,
@@ -516,6 +529,10 @@ void Foam::cyclicAMIFvPatchField<Type>::evaluate
                 defaultValues
             ).ptr()
         );
+
+        // Receive requests all handled by last function call
+        recvRequests_.clear();
+
         auto& patchNeighbourField = patchNeighbourFieldPtr_.ref();
 
         if (doTransform())
@@ -563,6 +580,19 @@ void Foam::cyclicAMIFvPatchField<Type>::initInterfaceMatrixUpdate
 
         const cyclicAMIPolyPatch& cpp = cyclicAMIPatch_.cyclicAMIPatch();
 
+        // Assert that all receives are known to have finished
+        if (!recvRequests_.empty())
+        {
+            FatalErrorInFunction
+                << "Outstanding recv request(s) on patch "
+                << cyclicAMIPatch_.name()
+                << " field " << this->internalField().name()
+                << abort(FatalError);
+        }
+
+        // Assume that sends are also OK
+        sendRequests_.clear();
+
         cpp.initInterpolate
         (
             pnf,
@@ -624,6 +654,9 @@ void Foam::cyclicAMIFvPatchField<Type>::updateInterfaceMatrix
                 scalarRecvBufs_,
                 defaultValues
             );
+
+        // Receive requests all handled by last function call
+        recvRequests_.clear();
     }
     else
     {
@@ -682,6 +715,19 @@ void Foam::cyclicAMIFvPatchField<Type>::initInterfaceMatrixUpdate
 
         const cyclicAMIPolyPatch& cpp = cyclicAMIPatch_.cyclicAMIPatch();
 
+        // Assert that all receives are known to have finished
+        if (!recvRequests_.empty())
+        {
+            FatalErrorInFunction
+                << "Outstanding recv request(s) on patch "
+                << cyclicAMIPatch_.name()
+                << " field " << this->internalField().name()
+                << abort(FatalError);
+        }
+
+        // Assume that sends are also OK
+        sendRequests_.clear();
+
         cpp.initInterpolate
         (
             pnf,
@@ -742,6 +788,9 @@ void Foam::cyclicAMIFvPatchField<Type>::updateInterfaceMatrix
                 recvBufs_,
                 defaultValues
             );
+
+        // Receive requests all handled by last function call
+        recvRequests_.clear();
     }
     else
     {

src/meshTools/AMIInterpolation/GAMG/interfaceFields/cyclicACMIGAMGInterfaceField/cyclicACMIGAMGInterfaceField.C

@@ -6,7 +6,7 @@
      \\/     M anipulation  |
 -------------------------------------------------------------------------------
     Copyright (C) 2013 OpenFOAM Foundation
-    Copyright (C) 2019,2023 OpenCFD Ltd.
+    Copyright (C) 2019-2024 OpenCFD Ltd.
 -------------------------------------------------------------------------------
 License
     This file is part of OpenFOAM.
@@ -217,6 +217,18 @@ void Foam::cyclicACMIGAMGInterfaceField::initInterfaceMatrixUpdate
           : AMI.srcMap()
         );
 
+        // Assert that all receives are known to have finished
+        if (!recvRequests_.empty())
+        {
+            FatalErrorInFunction
+                << "Outstanding recv request(s) on patch "
+                << cyclicACMIInterface_.index()
+                << abort(FatalError);
+        }
+
+        // Assume that sends are also OK
+        sendRequests_.clear();
+
         // Insert send/receive requests (non-blocking). See e.g.
         // cyclicAMIPolyPatchTemplates.C
         const label oldWarnComm = UPstream::commWarn(AMI.comm());
@@ -276,6 +288,9 @@ void Foam::cyclicACMIGAMGInterfaceField::updateInterfaceMatrix
         solveScalarField work;
         map.receive(recvRequests_, scalarRecvBufs_, work);
 
+        // Receive requests all handled by last function call
+        recvRequests_.clear();
+
         solveScalarField pnf(faceCells.size(), Zero);
         AMI.weightedSum
         (

src/meshTools/AMIInterpolation/GAMG/interfaceFields/cyclicAMIGAMGInterfaceField/cyclicAMIGAMGInterfaceField.C

@@ -6,7 +6,7 @@
      \\/     M anipulation  |
 -------------------------------------------------------------------------------
     Copyright (C) 2011-2013 OpenFOAM Foundation
-    Copyright (C) 2019,2023 OpenCFD Ltd.
+    Copyright (C) 2019-2024 OpenCFD Ltd.
 -------------------------------------------------------------------------------
 License
     This file is part of OpenFOAM.
@@ -218,6 +218,18 @@ void Foam::cyclicAMIGAMGInterfaceField::initInterfaceMatrixUpdate
           : AMI.srcMap()
         );
 
+        // Assert that all receives are known to have finished
+        if (!recvRequests_.empty())
+        {
+            FatalErrorInFunction
+                << "Outstanding recv request(s) on patch "
+                << cyclicAMIInterface_.index()
+                << abort(FatalError);
+        }
+
+        // Assume that sends are also OK
+        sendRequests_.clear();
+
         // Insert send/receive requests (non-blocking). See e.g.
         // cyclicAMIPolyPatchTemplates.C
         const label oldWarnComm = UPstream::commWarn(AMI.comm());
@@ -290,6 +302,9 @@ void Foam::cyclicAMIGAMGInterfaceField::updateInterfaceMatrix
         solveScalarField work;
         map.receive(recvRequests_, scalarRecvBufs_, work);
 
+        // Receive requests all handled by last function call
+        recvRequests_.clear();
+
         solveScalarField pnf(faceCells.size(), Zero);
         AMI.weightedSum
         (

src/sampling/sampledSurface/sampledMeshedSurface/sampledMeshedSurfaceTemplates.C

@@ -6,7 +6,7 @@
      \\/     M anipulation  |
 -------------------------------------------------------------------------------
     Copyright (C) 2011-2016 OpenFOAM Foundation
-    Copyright (C) 2016-2020 OpenCFD Ltd.
+    Copyright (C) 2016-2024 OpenCFD Ltd.
 -------------------------------------------------------------------------------
 License
     This file is part of OpenFOAM.
@@ -39,7 +39,11 @@ Foam::sampledMeshedSurface::sampleOnFaces
 {
     const Type deflt
     (
-        defaultValues_.getOrDefault<Type>(sampler.psi().name(), Zero)
+        defaultValues_.getOrDefault<Type>
+        (
+            sampler.psi().name(),
+            Foam::zero{}
+        )
     );
 
     const labelList& elements = sampleElements_;
@@ -71,13 +75,16 @@ Foam::sampledMeshedSurface::sampleOnFaces
 
     const polyBoundaryMesh& pbm = mesh().boundaryMesh();
 
-    Field<Type> bVals(mesh().nBoundaryFaces(), Zero);
+    Field<Type> bVals(mesh().nBoundaryFaces(), deflt);
 
     const auto& bField = sampler.psi().boundaryField();
 
     forAll(bField, patchi)
     {
-        SubList<Type>(bVals, pbm[patchi].range()) = bField[patchi];
+        // Note: restrict transcribing to actual size of the patch field
+        // - handles "empty" patch type etc.
+        const auto& pfld = bField[patchi];
+        SubList<Type>(bVals, pfld.size(), pbm[patchi].offset()) = pfld;
     }
 
     // Sample within the flat boundary field
@@ -109,7 +116,11 @@ Foam::sampledMeshedSurface::sampleOnPoints
 {
     const Type deflt
     (
-        defaultValues_.getOrDefault<Type>(interpolator.psi().name(), Zero)
+        defaultValues_.getOrDefault<Type>
+        (
+            interpolator.psi().name(),
+            Foam::zero{}
+        )
     );
 
     const labelList& elements = sampleElements_;