ENH: restrict scope of waitRequests() in LduMatrix update

- Only wait for locally invoked requests.
  Reflects behaviour of lduMatrix update.

src/OpenFOAM/matrices/LduMatrix/LduMatrix/LduMatrix.H

@@ -6,7 +6,7 @@
      \\/     M anipulation  |
 -------------------------------------------------------------------------------
     Copyright (C) 2011-2017 OpenFOAM Foundation
-    Copyright (C) 2019-2022 OpenCFD Ltd.
+    Copyright (C) 2019-2023 OpenCFD Ltd.
 -------------------------------------------------------------------------------
 License
     This file is part of OpenFOAM.
@@ -637,7 +637,8 @@ public:
                 const bool add,
                 const FieldField<Field, LUType>& interfaceCoeffs,
                 const Field<Type>& psiif,
-                Field<Type>& result
+                Field<Type>& result,
+                const label startRequest // starting request (for non-blocking)
             ) const;
 
 

src/OpenFOAM/matrices/LduMatrix/LduMatrix/LduMatrixATmul.C

@@ -6,7 +6,7 @@
      \\/     M anipulation  |
 -------------------------------------------------------------------------------
     Copyright (C) 2011-2016 OpenFOAM Foundation
-    Copyright (C) 2017-2019 OpenCFD Ltd.
+    Copyright (C) 2017-2023 OpenCFD Ltd.
 -------------------------------------------------------------------------------
 License
     This file is part of OpenFOAM.
@@ -80,6 +80,8 @@ void Foam::LduMatrix<Type, DType, LUType>::Amul
     const LUType* const __restrict__ upperPtr = upper().begin();
     const LUType* const __restrict__ lowerPtr = lower().begin();
 
+    const label startRequest = UPstream::nRequests();
+
     // Initialise the update of interfaced interfaces
     initMatrixInterfaces
     (
@@ -109,7 +111,8 @@ void Foam::LduMatrix<Type, DType, LUType>::Amul
         true,
         interfacesUpper_,
         psi,
-        Apsi
+        Apsi,
+        startRequest
     );
 
     tpsi.clear();
@@ -136,6 +139,8 @@ void Foam::LduMatrix<Type, DType, LUType>::Tmul
     const LUType* const __restrict__ lowerPtr = lower().begin();
     const LUType* const __restrict__ upperPtr = upper().begin();
 
+    const label startRequest = UPstream::nRequests();
+
     // Initialise the update of interfaced interfaces
     initMatrixInterfaces
     (
@@ -164,7 +169,8 @@ void Foam::LduMatrix<Type, DType, LUType>::Tmul
         true,
         interfacesLower_,
         psi,
-        Tpsi
+        Tpsi,
+        startRequest
     );
 
     tpsi.clear();
@@ -242,6 +248,8 @@ void Foam::LduMatrix<Type, DType, LUType>::residual
     // Note: there is a change of sign in the coupled
     // interface update to add the contibution to the r.h.s.
 
+    const label startRequest = UPstream::nRequests();
+
     // Initialise the update of interfaced interfaces
     initMatrixInterfaces
     (
@@ -271,7 +279,8 @@ void Foam::LduMatrix<Type, DType, LUType>::residual
         false,
         interfacesUpper_,
         psi,
-        rA
+        rA,
+        startRequest
     );
 }
 

src/OpenFOAM/matrices/LduMatrix/LduMatrix/LduMatrixUpdateMatrixInterfaces.C

@@ -6,7 +6,7 @@
      \\/     M anipulation  |
 -------------------------------------------------------------------------------
     Copyright (C) 2011-2017 OpenFOAM Foundation
-    Copyright (C) 2019-2022 OpenCFD Ltd.
+    Copyright (C) 2019-2023 OpenCFD Ltd.
 -------------------------------------------------------------------------------
 License
     This file is part of OpenFOAM.
@@ -111,23 +111,24 @@ void Foam::LduMatrix<Type, DType, LUType>::updateMatrixInterfaces
     const bool add,
     const FieldField<Field, LUType>& interfaceCoeffs,
     const Field<Type>& psiif,
-    Field<Type>& result
+    Field<Type>& result,
+    const label startRequest
 ) const
 {
     const UPstream::commsTypes commsType = UPstream::defaultCommsType;
 
+    // Block until sends/receives have finished
+    if (commsType == UPstream::commsTypes::nonBlocking)
+    {
+        UPstream::waitRequests(startRequest);
+    }
+
     if
     (
         commsType == UPstream::commsTypes::blocking
      || commsType == UPstream::commsTypes::nonBlocking
     )
     {
-        // Block until all sends/receives have been finished
-        if (commsType == UPstream::commsTypes::nonBlocking)
-        {
-            UPstream::waitRequests();
-        }
-
         forAll(interfaces_, interfacei)
         {
             if (interfaces_.set(interfacei))

src/OpenFOAM/matrices/LduMatrix/Smoothers/GaussSeidel/TGaussSeidelSmoother.C

@@ -6,7 +6,7 @@
      \\/     M anipulation  |
 -------------------------------------------------------------------------------
     Copyright (C) 2011-2016 OpenFOAM Foundation
-    Copyright (C) 2017 OpenCFD Ltd.
+    Copyright (C) 2017-2023 OpenCFD Ltd.
 -------------------------------------------------------------------------------
 License
     This file is part of OpenFOAM.
@@ -98,6 +98,8 @@ void Foam::TGaussSeidelSmoother<Type, DType, LUType>::smooth
     {
         bPrime = matrix_.source();
 
+        const label startRequest = UPstream::nRequests();
+
         matrix_.initMatrixInterfaces
         (
             false,
@@ -111,7 +113,8 @@ void Foam::TGaussSeidelSmoother<Type, DType, LUType>::smooth
             false,
             matrix_.interfacesUpper(),
             psi,
-            bPrime
+            bPrime,
+            startRequest
         );
 
         Type curPsi;

src/OpenFOAM/meshes/lduMesh/lduPrimitiveMesh.C

@@ -6,7 +6,7 @@
      \\/     M anipulation  |
 -------------------------------------------------------------------------------
     Copyright (C) 2013-2017 OpenFOAM Foundation
-    Copyright (C) 2019,2022 OpenCFD Ltd.
+    Copyright (C) 2019-2023 OpenCFD Ltd.
 -------------------------------------------------------------------------------
 License
     This file is part of OpenFOAM.
@@ -116,6 +116,8 @@ Foam::labelListList Foam::lduPrimitiveMesh::globalCellCells
     // Get the interface cells
     PtrList<labelList> nbrGlobalCells(interfaces.size());
     {
+        const label startOfRequests = UPstream::nRequests();
+
         // Initialise transfer of restrict addressing on the interface
         forAll(interfaces, inti)
         {
@@ -129,8 +131,7 @@ Foam::labelListList Foam::lduPrimitiveMesh::globalCellCells
             }
         }
 
-        // Wait for all
+        UPstream::waitRequests(startOfRequests);
-        UPstream::waitRequests();
 
         forAll(interfaces, inti)
         {