diff --git a/src/OpenFOAM/matrices/LduMatrix/LduMatrix/LduMatrix.H b/src/OpenFOAM/matrices/LduMatrix/LduMatrix/LduMatrix.H index db730d7a97..96e9fd14f4 100644 --- a/src/OpenFOAM/matrices/LduMatrix/LduMatrix/LduMatrix.H +++ b/src/OpenFOAM/matrices/LduMatrix/LduMatrix/LduMatrix.H @@ -6,7 +6,7 @@ \\/ M anipulation | ------------------------------------------------------------------------------- Copyright (C) 2011-2017 OpenFOAM Foundation - Copyright (C) 2019-2022 OpenCFD Ltd. + Copyright (C) 2019-2023 OpenCFD Ltd. ------------------------------------------------------------------------------- License This file is part of OpenFOAM. @@ -637,7 +637,8 @@ public: const bool add, const FieldField& interfaceCoeffs, const Field& psiif, - Field& result + Field& result, + const label startRequest // starting request (for non-blocking) ) const; diff --git a/src/OpenFOAM/matrices/LduMatrix/LduMatrix/LduMatrixATmul.C b/src/OpenFOAM/matrices/LduMatrix/LduMatrix/LduMatrixATmul.C index 9ae8571e86..565068d4d9 100644 --- a/src/OpenFOAM/matrices/LduMatrix/LduMatrix/LduMatrixATmul.C +++ b/src/OpenFOAM/matrices/LduMatrix/LduMatrix/LduMatrixATmul.C @@ -6,7 +6,7 @@ \\/ M anipulation | ------------------------------------------------------------------------------- Copyright (C) 2011-2016 OpenFOAM Foundation - Copyright (C) 2017-2019 OpenCFD Ltd. + Copyright (C) 2017-2023 OpenCFD Ltd. ------------------------------------------------------------------------------- License This file is part of OpenFOAM. @@ -80,6 +80,8 @@ void Foam::LduMatrix::Amul const LUType* const __restrict__ upperPtr = upper().begin(); const LUType* const __restrict__ lowerPtr = lower().begin(); + const label startRequest = UPstream::nRequests(); + // Initialise the update of interfaced interfaces initMatrixInterfaces ( @@ -109,7 +111,8 @@ void Foam::LduMatrix::Amul true, interfacesUpper_, psi, - Apsi + Apsi, + startRequest ); tpsi.clear(); @@ -136,6 +139,8 @@ void Foam::LduMatrix::Tmul const LUType* const __restrict__ lowerPtr = lower().begin(); const LUType* const __restrict__ upperPtr = upper().begin(); + const label startRequest = UPstream::nRequests(); + // Initialise the update of interfaced interfaces initMatrixInterfaces ( @@ -164,7 +169,8 @@ void Foam::LduMatrix::Tmul true, interfacesLower_, psi, - Tpsi + Tpsi, + startRequest ); tpsi.clear(); @@ -242,6 +248,8 @@ void Foam::LduMatrix::residual // Note: there is a change of sign in the coupled // interface update to add the contibution to the r.h.s. + const label startRequest = UPstream::nRequests(); + // Initialise the update of interfaced interfaces initMatrixInterfaces ( @@ -271,7 +279,8 @@ void Foam::LduMatrix::residual false, interfacesUpper_, psi, - rA + rA, + startRequest ); } diff --git a/src/OpenFOAM/matrices/LduMatrix/LduMatrix/LduMatrixUpdateMatrixInterfaces.C b/src/OpenFOAM/matrices/LduMatrix/LduMatrix/LduMatrixUpdateMatrixInterfaces.C index f3679554e8..0b63849d53 100644 --- a/src/OpenFOAM/matrices/LduMatrix/LduMatrix/LduMatrixUpdateMatrixInterfaces.C +++ b/src/OpenFOAM/matrices/LduMatrix/LduMatrix/LduMatrixUpdateMatrixInterfaces.C @@ -6,7 +6,7 @@ \\/ M anipulation | ------------------------------------------------------------------------------- Copyright (C) 2011-2017 OpenFOAM Foundation - Copyright (C) 2019-2022 OpenCFD Ltd. + Copyright (C) 2019-2023 OpenCFD Ltd. ------------------------------------------------------------------------------- License This file is part of OpenFOAM. @@ -111,23 +111,24 @@ void Foam::LduMatrix::updateMatrixInterfaces const bool add, const FieldField& interfaceCoeffs, const Field& psiif, - Field& result + Field& result, + const label startRequest ) const { const UPstream::commsTypes commsType = UPstream::defaultCommsType; + // Block until sends/receives have finished + if (commsType == UPstream::commsTypes::nonBlocking) + { + UPstream::waitRequests(startRequest); + } + if ( commsType == UPstream::commsTypes::blocking || commsType == UPstream::commsTypes::nonBlocking ) { - // Block until all sends/receives have been finished - if (commsType == UPstream::commsTypes::nonBlocking) - { - UPstream::waitRequests(); - } - forAll(interfaces_, interfacei) { if (interfaces_.set(interfacei)) diff --git a/src/OpenFOAM/matrices/LduMatrix/Smoothers/GaussSeidel/TGaussSeidelSmoother.C b/src/OpenFOAM/matrices/LduMatrix/Smoothers/GaussSeidel/TGaussSeidelSmoother.C index f4a7ca0e17..61a90dba53 100644 --- a/src/OpenFOAM/matrices/LduMatrix/Smoothers/GaussSeidel/TGaussSeidelSmoother.C +++ b/src/OpenFOAM/matrices/LduMatrix/Smoothers/GaussSeidel/TGaussSeidelSmoother.C @@ -6,7 +6,7 @@ \\/ M anipulation | ------------------------------------------------------------------------------- Copyright (C) 2011-2016 OpenFOAM Foundation - Copyright (C) 2017 OpenCFD Ltd. + Copyright (C) 2017-2023 OpenCFD Ltd. ------------------------------------------------------------------------------- License This file is part of OpenFOAM. @@ -98,6 +98,8 @@ void Foam::TGaussSeidelSmoother::smooth { bPrime = matrix_.source(); + const label startRequest = UPstream::nRequests(); + matrix_.initMatrixInterfaces ( false, @@ -111,7 +113,8 @@ void Foam::TGaussSeidelSmoother::smooth false, matrix_.interfacesUpper(), psi, - bPrime + bPrime, + startRequest ); Type curPsi; diff --git a/src/OpenFOAM/meshes/lduMesh/lduPrimitiveMesh.C b/src/OpenFOAM/meshes/lduMesh/lduPrimitiveMesh.C index 5bade23b55..8f0c928b39 100644 --- a/src/OpenFOAM/meshes/lduMesh/lduPrimitiveMesh.C +++ b/src/OpenFOAM/meshes/lduMesh/lduPrimitiveMesh.C @@ -6,7 +6,7 @@ \\/ M anipulation | ------------------------------------------------------------------------------- Copyright (C) 2013-2017 OpenFOAM Foundation - Copyright (C) 2019,2022 OpenCFD Ltd. + Copyright (C) 2019-2023 OpenCFD Ltd. ------------------------------------------------------------------------------- License This file is part of OpenFOAM. @@ -116,6 +116,8 @@ Foam::labelListList Foam::lduPrimitiveMesh::globalCellCells // Get the interface cells PtrList nbrGlobalCells(interfaces.size()); { + const label startOfRequests = UPstream::nRequests(); + // Initialise transfer of restrict addressing on the interface forAll(interfaces, inti) { @@ -129,8 +131,7 @@ Foam::labelListList Foam::lduPrimitiveMesh::globalCellCells } } - // Wait for all - UPstream::waitRequests(); + UPstream::waitRequests(startOfRequests); forAll(interfaces, inti) {