From a00d3fdec188e2ac646ff7b9a044fde095f3947b Mon Sep 17 00:00:00 2001
From: Tim MJ Nijssen <37873965+tmjnijssen@users.noreply.github.com>
Date: Wed, 30 Nov 2022 11:26:38 +0100
Subject: [PATCH] use nonblocking sends

---
 .../dataExchangeModel/twoWayOne2One/one2one.C | 28 ++++++++++++++-----
 1 file changed, 21 insertions(+), 7 deletions(-)

diff --git a/src/lagrangian/cfdemParticle/subModels/dataExchangeModel/twoWayOne2One/one2one.C b/src/lagrangian/cfdemParticle/subModels/dataExchangeModel/twoWayOne2One/one2one.C
index be6372f2..ee14d499 100644
--- a/src/lagrangian/cfdemParticle/subModels/dataExchangeModel/twoWayOne2One/one2one.C
+++ b/src/lagrangian/cfdemParticle/subModels/dataExchangeModel/twoWayOne2One/one2one.C
@@ -100,6 +100,7 @@ void One2One::setup
     comm_
   );
 
+  // count number of receives
   ncollected_ = 0;
   int nrequests = 0;
   for (int i = 0; i < nsrc_procs_; i++)
@@ -115,6 +116,18 @@ void One2One::setup
     }
   }
 
+  // count number of sends
+  if (nlocal_ > 0)
+  {
+    for (int i = 0; i < ndst_procs_; i++)
+    {
+      if (dst_procs_[i] != me_)
+      {
+        nrequests++;
+      }
+    }
+  }
+
   if (nrequests > 0)
   {
     request_ = new MPI_Request[nrequests];
@@ -175,11 +188,7 @@ void One2One::exchange(T *&src, T *&dst, int data_length)
     offset += natoms_[src_procs_[i]]*data_length;
   }
 
-  // make sure all receives are posted
-  MPI_Barrier(comm_);
-
-  // blocking sends - do nonblocking instead
-  //                  since doing many-2-many here?
+  // nonblocking sends
   // only do sends if I have particles
   if (nlocal_ > 0)
   {
@@ -195,15 +204,17 @@ void One2One::exchange(T *&src, T *&dst, int data_length)
              << " data_length " << data_length
              << std::endl;
     #endif
-        MPI_Send
+        MPI_Isend
         (
           src,
           nlocal_*data_length,
           wrap.mpi_type,
           dst_procs_[i],
           tag,
-          comm_
+          comm_,
+          &request_[requesti]
         );
+        requesti++;
       }
     }
   }
@@ -226,6 +237,9 @@ void One2One::exchange(T *&src, T *&dst, int data_length)
       dst[locali+offset_local] = src[locali];
     }
   }
+
+  // wait for all procs to complete data exchange
+  MPI_Barrier(comm_);
 }
 
 template void One2One::exchange<int>(int*&, int*&, int);