From 812f4c4f09962f33f7bbfc9b76661dfb8016014d Mon Sep 17 00:00:00 2001
From: Mark Olesen <Mark.Olesen@esi-group.com>
Date: Thu, 12 May 2022 11:17:07 +0200
Subject: [PATCH] BUG: globalIndex fails non-blocking gather of indirect list
 (fixes #2467)

- for indirect lists we use element-wise output streaming and read
  back as a regular list. This approach cannot however work with
  non-blocking mode - the receive buffers will simply not be filled
  before attempting to read from them.

  For contiguous data, the lowest overhead solution is to locally
  flatten the indirect list and use the regular gather routines
  for non-blocking mode. For non-contiguous data, can continue to
  use the element-wise output, but cannot use non-blocking for it.

STYLE: use non-blocking consistently as default for globalIndex gather(s)

- most of the front-facing code was already using non-blocking,
  but there were a few low-level routines defaulting to scheduled
  (but never relied upon in the code).
---
 .../parallel/globalIndex/globalIndex.H         |  4 ++--
 .../globalIndex/globalIndexTemplates.C         | 18 ++++++++++++++++++
 2 files changed, 20 insertions(+), 2 deletions(-)

diff --git a/src/OpenFOAM/parallel/globalIndex/globalIndex.H b/src/OpenFOAM/parallel/globalIndex/globalIndex.H
index 213482ee99..4f84920416 100644
--- a/src/OpenFOAM/parallel/globalIndex/globalIndex.H
+++ b/src/OpenFOAM/parallel/globalIndex/globalIndex.H
@@ -421,7 +421,7 @@ public:
             const Type& localValue,
             List<Type>& allValues,      //! output field (master only)
             const int tag = UPstream::msgType(),
-            const UPstream::commsTypes = UPstream::commsTypes::scheduled
+            const UPstream::commsTypes = UPstream::commsTypes::nonBlocking
         );
 
         //- Collect data in processor order on master (== procIDs[0]).
@@ -449,7 +449,7 @@ public:
             const IndirectListBase<Type, Addr>& fld,
             List<Type>& allFld,         //! output field (master only)
             const int tag = UPstream::msgType(),
-            const UPstream::commsTypes = UPstream::commsTypes::scheduled
+            const UPstream::commsTypes = UPstream::commsTypes::nonBlocking
         );
 
 
diff --git a/src/OpenFOAM/parallel/globalIndex/globalIndexTemplates.C b/src/OpenFOAM/parallel/globalIndex/globalIndexTemplates.C
index e26a77265a..2029ea4f07 100644
--- a/src/OpenFOAM/parallel/globalIndex/globalIndexTemplates.C
+++ b/src/OpenFOAM/parallel/globalIndex/globalIndexTemplates.C
@@ -262,6 +262,24 @@ void Foam::globalIndex::gather
 {
     // low-level: no parRun guard
 
+    if (is_contiguous<Type>::value)
+    {
+        // Flatten list (locally) so that we can benefit from using direct
+        // read/write of contiguous data
+
+        gather
+        (
+            off,
+            comm,
+            procIDs,
+            List<Type>(fld),
+            allFld,
+            tag,
+            preferredCommsType
+        );
+        return;
+    }
+
     // Automatically change from nonBlocking to scheduled for
     // non-contiguous data.
     const UPstream::commsTypes commsType =