BUG: globalIndex gather/scatter fails with multi-world (fixes #2706)

- was using UPstream::procIDs(), which returns the sub-ranks with
  respect to the parent communicator. This is normally just an
  identity list (single-world) but with multi-world the indexing
  is incorrect.  Use UPstream::allProcs() instead.

src/OpenFOAM/parallel/globalIndex/globalIndex.H

@@ -489,12 +489,12 @@ public:
 
         //- Collect indirect data in processor order on master
         //  Handles contiguous/non-contiguous data, skips empty fields.
-        template<class Type, class Addr>
+        template<class ProcIDsContainer, class Type, class Addr>
         static void gather
         (
             const labelUList& offsets,  //!< offsets (master only)
             const label comm,           //!< communicator
-            const UList<int>& procIDs,
+            const ProcIDsContainer& procIDs,
             const IndirectListBase<Type, Addr>& fld,
             List<Type>& allFld,         //! output field (master only)
             const int tag = UPstream::msgType(),

src/OpenFOAM/parallel/globalIndex/globalIndexTemplates.C

@@ -6,7 +6,7 @@
      \\/     M anipulation  |
 -------------------------------------------------------------------------------
     Copyright (C) 2013-2017 OpenFOAM Foundation
-    Copyright (C) 2019-2022 OpenCFD Ltd.
+    Copyright (C) 2019-2023 OpenCFD Ltd.
 -------------------------------------------------------------------------------
 License
     This file is part of OpenFOAM.
@@ -248,12 +248,12 @@ void Foam::globalIndex::gather
 }
 
 
-template<class Type, class Addr>
+template<class ProcIDsContainer, class Type, class Addr>
 void Foam::globalIndex::gather
 (
     const labelUList& off,  // needed on master only
     const label comm,
-    const UList<int>& procIDs,
+    const ProcIDsContainer& procIDs,
     const IndirectListBase<Type, Addr>& fld,
     List<Type>& allFld,
     const int tag,
@@ -368,7 +368,7 @@ void Foam::globalIndex::gather
         (
             offsets_,  // needed on master only
             comm,
-            UPstream::procID(comm),
+            UPstream::allProcs(comm),   // All communicator ranks
             sendData,
             allData,
             tag,
@@ -404,7 +404,7 @@ void Foam::globalIndex::gather
         (
             offsets_,  // needed on master only
             comm,
-            UPstream::procID(comm),
+            UPstream::allProcs(comm),   // All communicator ranks
             sendData,
             allData,
             tag,
@@ -622,7 +622,7 @@ void Foam::globalIndex::mpiGather
             (
                 offsets_,  // needed on master only
                 comm,
-                UPstream::procID(comm),
+                UPstream::allProcs(comm),   // All communicator ranks
                 sendData,
                 allData,
                 tag,
@@ -967,7 +967,7 @@ void Foam::globalIndex::scatter
         (
             offsets_,  // needed on master only
             comm,
-            UPstream::procID(comm),
+            UPstream::allProcs(comm),  // All communicator ranks
             allData,
             localData,
             tag,

src/finiteArea/faMesh/faBoundaryMesh/faBoundaryMesh.C

@@ -6,7 +6,7 @@
      \\/     M anipulation  |
 -------------------------------------------------------------------------------
     Copyright (C) 2016-2017 Wikki Ltd
-    Copyright (C) 2018-2022 OpenCFD Ltd.
+    Copyright (C) 2018-2023 OpenCFD Ltd.
 -------------------------------------------------------------------------------
 License
     This file is part of OpenFOAM.
@@ -180,12 +180,26 @@ Foam::faBoundaryMesh::faBoundaryMesh
 
 void Foam::faBoundaryMesh::calcGeometry()
 {
-    // processorFaPatch geometry triggers calculation of pointNormals.
+    // processor initGeometry send/recv the following:
+    //   - edgeCentres() : faMesh::edgeCentres()
+    //   - edgeLengths() : faMesh::Le()
+    //   - edgeFaceCentres() : faMesh::areaCentres()
+    //
+    // faMesh::Le() has its own point-to-point communication (OK) but
+    // triggers either/or edgeAreaNormals(), pointAreaNormals()
+    // with their own communication that can block.
+
     // This uses parallel comms and hence will not be trigggered
     // on processors that do not have a processorFaPatch so instead
     // force construction.
+
+    (void)mesh_.edgeAreaNormals();
     (void)mesh_.pointAreaNormals();
 
+    (void)mesh_.areaCentres();
+    (void)mesh_.faceAreaNormals();
+
+
     PstreamBuffers pBufs(Pstream::defaultCommsType);
 
     if
@@ -773,12 +787,15 @@ bool Foam::faBoundaryMesh::checkDefinition(const bool report) const
 
 void Foam::faBoundaryMesh::movePoints(const pointField& p)
 {
-    // processorFaPatch geometry triggers calculation of pointNormals.
-    // This uses parallel comms and hence will not be trigggered
-    // on processors that do not have a processorFaPatch so instead
-    // force construction.
+    // See comments in calcGeometry()
+
+    (void)mesh_.edgeAreaNormals();
     (void)mesh_.pointAreaNormals();
 
+    (void)mesh_.areaCentres();
+    (void)mesh_.faceAreaNormals();
+
+
     PstreamBuffers pBufs(Pstream::defaultCommsType);
 
     if

src/finiteArea/faMesh/faMesh.C

@@ -6,7 +6,7 @@
      \\/     M anipulation  |
 -------------------------------------------------------------------------------
     Copyright (C) 2016-2017 Wikki Ltd
-    Copyright (C) 2020-2022 OpenCFD Ltd.
+    Copyright (C) 2020-2023 OpenCFD Ltd.
 -------------------------------------------------------------------------------
 License
     This file is part of OpenFOAM.
@@ -277,6 +277,27 @@ void Foam::faMesh::clearOut() const
 
 // * * * * * * * * * * * * * * * * Constructors  * * * * * * * * * * * * * * //
 
+void Foam::faMesh::syncGeom()
+{
+    if (UPstream::parRun())
+    {
+        // areaCentres()
+        if (faceCentresPtr_)
+        {
+            faceCentresPtr_->boundaryFieldRef()
+                .evaluateCoupled<processorFaPatch>();
+        }
+
+        // faceAreaNormals()
+        if (faceAreaNormalsPtr_)
+        {
+            faceAreaNormalsPtr_->boundaryFieldRef()
+                .evaluateCoupled<processorFaPatch>();
+        }
+    }
+}
+
+
 bool Foam::faMesh::init(const bool doInit)
 {
     if (doInit)
@@ -296,18 +317,7 @@ bool Foam::faMesh::init(const bool doInit)
     // Calculate the geometry for the patches (transformation tensors etc.)
     boundary_.calcGeometry();
 
-    // Ensure processor/processor information is properly synchronised
-    if (Pstream::parRun())
-    {
-        const_cast<areaVectorField&>(areaCentres()).boundaryFieldRef()
-            .evaluateCoupled<processorFaPatch>();
-
-        // This roughly corresponds to what OpenFOAM-v2112 (and earlier) had,
-        // but should nominally be unnecessary.
-        //
-        /// const_cast<areaVectorField&>(faceAreaNormals()).boundaryFieldRef()
-        ///     .evaluateCoupled<processorFaPatch>();
-    }
+    syncGeom();
 
     return false;
 }
@@ -989,7 +999,6 @@ bool Foam::faMesh::movePoints()
 
     clearGeomNotAreas();
 
-    // To satisfy the motion interface for MeshObject, const cast is needed
     if (patchPtr_)
     {
         patchPtr_->movePoints(newPoints);
@@ -1003,6 +1012,8 @@ bool Foam::faMesh::movePoints()
 
     // Note: Fluxes were dummy?
 
+    syncGeom();
+
     return true;
 }
 

src/finiteArea/faMesh/faMesh.H

@@ -6,7 +6,7 @@
      \\/     M anipulation  |
 -------------------------------------------------------------------------------
     Copyright (C) 2016-2017 Wikki Ltd
-    Copyright (C) 2021-2022 OpenCFD Ltd.
+    Copyright (C) 2021-2023 OpenCFD Ltd.
 -------------------------------------------------------------------------------
 License
     This file is part of OpenFOAM.
@@ -617,6 +617,10 @@ public:
         //- Initialise non-demand-driven data etc
         bool init(const bool doInit);
 
+        //- Processor/processor synchronisation for geometry fields.
+        //  Largely internal use only (slightly hacky).
+        void syncGeom();
+
 
     // Database
 

src/finiteArea/faMesh/faMeshDemandDrivenData.C

@@ -898,6 +898,12 @@ void Foam::faMesh::calcFaceCentres() const
             }
         }
     }
+
+    // Parallel consistency, exchange on processor patches
+    if (UPstream::parRun())
+    {
+        centres.boundaryFieldRef().evaluateCoupled<processorFaPatch>();
+    }
 }
 
 
@@ -1110,6 +1116,12 @@ void Foam::faMesh::calcFaceAreaNormals() const
                 = edgeNormalsBoundary[patchi];
         }
     }
+
+    // Parallel consistency, exchange on processor patches
+    if (UPstream::parRun())
+    {
+        faceNormals.boundaryFieldRef().evaluateCoupled<processorFaPatch>();
+    }
 }
 
 

src/finiteArea/faMesh/faMeshTools/faMeshTools.C

@@ -6,7 +6,7 @@
      \\/     M anipulation  |
 -------------------------------------------------------------------------------
     Copyright (C) 2012-2016 OpenFOAM Foundation
-    Copyright (C) 2015-2022 OpenCFD Ltd.
+    Copyright (C) 2015-2023 OpenCFD Ltd.
 -------------------------------------------------------------------------------
 License
     This file is part of OpenFOAM.
@@ -61,6 +61,8 @@ void Foam::faMeshTools::forceDemandDriven(faMesh& mesh)
     (void)mesh.pointAreaNormals();
     (void)mesh.faceCurvatures();
     (void)mesh.edgeTransformTensors();
+
+    mesh.syncGeom();
 }
 
 

src/parallel/distributed/distributedTriSurfaceMesh/distributedTriSurfaceMesh.C

@@ -1949,7 +1949,7 @@ Foam::distributedTriSurfaceMesh::independentlyDistributedBbs
 //        //globalBorderTris.gather
 //        //(
 //        //    UPstream::worldComm,
-//        //    UPstream::procID(Pstream::worldComm),
+//        //    UPstream::allProcs(UPstream::worldComm),
 //        //    globalBorderCentres
 //        //);
 //        pointField globalBorderCentres(allCentres);
@@ -1996,7 +1996,7 @@ Foam::distributedTriSurfaceMesh::independentlyDistributedBbs
 //        //globalBorderTris.scatter
 //        //(
 //        //    UPstream::worldComm,
-//        //    UPstream::procID(Pstream::worldComm),
+//        //    UPstream::allProcs(UPstream::worldComm),
 //        //    isMasterPoint
 //        //);
 //        //boolList isMasterBorder(s.size(), false);
@@ -2094,7 +2094,7 @@ Foam::distributedTriSurfaceMesh::independentlyDistributedBbs
             globalTris().gather
             (
                 UPstream::worldComm,
-                UPstream::procID(Pstream::worldComm),
+                UPstream::allProcs(UPstream::worldComm),
                 allCentres
             );
         }
@@ -2144,7 +2144,7 @@ Foam::distributedTriSurfaceMesh::independentlyDistributedBbs
             globalTris().scatter
             (
                 UPstream::worldComm,
-                UPstream::procID(Pstream::worldComm),
+                UPstream::allProcs(UPstream::worldComm),
                 allDistribution,
                 distribution
             );

src/surfMesh/readers/ensight/ensightSurfaceReaderTemplates.C

@@ -82,7 +82,7 @@ Foam::tmp<Foam::Field<Type>> Foam::ensightSurfaceReader::readField
         }
 
         // Check that data type is as expected
-        // (assumes OpenFOAM generated the data set)
+        // (assuming OpenFOAM generated the data set)
         string primitiveType;
         is.read(primitiveType);
 
@@ -90,7 +90,8 @@ Foam::tmp<Foam::Field<Type>> Foam::ensightSurfaceReader::readField
 
         if
         (
-            primitiveType != ensightPTraits<Type>::typeName
+            debug
+         && primitiveType != ensightPTraits<Type>::typeName
          && primitiveType != pTraits<Type>::typeName
         )
         {