WIP: PstreamBuffers with hybrid PEX/NBX exchange

CONFIG: enable nonBlockingExchange as default (for testing)

- changes Pstream::exchangeSizes to use NBX instead of all-to-all

applications/utilities/finiteArea/checkFaMesh/faMeshWriteVTK.H

@@ -5,7 +5,7 @@
     \\  /    A nd           | www.openfoam.com
      \\/     M anipulation  |
 -------------------------------------------------------------------------------
-    Copyright (C) 2021-2022 OpenCFD Ltd.
+    Copyright (C) 2021-2023 OpenCFD Ltd.
 -------------------------------------------------------------------------------
 License
     This file is part of OpenFOAM, distributed under GPL-3.0-or-later.
@@ -76,9 +76,9 @@ Description
     writer.beginCellData(4);
     writer.writeProcIDs();
     {
-        // Use primitive patch order
         Field<scalar> fld
         (
+            // Use primitive patch order
             faMeshTools::flattenEdgeField(aMesh.magLe(), true)
         );
         writer.write("magLe", fld);
@@ -93,11 +93,17 @@ Description
 }
 
 {
+    const Field<vector> edgeCentres
+    (
+        // Use primitive patch order
+        faMeshTools::flattenEdgeField(aMesh.edgeCentres(), true)
+    );
+
     // finiteArea - edgeCentres
     // (no other convenient way to display vectors on the edges)
     vtk::lineWriter writer
     (
-        aMesh.edgeCentres(),
+        edgeCentres,
         edgeList::null(),
         // vtk::formatType::INLINE_ASCII,
         fileName
@@ -109,19 +115,20 @@ Description
     writer.writeGeometry();
 
     // PointData
-    writer.beginPointData(4);
+    writer.beginPointData(3);
+    writer.writeProcIDs();  // Unfortunately cannot threshold on points
     {
-        // Use primitive patch order
         Field<vector> fld
         (
+            // Use primitive patch order
             faMeshTools::flattenEdgeField(aMesh.Le(), true)
         );
         writer.write("Le", fld);
     }
     {
-        // Use primitive patch order
         Field<vector> fld
         (
+            // Use primitive patch order
             faMeshTools::flattenEdgeField(aMesh.edgeAreaNormals(), true)
         );
         writer.write("normal", fld);

bin/mpirunDebug

@@ -7,7 +7,7 @@
 #    \\/     M anipulation  |
 #------------------------------------------------------------------------------
 #     Copyright (C) 2011-2015 OpenFOAM Foundation
-#     Copyright (C) 2017-2021 OpenCFD Ltd.
+#     Copyright (C) 2017-2023 OpenCFD Ltd.
 #------------------------------------------------------------------------------
 # License
 #     This file is part of OpenFOAM, distributed under GPL-3.0-or-later.
@@ -45,6 +45,7 @@ options:
   -local        Same as -spawn=1
   -remote       Same as -spawn=2
   -clean        Remove old processor*.{log,sh} files, mpirun.schema etc
+  -decompose-dict=<file>   Specific decomposeParDict name
   -help         Print the usage
 
 Invoke mpirun with separate per-processor log files or running in
@@ -111,7 +112,7 @@ Linux)
 esac
 
 unset appName appArgs nProcs
-unset method spawn optClean
+unset method spawn optClean optValue
 optConfirm=true
 
 decompDict="system/decomposeParDict"
@@ -126,7 +127,7 @@ do
     then
         knownOption=true  # Assume success
         case "$1" in
-        '') ;;  # ignore junk
+        ('') ;;  # Ignore junk
 
         -clean)     optClean=true ;;
         -yes)       unset optConfirm ;;
@@ -173,14 +174,25 @@ do
             shift
             ;;
 
+        (-decompose-dict=*)
+            optValue="${1#*=}"
+            case "$optValue" in
+            ('' | none | false) ;;  ## Ignore
+            (*)
+                decompDict="$optValue"
+                appArgs="${appArgs}${appArgs:+ }-decomposeParDict '$decompDict'"
+                ;;
+            esac
+            ;;
+
         -decomposeParDict)
             # Grab values and add to args immediately
             decompDict="$2"
-            appArgs="${appArgs}${appArgs:+ }$1 '$2'"
             shift
+            appArgs="${appArgs}${appArgs:+ }-decomposeParDict '$decompDict'"
             ;;
 
-        *)
+        (*)
             knownOption=false  # Fallthrough to regular processing
             ;;
         esac
@@ -192,23 +204,24 @@ do
         fi
     fi
 
+    # Processing application arguments
     case "$1" in
-    -help* | --help*) usage ;;
-    '') ;;  # ignore junk
+    (-help* | --help*) usage ;;
+    ('') ;;  ## Ignore junk
 
-    -np)
+    (-np)
         nProcs="$2"
         shift
         ;;
 
-    -decomposeParDict)
+    (-decomposeParDict)
         # Grab values and add to args immediately
         decompDict="$2"
-        appArgs="${appArgs}${appArgs:+ }$1 '$2'"
+        appArgs="${appArgs}${appArgs:+ }-decomposeParDict '$decompDict'"
         shift
         ;;
 
-    *)
+    (*)
         if [ -z "$appName" ]
         then
             appName="$1"

bin/tools/RunFunctions

@@ -6,7 +6,7 @@
 #    \\/     M anipulation  |
 #------------------------------------------------------------------------------
 #     Copyright (C) 2011-2016 OpenFOAM Foundation
-#     Copyright (C) 2015-2022 OpenCFD Ltd.
+#     Copyright (C) 2015-2023 OpenCFD Ltd.
 #------------------------------------------------------------------------------
 # License
 #     This file is part of OpenFOAM, distributed under GPL-3.0-or-later.
@@ -180,6 +180,16 @@ getNumberOfProcessors()
 {
     local dict="${1:-system/decomposeParDict}"
 
+    case "$dict" in
+    (system/*)  # Already qualified
+        ;;
+    (*)
+        # If it does not exist, assume it refers to location in system/
+        [ -f "$dict" ] || dict="system/$dict"
+        ;;
+    esac
+
+
     # Re-use positional parameters for automatic whitespace elimination
     set -- $(foamDictionary -entry numberOfSubdomains -value "$dict" 2>/dev/null)
 
@@ -223,7 +233,7 @@ getApplication()
 #
 runApplication()
 {
-    local appName appRun logFile logMode
+    local appName appRun optValue logFile logMode
 
     # Any additional parsed arguments (eg, decomposeParDict)
     local appArgs
@@ -232,25 +242,39 @@ runApplication()
     while [ "$#" -gt 0 ] && [ -z "$appRun" ]
     do
         case "$1" in
-            -a | -append)
-                logMode=append
-                ;;
-            -o | -overwrite)
-                logMode=overwrite
-                ;;
-            -s | -suffix)
-                logFile=".$2"
-                shift
-                ;;
-            -decomposeParDict)
-                appArgs="$appArgs $1 $2"
-                shift
-                ;;
-            '')
-                ;;
-            *)
-                appRun="$1"
-                ;;
+        ('') ;;  # Ignore junk
+
+        (-a | -append)
+            logMode=append
+            ;;
+        (-o | -overwrite)
+            logMode=overwrite
+            ;;
+        (-s | -suffix)
+            logFile=".$2"
+            shift
+            ;;
+
+        (-decompose-dict=*)
+            optValue="${1#*=}"
+            case "$optValue" in
+            ('' | none | false) ;;  ## Ignore
+            (*) appArgs="$appArgs -decomposeParDict $optValue" ;;
+            esac
+            ;;
+
+        (-decomposeParDict)
+            optValue="$2"
+            shift
+            case "$optValue" in
+            ('' | none | false) ;; ## Ignore
+            (*) appArgs="$appArgs -decomposeParDict $optValue" ;;
+            esac
+            ;;
+
+        (*)
+            appRun="$1"
+            ;;
         esac
         shift
     done
@@ -280,7 +304,7 @@ runApplication()
 #
 runParallel()
 {
-    local appName appRun logFile logMode nProcs
+    local appName appRun optValue logFile logMode nProcs
 
     # Any additional parsed arguments (eg, decomposeParDict)
     local appArgs="-parallel"
@@ -295,30 +319,46 @@ runParallel()
     while [ "$#" -gt 0 ] && [ -z "$appRun" ]
     do
         case "$1" in
-            -a | -append)
-                logMode=append
+        ('') ;;  # Ignore junk
+
+        (-a | -append)
+            logMode=append
+            ;;
+        (-o | -overwrite)
+            logMode=overwrite
+            ;;
+        (-s | -suffix)
+            logFile=".$2"
+            shift
+            ;;
+        (-n | -np)
+            nProcs="$2"
+            shift
+            ;;
+        (-decompose-dict=*)
+            optValue="${1#*=}"
+            case "$optValue" in
+            ('' | none | false) ;;  ## Ignore
+            (*)
+                appArgs="$appArgs -decomposeParDict $optValue"
+                nProcs="$(getNumberOfProcessors "$optValue")"
                 ;;
-            -o | -overwrite)
-                logMode=overwrite
-                ;;
-            -s | -suffix)
-                logFile=".$2"
-                shift
-                ;;
-            -n | -np)
-                nProcs="$2"
-                shift
-                ;;
-            -decomposeParDict)
-                appArgs="$appArgs $1 $2"
-                nProcs=$(getNumberOfProcessors "$2")
-                shift
-                ;;
-            '')
-                ;;
-            *)
-                appRun="$1"
+            esac
+            ;;
+        (-decomposeParDict)
+            optValue="$2"
+            shift
+            case "$optValue" in
+            ('' | none | false) ;;  ## Ignore
+            (*)
+                appArgs="$appArgs -decomposeParDict $optValue"
+                nProcs="$(getNumberOfProcessors "$optValue")"
                 ;;
+            esac
+            ;;
+         (*)
+            appRun="$1"
+            ;;
         esac
         shift
     done

etc/controlDict

@@ -130,7 +130,9 @@ OptimisationSwitches
     // Number processors to change to tree communication
     nProcsSimpleSum 0;
     // Min numProc to use non-blocking exchange algorithm (Hoeffler: NBX)
-    nonBlockingExchange 0;
+    nonBlockingExchange 1;
+    // Use hybrid NBX/PEX for PstreamBuffers
+    pbufs.algorithm 0;
 
     // MPI buffer size (bytes)
     // Can override with the MPI_BUFFER_SIZE env variable.

src/OpenFOAM/db/IOstreams/Pstreams/Pstream.H

@@ -470,8 +470,7 @@ public:
 
     // Exchange
 
-        //- Helper: exchange sizes of sendBufs for specified
-        //- set of send/receive processes.
+        //- Helper: exchange sizes of sendBufs for specified send/recv ranks
         template<class Container>
         static void exchangeSizes
         (
@@ -483,6 +482,17 @@ public:
             const label comm = UPstream::worldComm
         );
 
+        //- Helper: exchange sizes of sendBufs for specified neighbour ranks
+        template<class Container>
+        static void exchangeSizes
+        (
+            const labelUList& neighProcs,
+            const Container& sendBufs,
+            labelList& sizes,
+            const label tag = UPstream::msgType(),
+            const label comm = UPstream::worldComm
+        );
+
         //- Helper: exchange sizes of sendBufs.
         //- The sendBufs is the data per processor (in the communicator).
         //  Returns sizes of sendBufs on the sending processor.

src/OpenFOAM/db/IOstreams/Pstreams/PstreamBuffers.C

@@ -27,7 +27,30 @@ License
 \*---------------------------------------------------------------------------*/
 
 #include "PstreamBuffers.H"
-#include "bitSet.H"
+#include "debug.H"
+#include "registerSwitch.H"
+
+// * * * * * * * * * * * * * * Static Data Members * * * * * * * * * * * * * //
+
+int Foam::PstreamBuffers::algorithm
+(
+    // Not really the most creative name...
+    Foam::debug::optimisationSwitch("pbufs.algorithm", 0)
+);
+registerOptSwitch
+(
+    "pbufs.algorithm",
+    int,
+    Foam::PstreamBuffers::algorithm
+);
+
+
+// Simple enumerations
+// -------------------
+static constexpr int algorithm_PEX_allToAll = -1;  // OpenFOAM 2212 and earlier
+//static constexpr int algorithm_PEX_hybrid = 0;   // New default?
+static constexpr int algorithm_full_NBX = 1;       // Experimental
+
 
 // * * * * * * * * * * * * * Private Member Functions  * * * * * * * * * * * //
 
@@ -43,9 +66,58 @@ void Foam::PstreamBuffers::finalExchange
 
     if (commsType_ == UPstream::commsTypes::nonBlocking)
     {
-        // Dense storage uses all-to-all
-        Pstream::exchangeSizes(sendBuffers_, recvSizes, comm_);
+        if
+        (
+            wait
+         && (algorithm >= algorithm_full_NBX)
+         && (UPstream::maxCommsSize <= 0)
+        )
+        {
+            // NBX algorithm (nonblocking exchange)
+            // - when requested and waiting, no data chunking etc
 
+            PstreamDetail::exchangeConsensus<DynamicList<char>, char>
+            (
+                sendBuffers_,
+                recvBuffers_,
+                recvSizes,
+                (tag_ + 271828),  // some unique tag?
+                comm_,
+                wait
+            );
+
+            return;
+        }
+
+
+        // PEX algorithm with two different flavours of exchanging sizes
+
+        // Assemble the send sizes (cf. Pstream::exchangeSizes)
+        labelList sendSizes(nProcs_);
+        forAll(sendBuffers_, proci)
+        {
+            sendSizes[proci] = sendBuffers_[proci].size();
+        }
+        recvSizes.resize_nocopy(nProcs_);
+
+        if (algorithm == algorithm_PEX_allToAll)
+        {
+            // PEX stage 1: exchange sizes (all-to-all)
+            UPstream::allToAll(sendSizes, recvSizes, comm_);
+        }
+        else
+        {
+            // PEX stage 1: exchange sizes (non-blocking consensus)
+            UPstream::allToAllConsensus
+            (
+                sendSizes,
+                recvSizes,
+                (tag_ + 314159),  // some unique tag?
+                comm_
+            );
+        }
+
+        // PEX stage 2: point-to-point data exchange
         Pstream::exchange<DynamicList<char>, char>
         (
             sendBuffers_,
@@ -61,8 +133,7 @@ void Foam::PstreamBuffers::finalExchange
 
 void Foam::PstreamBuffers::finalExchange
 (
-    const labelUList& sendProcs,
-    const labelUList& recvProcs,
+    const labelUList& neighProcs,
     const bool wait,
     labelList& recvSizes
 )
@@ -73,16 +144,40 @@ void Foam::PstreamBuffers::finalExchange
 
     if (commsType_ == UPstream::commsTypes::nonBlocking)
     {
+        // Preparation. Temporarily abuse recvSizes as logic to clear
+        // send buffers that are not in the neighbourhood connection
+        {
+            recvSizes.resize_nocopy(nProcs_);
+            recvSizes = 0;
+
+            // Preserve self-send, even if not described by neighbourhood
+            recvSizes[UPstream::myProcNo(comm_)] = 1;
+
+            for (const label proci : neighProcs)
+            {
+                recvSizes[proci] = 1;  // Connected
+            }
+
+            for (label proci=0; proci < nProcs_; ++proci)
+            {
+                if (!recvSizes[proci])  // Not connected
+                {
+                    sendBuffers_[proci].clear();
+                }
+            }
+        }
+
+        // PEX stage 1: exchange sizes (limited neighbourhood)
         Pstream::exchangeSizes
         (
-            sendProcs,
-            recvProcs,
+            neighProcs,
             sendBuffers_,
             recvSizes,
             tag_,
             comm_
         );
 
+        // PEX stage 2: point-to-point data exchange
         Pstream::exchange<DynamicList<char>, char>
         (
             sendBuffers_,
@@ -96,11 +191,10 @@ void Foam::PstreamBuffers::finalExchange
 }
 
 
-void Foam::PstreamBuffers::finalExchangeGatherScatter
+void Foam::PstreamBuffers::finalGatherScatter
 (
     const bool isGather,
     const bool wait,
-    const bool needSizes,
     labelList& recvSizes
 )
 {
@@ -133,6 +227,12 @@ void Foam::PstreamBuffers::finalExchangeGatherScatter
 
     if (commsType_ == UPstream::commsTypes::nonBlocking)
     {
+        // Use PEX algorithm
+        // - for a non-sparse gather/scatter, it is presumed that
+        // MPI_Gather/MPI_Scatter will be the most efficient way to
+        // communicate the sizes.
+
+        // PEX stage 1: exchange sizes (gather or scatter)
         if (isGather)
         {
             // gather mode (all-to-one): master [0] <- everyone
@@ -166,7 +266,7 @@ void Foam::PstreamBuffers::finalExchangeGatherScatter
             recvSizes[0] = myRecv;
         }
 
-
+        // PEX stage 2: point-to-point data exchange
         Pstream::exchange<DynamicList<char>, char>
         (
             sendBuffers_,
@@ -378,7 +478,7 @@ Foam::label Foam::PstreamBuffers::recvDataCount(const label proci) const
 
 Foam::labelList Foam::PstreamBuffers::recvDataCounts() const
 {
-    labelList counts(recvPositions_.size(), Zero);
+    labelList counts(nProcs_, Zero);
 
     if (finishedSendsCalled_)
     {
@@ -404,17 +504,20 @@ Foam::labelList Foam::PstreamBuffers::recvDataCounts() const
 }
 
 
-Foam::label Foam::PstreamBuffers::maxNonLocalRecvCount(const label proci) const
+Foam::label Foam::PstreamBuffers::maxNonLocalRecvCount
+(
+    const label excludeProci
+) const
 {
     label maxLen = 0;
 
     if (finishedSendsCalled_)
     {
-        forAll(recvBuffers_, idx)
+        forAll(recvBuffers_, proci)
         {
-            const label len(recvBuffers_[idx].size() - recvPositions_[idx]);
-            if (idx != proci)
+            if (excludeProci != proci)
             {
+                label len(recvBuffers_[proci].size() - recvPositions_[proci]);
                 maxLen = max(maxLen, len);
             }
         }
@@ -494,6 +597,9 @@ void Foam::PstreamBuffers::finishedSends
     const bool wait
 )
 {
+    // Resize for copying back
+    recvSizes.resize_nocopy(sendBuffers_.size());
+
     finalExchange(wait, recvSizes);
 
     if (commsType_ != UPstream::commsTypes::nonBlocking)
@@ -510,142 +616,39 @@ void Foam::PstreamBuffers::finishedSends
 }
 
 
-void Foam::PstreamBuffers::finishedSends
+void Foam::PstreamBuffers::finishedNeighbourSends
 (
-    const labelUList& sendProcs,
-    const labelUList& recvProcs,
+    const labelUList& neighProcs,
+    labelList& recvSizes,
+    const bool wait
+)
+{
+    finalExchange(neighProcs, wait, recvSizes);
+}
+
+
+void Foam::PstreamBuffers::finishedNeighbourSends
+(
+    const labelUList& neighProcs,
     const bool wait
 )
 {
     labelList recvSizes;
-    finalExchange(sendProcs, recvProcs, wait, recvSizes);
-}
-
-
-void Foam::PstreamBuffers::finishedSends
-(
-    const labelUList& sendProcs,
-    const labelUList& recvProcs,
-    labelList& recvSizes,
-    const bool wait
-)
-{
-    finalExchange(sendProcs, recvProcs, wait, recvSizes);
-
-    if (commsType_ != UPstream::commsTypes::nonBlocking)
-    {
-        FatalErrorInFunction
-            << "Obtaining sizes not supported in "
-            << UPstream::commsTypeNames[commsType_] << endl
-            << " since transfers already in progress. Use non-blocking instead."
-            << exit(FatalError);
-
-        // Note: maybe possible only if using different tag from write started
-        // by ~UOPstream. Needs some work.
-    }
-}
-
-
-bool Foam::PstreamBuffers::finishedSends
-(
-    bitSet& sendConnections,
-    DynamicList<label>& sendProcs,
-    DynamicList<label>& recvProcs,
-    const bool wait
-)
-{
-    bool changed = (sendConnections.size() != nProcs());
-
-    if (changed)
-    {
-        sendConnections.resize(nProcs());
-    }
-
-    // Update send connections
-    // - reasonable to assume there are no self-sends on UPstream::myProcNo
-    forAll(sendBuffers_, proci)
-    {
-        // ie, sendDataCount(proci) != 0
-        if (sendConnections.set(proci, !sendBuffers_[proci].empty()))
-        {
-            // The state changed
-            changed = true;
-        }
-    }
-
-    UPstream::reduceOr(changed, comm_);
-
-    if (changed)
-    {
-        // Create send/recv topology
-
-        // The send ranks
-        sendProcs.clear();
-        forAll(sendBuffers_, proci)
-        {
-            // ie, sendDataCount(proci) != 0
-            if (!sendBuffers_[proci].empty())
-            {
-                sendProcs.push_back(proci);
-            }
-        }
-
-        labelList recvSizes;
-        finishedSends(recvSizes, wait);  // All-to-all
-
-        // The recv ranks
-        recvProcs.clear();
-        forAll(recvSizes, proci)
-        {
-            if (recvSizes[proci] > 0)
-            {
-                recvProcs.push_back(proci);
-            }
-        }
-    }
-    else
-    {
-        // Use existing send/recv ranks
-
-        finishedSends(sendProcs, recvProcs, wait);
-    }
-
-    return changed;
-}
-
-
-void Foam::PstreamBuffers::finishedNeighbourSends
-(
-    const labelUList& neighProcs,
-    labelList& recvSizes,
-    const bool wait
-)
-{
-    finishedSends(neighProcs, neighProcs, recvSizes, wait);
-}
-
-
-void Foam::PstreamBuffers::finishedNeighbourSends
-(
-    const labelUList& neighProcs,
-    const bool wait
-)
-{
-    finishedSends(neighProcs, neighProcs, wait);
+    finalExchange(neighProcs, wait, recvSizes);
 }
 
 
 void Foam::PstreamBuffers::finishedGathers(const bool wait)
 {
     labelList recvSizes;
-    finalExchangeGatherScatter(true, wait, false, recvSizes);
+    finalGatherScatter(true, wait, recvSizes);
 }
 
 
 void Foam::PstreamBuffers::finishedScatters(const bool wait)
 {
     labelList recvSizes;
-    finalExchangeGatherScatter(false, wait, false, recvSizes);
+    finalGatherScatter(false, wait, recvSizes);
 }
 
 
@@ -655,7 +658,7 @@ void Foam::PstreamBuffers::finishedGathers
     const bool wait
 )
 {
-    finalExchangeGatherScatter(true, wait, true, recvSizes);
+    finalGatherScatter(true, wait, recvSizes);
 
     if (commsType_ != UPstream::commsTypes::nonBlocking)
     {
@@ -677,7 +680,7 @@ void Foam::PstreamBuffers::finishedScatters
     const bool wait
 )
 {
-    finalExchangeGatherScatter(false, wait, true, recvSizes);
+    finalGatherScatter(false, wait, recvSizes);
 
     if (commsType_ != UPstream::commsTypes::nonBlocking)
     {

src/OpenFOAM/db/IOstreams/Pstreams/PstreamBuffers.H

@@ -108,9 +108,6 @@ SourceFiles
 namespace Foam
 {
 
-// Forward Declarations
-class bitSet;
-
 /*---------------------------------------------------------------------------*\
                        Class PstreamBuffers Declaration
 \*---------------------------------------------------------------------------*/
@@ -164,22 +161,20 @@ class PstreamBuffers
         );
 
         //- Mark sends as done.
-        //  Only exchange sizes using the sendProcs/recvProcs subset
+        //  Only exchange sizes using the neighbour ranks
         //  (nonBlocking comms).
         void finalExchange
         (
-            const labelUList& sendProcs,
-            const labelUList& recvProcs,
+            const labelUList& neighProcs,
             const bool wait,
             labelList& recvSizes
         );
 
         //- For all-to-one or one-to-all
-        void finalExchangeGatherScatter
+        void finalGatherScatter
         (
             const bool isGather,
             const bool wait,
-            const bool needSizes,   // If recvSizes needed or scratch
             labelList& recvSizes
         );
 
@@ -202,6 +197,12 @@ class PstreamBuffers
 
 public:
 
+    // Static Data
+
+        //- Preferred exchange algorithm (may change or be removed in future)
+        static int algorithm;
+
+
     // Constructors
 
         //- Construct given communication type (default: nonBlocking), message
@@ -340,7 +341,7 @@ public:
 
         //- Maximum receive size, excluding the specified processor rank
         //- Must call finishedSends() or other finished.. method first!
-        label maxNonLocalRecvCount(const label proci) const;
+        label maxNonLocalRecvCount(const label excludeProci) const;
 
         //- Number of unconsumed receive bytes for the specified processor.
         //- Must call finishedSends() or other finished.. method first!
@@ -397,60 +398,6 @@ public:
 
     // Functions with restricted neighbours
 
-        //- Mark sends as done using subset of send/recv ranks
-        //- to exchange data on.
-        //
-        //  Non-blocking mode: populates receive buffers.
-        //  \param sendProcs ranks used for sends
-        //  \param recvProcs ranks used for recvs
-        //  \param wait wait for requests to complete (in nonBlocking mode)
-        //
-        //  \warning currently only valid for nonBlocking comms.
-        void finishedSends
-        (
-            const labelUList& sendProcs,
-            const labelUList& recvProcs,
-            const bool wait = true
-        );
-
-        //- Mark sends as done using subset of send/recv ranks
-        //- to exchange data on. Recovers the sizes (bytes) received.
-        //
-        //  Non-blocking mode: populates receive buffers.
-        //
-        //  \param sendProcs ranks used for sends
-        //  \param recvProcs ranks used for recvs
-        //  \param[out] recvSizes the sizes (bytes) received
-        //  \param wait wait for requests to complete (in nonBlocking mode)
-        //
-        //  \warning currently only valid for nonBlocking comms.
-        void finishedSends
-        (
-            const labelUList& sendProcs,
-            const labelUList& recvProcs,
-            labelList& recvSizes,
-            const bool wait = true
-        );
-
-        //- A caching version that uses a limited send/recv connectivity.
-        //
-        //  Non-blocking mode: populates receive buffers.
-        //  \param sendConnections on/off for sending ranks
-        //  \param sendProcs ranks used for sends
-        //  \param recvProcs ranks used for recvs
-        //  \param wait wait for requests to complete (in nonBlocking mode)
-        //
-        //  \return True if the send/recv connectivity changed
-        //
-        //  \warning currently only valid for nonBlocking comms.
-        bool finishedSends
-        (
-            bitSet& sendConnections,
-            DynamicList<label>& sendProcs,
-            DynamicList<label>& recvProcs,
-            const bool wait = true
-        );
-
         //- Mark sends as done using subset of send/recv ranks
         //- and recover the sizes (bytes) received.
         //

src/OpenFOAM/db/IOstreams/Pstreams/PstreamExchange.C

@@ -746,58 +746,89 @@ void Foam::Pstream::exchangeSizes
     const label comm
 )
 {
-    //const label myProci = UPstream::myProcNo(comm);
+    const label myProci = UPstream::myProcNo(comm);
+    const label numProcs = UPstream::nProcs(comm);
 
-    if (sendBufs.size() != UPstream::nProcs(comm))
+    if (sendBufs.size() != numProcs)
     {
         FatalErrorInFunction
             << "Size of container " << sendBufs.size()
-            << " does not equal the number of processors "
-            << UPstream::nProcs(comm)
+            << " does not equal the number of processors " << numProcs
             << Foam::abort(FatalError);
     }
 
-    labelList sendSizes(sendProcs.size());
-    forAll(sendProcs, i)
+    labelList sendSizes(numProcs);
+    for (label proci = 0; proci < numProcs; ++proci)
     {
-        sendSizes[i] = sendBufs[sendProcs[i]].size();
+        sendSizes[proci] = sendBufs[proci].size();
     }
 
-    recvSizes.resize_nocopy(sendBufs.size());
+    recvSizes.resize_nocopy(numProcs);
     recvSizes = 0;  // Ensure non-received entries are properly zeroed
 
+    // Preserve self-send, even if not described by neighbourhood
+    recvSizes[myProci] = sendSizes[myProci];
+
     const label startOfRequests = UPstream::nRequests();
 
     for (const label proci : recvProcs)
     {
-        UIPstream::read
-        (
-            UPstream::commsTypes::nonBlocking,
-            proci,
-            reinterpret_cast<char*>(&recvSizes[proci]),
-            sizeof(label),
-            tag,
-            comm
-        );
+        if (proci != myProci)
+        {
+            UIPstream::read
+            (
+                UPstream::commsTypes::nonBlocking,
+                proci,
+                reinterpret_cast<char*>(&recvSizes[proci]),
+                sizeof(label),
+                tag,
+                comm
+            );
+        }
     }
 
-    forAll(sendProcs, i)
+    for (const label proci : sendProcs)
     {
-        UOPstream::write
-        (
-            UPstream::commsTypes::nonBlocking,
-            sendProcs[i],
-            reinterpret_cast<char*>(&sendSizes[i]),
-            sizeof(label),
-            tag,
-            comm
-        );
+        if (proci != myProci)
+        {
+            UOPstream::write
+            (
+                UPstream::commsTypes::nonBlocking,
+                proci,
+                reinterpret_cast<char*>(&sendSizes[proci]),
+                sizeof(label),
+                tag,
+                comm
+            );
+        }
     }
 
     UPstream::waitRequests(startOfRequests);
 }
 
 
+template<class Container>
+void Foam::Pstream::exchangeSizes
+(
+    const labelUList& neighProcs,
+    const Container& sendBufs,
+    labelList& recvSizes,
+    const label tag,
+    const label comm
+)
+{
+    Pstream::exchangeSizes<Container>
+    (
+        neighProcs,  // send
+        neighProcs,  // recv
+        sendBufs,
+        recvSizes,
+        tag,
+        comm
+    );
+}
+
+
 // Sparse sending
 template<class Container>
 void Foam::Pstream::exchangeSizes

src/OpenFOAM/global/argList/argList.C

@@ -1720,6 +1720,8 @@ void Foam::argList::parse
                 << "    nProcsSimpleSum    : " << Pstream::nProcsSimpleSum << nl
                 << "    nonBlockingExchange: "
                 << Pstream::nProcsNonblockingExchange << nl
+                << "    exchange algorithm : "
+                << PstreamBuffers::algorithm << nl
                 << "    commsType          : "
                 << Pstream::commsTypeNames[Pstream::defaultCommsType] << nl
                 << "    polling iterations : " << Pstream::nPollProcInterfaces

src/OpenFOAM/parallel/globalIndex/globalIndex.C

@@ -214,7 +214,7 @@ void Foam::globalIndex::reset
             // Non-parallel branch: use localSize on-proc, zero elsewhere
 
             localLens.resize(len, Zero);
-            localLens[Pstream::myProcNo(comm)] = localSize;
+            localLens[UPstream::myProcNo(comm)] = localSize;
         }
 
         reset(localLens, true);  // checkOverflow = true

src/OpenFOAM/parallel/globalIndex/globalIndex.H

@@ -519,7 +519,9 @@ public:
         {
             List<Type> allData;
             gather(offsets, comm, procIDs, fld, allData, tag, ct);
-            if (Pstream::myProcNo(comm) == procIDs[0])
+
+            const int masterProci = procIDs.size() ? procIDs[0] : 0;
+            if (UPstream::myProcNo(comm) == masterProci)
             {
                 fld.transfer(allData);
             }

src/OpenFOAM/parallel/globalIndex/globalIndexI.H

@@ -137,14 +137,14 @@ inline Foam::globalIndex::globalIndex
 
 inline bool Foam::globalIndex::empty() const
 {
-    return offsets_.empty() || offsets_.last() == 0;
+    return offsets_.empty() || offsets_.back() == 0;
 }
 
 
 inline Foam::label Foam::globalIndex::totalSize() const
 {
     const label len = (offsets_.size() - 1);
-    return (len < 1) ? static_cast<label>(0) : offsets_[len];
+    return (len < 1) ? 0 : offsets_[len];
 }
 
 
@@ -163,7 +163,7 @@ inline Foam::labelList Foam::globalIndex::sizes() const
 inline Foam::label Foam::globalIndex::nProcs() const noexcept
 {
     const label len = (offsets_.size() - 1);
-    return (len < 1) ? static_cast<label>(0) : len;
+    return (len < 1) ? 0 : len;
 }
 
 
@@ -219,7 +219,7 @@ inline Foam::label Foam::globalIndex::localStart(const label proci) const
 
 inline Foam::label Foam::globalIndex::localStart() const
 {
-    return localStart(Pstream::myProcNo());
+    return localStart(UPstream::myProcNo());
 }
 
 
@@ -231,7 +231,7 @@ inline Foam::label Foam::globalIndex::localSize(const label proci) const
 
 inline Foam::label Foam::globalIndex::localSize() const
 {
-    return localSize(Pstream::myProcNo());
+    return localSize(UPstream::myProcNo());
 }
 
 
@@ -244,7 +244,7 @@ inline Foam::label Foam::globalIndex::maxSize() const
 
 inline Foam::label Foam::globalIndex::maxNonLocalSize() const
 {
-    return maxNonLocalSize(Pstream::myProcNo());
+    return maxNonLocalSize(UPstream::myProcNo());
 }
 
 
@@ -256,7 +256,7 @@ inline Foam::labelRange Foam::globalIndex::range(const label proci) const
 
 inline Foam::labelRange Foam::globalIndex::range() const
 {
-    return range(Pstream::myProcNo());
+    return range(UPstream::myProcNo());
 }
 
 
@@ -268,7 +268,7 @@ inline bool Foam::globalIndex::isLocal(const label proci, const label i) const
 
 inline bool Foam::globalIndex::isLocal(const label i) const
 {
-    return isLocal(Pstream::myProcNo(), i);
+    return isLocal(UPstream::myProcNo(), i);
 }
 
 
@@ -284,7 +284,7 @@ inline Foam::label Foam::globalIndex::toGlobal
 
 inline Foam::label Foam::globalIndex::toGlobal(const label i) const
 {
-    return toGlobal(Pstream::myProcNo(), i);
+    return toGlobal(UPstream::myProcNo(), i);
 }
 
 
@@ -306,7 +306,7 @@ inline Foam::labelList Foam::globalIndex::toGlobal
     const labelUList& labels
 ) const
 {
-    return toGlobal(Pstream::myProcNo(), labels);
+    return toGlobal(UPstream::myProcNo(), labels);
 }
 
 
@@ -327,7 +327,7 @@ inline void Foam::globalIndex::inplaceToGlobal
 
 inline void Foam::globalIndex::inplaceToGlobal(labelUList& labels) const
 {
-    inplaceToGlobal(Pstream::myProcNo(), labels);
+    inplaceToGlobal(UPstream::myProcNo(), labels);
 }
 
 
@@ -349,7 +349,7 @@ Foam::globalIndex::toLocal(const label proci, const label i) const
 
 inline Foam::label Foam::globalIndex::toLocal(const label i) const
 {
-    return toLocal(Pstream::myProcNo(), i);
+    return toLocal(UPstream::myProcNo(), i);
 }
 
 
@@ -363,7 +363,7 @@ inline Foam::label Foam::globalIndex::whichProcID(const label i) const
             << abort(FatalError);
     }
 
-    const label proci(Pstream::myProcNo());
+    const int proci = UPstream::myProcNo();
 
     return isLocal(proci, i) ? proci : findLower(offsets_, i+1);
 }

src/OpenFOAM/parallel/globalIndex/globalIndexTemplates.C

@@ -91,7 +91,9 @@ void Foam::globalIndex::gatherValues
 
     const label startOfRequests = UPstream::nRequests();
 
-    if (UPstream::myProcNo(comm) == procIDs[0])
+    const int masterProci = procIDs.size() ? procIDs[0] : 0;
+
+    if (UPstream::myProcNo(comm) == masterProci)
     {
         allValues.resize_nocopy(procIDs.size());
         allValues[0] = localValue;
@@ -126,7 +128,7 @@ void Foam::globalIndex::gatherValues
             UOPstream::write
             (
                 commsType,
-                procIDs[0],
+                masterProci,
                 reinterpret_cast<const char*>(&localValue),
                 sizeof(Type),
                 tag,
@@ -135,7 +137,7 @@ void Foam::globalIndex::gatherValues
         }
         else
         {
-            OPstream toMaster(commsType, procIDs[0], 0, tag, comm);
+            OPstream toMaster(commsType, masterProci, 0, tag, comm);
             toMaster << localValue;
         }
     }
@@ -176,9 +178,11 @@ void Foam::globalIndex::gather
 
     const label startOfRequests = UPstream::nRequests();
 
-    if (Pstream::myProcNo(comm) == procIDs[0])
+    const int masterProci = procIDs.size() ? procIDs[0] : 0;
+
+    if (UPstream::myProcNo(comm) == masterProci)
     {
-        allFld.resize_nocopy(off.last());  // == totalSize()
+        allFld.resize_nocopy(off.back());  // == totalSize()
 
         // Assign my local data - respect offset information
         // so that we can request 0 entries to be copied.
@@ -226,7 +230,7 @@ void Foam::globalIndex::gather
             UOPstream::write
             (
                 commsType,
-                procIDs[0],
+                masterProci,
                 fld.cdata_bytes(),
                 fld.size_bytes(),
                 tag,
@@ -235,7 +239,7 @@ void Foam::globalIndex::gather
         }
         else
         {
-            OPstream toMaster(commsType, procIDs[0], 0, tag, comm);
+            OPstream toMaster(commsType, masterProci, 0, tag, comm);
             toMaster << fld;
         }
     }
@@ -294,9 +298,11 @@ void Foam::globalIndex::gather
 
     const label startOfRequests = UPstream::nRequests();
 
-    if (Pstream::myProcNo(comm) == procIDs[0])
+    const int masterProci = procIDs.size() ? procIDs[0] : 0;
+
+    if (UPstream::myProcNo(comm) == masterProci)
     {
-        allFld.resize_nocopy(off.last());  // == totalSize()
+        allFld.resize_nocopy(off.back());  // == totalSize()
 
         // Assign my local data - respect offset information
         // so that we can request 0 entries to be copied
@@ -331,7 +337,7 @@ void Foam::globalIndex::gather
         }
         else
         {
-            OPstream toMaster(commsType, procIDs[0], 0, tag, comm);
+            OPstream toMaster(commsType, masterProci, 0, tag, comm);
             toMaster << fld;
         }
     }
@@ -877,7 +883,9 @@ void Foam::globalIndex::scatter
 
     const label startOfRequests = UPstream::nRequests();
 
-    if (Pstream::myProcNo(comm) == procIDs[0])
+    const int masterProci = procIDs.size() ? procIDs[0] : 0;
+
+    if (UPstream::myProcNo(comm) == masterProci)
     {
         for (label i = 1; i < procIDs.size(); ++i)
         {
@@ -929,7 +937,7 @@ void Foam::globalIndex::scatter
             UIPstream::read
             (
                 commsType,
-                procIDs[0],
+                masterProci,
                 fld.data_bytes(),
                 fld.size_bytes(),
                 tag,
@@ -938,7 +946,7 @@ void Foam::globalIndex::scatter
         }
         else
         {
-            IPstream fromMaster(commsType, procIDs[0], 0, tag, comm);
+            IPstream fromMaster(commsType, masterProci, 0, tag, comm);
             fromMaster >> fld;
         }
     }

src/fileFormats/vtk/file/foamVtkFileWriter.C

@@ -5,7 +5,7 @@
     \\  /    A nd           | www.openfoam.com
      \\/     M anipulation  |
 -------------------------------------------------------------------------------
-    Copyright (C) 2018-2022 OpenCFD Ltd.
+    Copyright (C) 2018-2023 OpenCFD Ltd.
 -------------------------------------------------------------------------------
 License
     This file is part of OpenFOAM.
@@ -76,11 +76,11 @@ Foam::Ostream& Foam::vtk::fileWriter::reportBadState
 Foam::Ostream& Foam::vtk::fileWriter::reportBadState
 (
     Ostream& os,
-    outputState expected,
+    outputState expected1,
     outputState expected2
 ) const
 {
-    reportBadState(os, expected)
+    reportBadState(os, expected1)
         << " or (" << stateNames[expected2] << ')';
     return os;
 }
@@ -554,10 +554,18 @@ bool Foam::vtk::fileWriter::writeProcIDs(const label nValues)
     {
         ++nCellData_;
     }
+    else if (isState(outputState::POINT_DATA))
+    {
+        ++nPointData_;
+    }
     else
     {
-        reportBadState(FatalErrorInFunction, outputState::CELL_DATA)
-            << " for procID field" << nl << endl
+        reportBadState
+        (
+            FatalErrorInFunction,
+            outputState::CELL_DATA,
+            outputState::POINT_DATA
+        )   << " for procID field" << nl << endl
             << exit(FatalError);
 
         return false;

src/fileFormats/vtk/file/foamVtkFileWriter.H

@@ -5,7 +5,7 @@
     \\  /    A nd           | www.openfoam.com
      \\/     M anipulation  |
 -------------------------------------------------------------------------------
-    Copyright (C) 2018-2022 OpenCFD Ltd.
+    Copyright (C) 2018-2023 OpenCFD Ltd.
 -------------------------------------------------------------------------------
 License
     This file is part of OpenFOAM.
@@ -201,7 +201,8 @@ protected:
             const UList<Type>& field
         );
 
-        //- Write nValues of processor ids as CellData (no-op in serial)
+        //- Write nValues of processor ids as CellData or PointData
+        //- (no-op in serial)
         bool writeProcIDs(const label nValues);
 
 
@@ -305,10 +306,16 @@ public:
         //  \return True if the state changed
         virtual bool beginPointData(label nFields = 0) = 0;
 
-        //- Return the number of CellData written for the Piece thus far.
+        //- True if output state corresponds to CELL_DATA
+        inline bool isCellData() const noexcept;
+
+        //- True if output state corresponds to POINT_DATA
+        inline bool isPointData() const noexcept;
+
+        //- The number of CellData written for the Piece thus far.
         inline label nCellData() const noexcept;
 
-        //- Return the number of PointData written for the Piece thus far.
+        //- The number of PointData written for the Piece thus far.
         inline label nPointData() const noexcept;
 
 

src/fileFormats/vtk/file/foamVtkFileWriterI.H

@@ -5,7 +5,7 @@
     \\  /    A nd           | www.openfoam.com
      \\/     M anipulation  |
 -------------------------------------------------------------------------------
-    Copyright (C) 2018-2022 OpenCFD Ltd.
+    Copyright (C) 2018-2023 OpenCFD Ltd.
 -------------------------------------------------------------------------------
 License
     This file is part of OpenFOAM.
@@ -107,4 +107,16 @@ inline Foam::label Foam::vtk::fileWriter::nPointData() const noexcept
 }
 
 
+inline bool Foam::vtk::fileWriter::isCellData() const noexcept
+{
+    return (outputState::CELL_DATA == state_);
+}
+
+
+inline bool Foam::vtk::fileWriter::isPointData() const noexcept
+{
+    return (outputState::POINT_DATA == state_);
+}
+
+
 // ************************************************************************* //

src/fileFormats/vtk/write/foamVtkLineWriter.C

@@ -5,7 +5,7 @@
     \\  /    A nd           | www.openfoam.com
      \\/     M anipulation  |
 -------------------------------------------------------------------------------
-    Copyright (C) 2021 OpenCFD Ltd.
+    Copyright (C) 2021-2023 OpenCFD Ltd.
 -------------------------------------------------------------------------------
 License
     This file is part of OpenFOAM.
@@ -131,6 +131,10 @@ void Foam::vtk::lineWriter::piece
 
 bool Foam::vtk::lineWriter::writeProcIDs()
 {
+    if (this->isPointData())
+    {
+        return vtk::fileWriter::writeProcIDs(nLocalPoints_);
+    }
     return vtk::fileWriter::writeProcIDs(nLocalLines_);
 }
 

src/fileFormats/vtk/write/foamVtkLineWriter.H

@@ -5,7 +5,7 @@
     \\  /    A nd           | www.openfoam.com
      \\/     M anipulation  |
 -------------------------------------------------------------------------------
-    Copyright (C) 2021 OpenCFD Ltd.
+    Copyright (C) 2021-2023 OpenCFD Ltd.
 -------------------------------------------------------------------------------
 License
     This file is part of OpenFOAM.
@@ -155,8 +155,8 @@ public:
         void piece(const pointField& points, const edgeList& edges);
 
 
-        //- Write processor ids for each line as CellData
-        //- (no-op in serial)
+        //- Write processor ids for each line as CellData or for each point
+        //- as PointData, depending on isPointData() state. No-op in serial.
         bool writeProcIDs();
 
         //- Write a uniform field of Cell (Line) or Point values

src/fileFormats/vtk/write/foamVtkSurfaceWriter.C

@@ -5,7 +5,7 @@
     \\  /    A nd           | www.openfoam.com
      \\/     M anipulation  |
 -------------------------------------------------------------------------------
-    Copyright (C) 2018-2021 OpenCFD Ltd.
+    Copyright (C) 2018-2023 OpenCFD Ltd.
 -------------------------------------------------------------------------------
 License
     This file is part of OpenFOAM.
@@ -131,6 +131,10 @@ void Foam::vtk::surfaceWriter::piece
 
 bool Foam::vtk::surfaceWriter::writeProcIDs()
 {
+    if (this->isPointData())
+    {
+        return vtk::fileWriter::writeProcIDs(nLocalPoints_);
+    }
     return vtk::fileWriter::writeProcIDs(nLocalPolys_);
 }
 

src/fileFormats/vtk/write/foamVtkSurfaceWriter.H

@@ -5,7 +5,7 @@
     \\  /    A nd           | www.openfoam.com
      \\/     M anipulation  |
 -------------------------------------------------------------------------------
-    Copyright (C) 2018-2021 OpenCFD Ltd.
+    Copyright (C) 2018-2023 OpenCFD Ltd.
 -------------------------------------------------------------------------------
 License
     This file is part of OpenFOAM.
@@ -155,8 +155,8 @@ public:
         void piece(const pointField& points, const faceList& faces);
 
 
-        //- Write processor ids for each poly as CellData
-        //- (no-op in serial)
+        //- Write processor ids for each poly as CellData or for each point
+        //- as PointData, depending on isPointData() state. No-op in serial.
         bool writeProcIDs();
 
         //- Write a uniform field of Cell (Poly) or Point values

src/finiteVolume/fvMesh/zoneDistribute/zoneDistribute.C

@@ -109,22 +109,22 @@ void Foam::zoneDistribute::setUpCommforZone
             }
         }
 
-        // Stream the send data into PstreamBuffers,
-        // which we also use to track the current topology.
-
+        // Stream the send data into PstreamBuffers
         pBufs_.clear();
 
-        for (const int proci : UPstream::allProcs())
+        for (const int proci : pBufs_.allProcs())
         {
-            if (proci != UPstream::myProcNo() && !needed[proci].empty())
+            const auto& indices = needed[proci];
+
+            if (proci != UPstream::myProcNo() && !indices.empty())
             {
                 // Serialize as List
                 UOPstream toProc(proci, pBufs_);
-                toProc << needed[proci].sortedToc();
+                toProc << indices.sortedToc();
             }
         }
 
-        pBufs_.finishedSends(sendConnections_, sendProcs_, recvProcs_);
+        pBufs_.finishedSends();
 
         for (const int proci : pBufs_.allProcs())
         {

src/finiteVolume/fvMesh/zoneDistribute/zoneDistribute.H

@@ -93,15 +93,6 @@ class zoneDistribute
         //- Global cell/face index to send for processor-to-processor comms
         List<labelList> send_;
 
-        //- Parallel [cache]: send connectivity (true/false)
-        bitSet sendConnections_;
-
-        //- Parallel [cache]: send data to these ranks
-        DynamicList<label> sendProcs_;
-
-        //- Parallel [cache]: recv data from these ranks
-        DynamicList<label> recvProcs_;
-
         //- Persistent set of exchange buffers
         PstreamBuffers pBufs_;
 

src/finiteVolume/fvMesh/zoneDistribute/zoneDistributeI.H

@@ -6,7 +6,7 @@
      \\/     M anipulation  |
 -------------------------------------------------------------------------------
     Copyright (C) 2019-2020 DLR
-    Copyright (C) 2020-2022 OpenCFD Ltd.
+    Copyright (C) 2020-2023 OpenCFD Ltd.
 -------------------------------------------------------------------------------
 License
     This file is part of OpenFOAM.
@@ -161,28 +161,18 @@ Foam::Map<Type> Foam::zoneDistribute::getDatafromOtherProc
 
     if (UPstream::parRun())
     {
-        if (sendConnections_.empty())
-        {
-            WarningInFunction
-                << "The send/recv connections not initialized - "
-                << "likely that setUpCommforZone() was not called"
-                << endl;
-            // But don't exit/abort for now
-        }
-
-        // Stream the send data into PstreamBuffers,
-        // which we also use to track the current topology.
-
         pBufs_.clear();
 
-        for (const int proci : UPstream::allProcs())
+        for (const int proci : pBufs_.allProcs())
         {
-            if (proci != UPstream::myProcNo() && !send_[proci].empty())
+            const auto& indices = send_[proci];
+
+            if (proci != UPstream::myProcNo() && !indices.empty())
             {
                 // Serialize as Map
-                Map<Type> sendValues(2*send_[proci].size());
+                Map<Type> sendValues(2*indices.size());
 
-                for (const label sendIdx : send_[proci])
+                for (const label sendIdx : indices)
                 {
                     sendValues.insert
                     (
@@ -196,7 +186,7 @@ Foam::Map<Type> Foam::zoneDistribute::getDatafromOtherProc
             }
         }
 
-        pBufs_.finishedSends(sendConnections_, sendProcs_, recvProcs_);
+        pBufs_.finishedSends();
 
         for (const int proci : pBufs_.allProcs())
         {

src/meshTools/coordSet/writers/vtk/foamVtkCoordSetWriter.H

@@ -201,8 +201,9 @@ public:
         void piece(const UPtrList<const pointField>& points);
 
 
-        //- Write processor ids for each line as CellData
-        //- (no-op in serial)
+        //- Write processor ids for each line as CellData or for each point
+        //- as PointData, depending on isPointData() state. No-op in serial.
+        //  Not implemented.
         bool writeProcIDs();
 
 

src/meshTools/output/vtk/patch/foamVtkGenericPatchWriter.H

@@ -5,7 +5,7 @@
     \\  /    A nd           | www.openfoam.com
      \\/     M anipulation  |
 -------------------------------------------------------------------------------
-    Copyright (C) 2018-2021 OpenCFD Ltd.
+    Copyright (C) 2018-2023 OpenCFD Ltd.
 -------------------------------------------------------------------------------
 License
     This file is part of OpenFOAM.
@@ -159,6 +159,10 @@ public:
         //- Write processor ids for each poly as CellData
         bool writeProcIDs()
         {
+            if (this->isPointData())
+            {
+                return vtk::fileWriter::writeProcIDs(nLocalPoints_);
+            }
             return vtk::polyWriter::writeProcIDs(nLocalPolys_);
         }
 };

src/meshTools/output/vtk/patch/foamVtkPatchMeshWriter.C

@@ -5,7 +5,7 @@
     \\  /    A nd           | www.openfoam.com
      \\/     M anipulation  |
 -------------------------------------------------------------------------------
-    Copyright (C) 2016-2022 OpenCFD Ltd.
+    Copyright (C) 2016-2023 OpenCFD Ltd.
 -------------------------------------------------------------------------------
 License
     This file is part of OpenFOAM.
@@ -585,6 +585,10 @@ void Foam::vtk::patchMeshWriter::writePatchIDs()
 
 bool Foam::vtk::patchMeshWriter::writeProcIDs()
 {
+    if (this->isPointData())
+    {
+        return vtk::fileWriter::writeProcIDs(nLocalPoints_);
+    }
     return vtk::fileWriter::writeProcIDs(nLocalPolys_);
 }
 

src/meshTools/output/vtk/patch/foamVtkPatchMeshWriter.H

@@ -5,7 +5,7 @@
     \\  /    A nd           | www.openfoam.com
      \\/     M anipulation  |
 -------------------------------------------------------------------------------
-    Copyright (C) 2016-2022 OpenCFD Ltd.
+    Copyright (C) 2016-2023 OpenCFD Ltd.
 -------------------------------------------------------------------------------
 License
     This file is part of OpenFOAM.
@@ -209,8 +209,8 @@ public:
         //  Must be called within the CELL_DATA state.
         void writePatchIDs();
 
-        //- Write processor ids as CellData. This is no-op in serial.
-        //  Must be called within the CELL_DATA state.
+        //- Write processor ids for each line as CellData or for each point
+        //- as PointData, depending on isPointData() state. No-op in serial.
         bool writeProcIDs();
 
         //- Write processor neighbour ids as CellData. This is no-op in serial.

tutorials/lagrangian/kinematicParcelFoam/drippingChair/Allrun

@@ -3,19 +3,30 @@ cd "${0%/*}" || exit                                # Run from this directory
 . ${WM_PROJECT_DIR:?}/bin/tools/RunFunctions        # Tutorial run functions
 #------------------------------------------------------------------------------
 
+unset decompDict
+# decompDict=system/decomposeParDict-6
+# decompDict=system/decomposeParDict-7
+
 runApplication blockMesh
 
 runApplication makeFaMesh
 
-runApplication decomposePar
+runApplication -decompose-dict="$decompDict" decomposePar
 
-# For the processor partitioning
-runParallel -s finiteVolume foamToVTK -name VTK-parallel -time 0 \
-    -no-finite-area -no-internal -no-lagrangian -no-fields -with-ids
+# For ids and processor partitioning
+runParallel -s volume -decompose-dict="$decompDict" \
+    foamToVTK -name VTK-parallel -time 0 \
+    -no-finite-area -no-internal -no-lagrangian -no-fields -with-ids \
+    -patches filmWalls
 
-runParallel $(getApplication)
+runParallel -decompose-dict="$decompDict" $(getApplication)
 
-runParallel -s finiteArea foamToVTK -name VTK-parallel \
-    -no-boundary -no-internal -no-lagrangian
+if false
+then
+    # Not usually needed - uses areaWrite
+    runParallel -s area -decompose-dict="$decompDict" \
+        foamToVTK -name VTK-parallel \
+        -no-boundary -no-internal -no-lagrangian -with-ids
+fi
 
 #------------------------------------------------------------------------------

tutorials/lagrangian/kinematicParcelFoam/drippingChair/Allrun-serial

@@ -7,13 +7,18 @@ runApplication blockMesh
 
 runApplication makeFaMesh
 
-# For the cell ids etc
-runApplication -s finiteVolume.serial foamToVTK -name VTK-serial -time 0 \
-    -no-finite-area -no-internal -no-lagrangian -no-fields -with-ids
+# For ids and processor partitioning
+runApplication -s volume.serial foamToVTK -name VTK-serial -time 0 \
+    -no-finite-area -no-internal -no-lagrangian -no-fields -with-ids \
+    -patches filmWalls
 
 runApplication $(getApplication)
 
-runApplication -s finiteArea.serial foamToVTK -name VTK-serial \
-    -no-boundary -no-internal -no-lagrangian
+if false
+then
+    # Not usually needed - uses areaWrite
+    runApplication -s area.serial foamToVTK -name VTK-serial \
+        -no-boundary -no-internal -no-lagrangian
+fi
 
 #------------------------------------------------------------------------------

tutorials/lagrangian/kinematicParcelFoam/drippingChair/system/areaWrite

@@ -0,0 +1,25 @@
+// -*- C++ -*-
+// Use the areaWrite function object
+
+areaWrite
+{
+    type    areaWrite;
+    libs    (utilityFunctionObjects);
+    log     true;
+
+    writeControl    writeTime;
+    writeInterval   1;
+
+    // Fields to output (words or regex)
+    fields  (Uf_film hf_film pf_film);
+
+    surfaceFormat ensight;
+
+    formatOptions
+    {
+        default { format binary; }
+    }
+}
+
+
+// ************************************************************************* //

tutorials/lagrangian/kinematicParcelFoam/drippingChair/system/controlDict

@@ -52,5 +52,10 @@ regionFaMaxCo   1;
 
 maxDeltaT       0.1;
 
+functions
+{
+    #include "areaWrite"
+}
+
 
 // ************************************************************************* //

tutorials/lagrangian/kinematicParcelFoam/drippingChair/system/decomposeParDict-6

@@ -0,0 +1,21 @@
+/*--------------------------------*- C++ -*----------------------------------*\
+| =========                 |                                                 |
+| \\      /  F ield         | OpenFOAM: The Open Source CFD Toolbox           |
+|  \\    /   O peration     | Version:  v2306                                 |
+|   \\  /    A nd           | Website:  www.openfoam.com                      |
+|    \\/     M anipulation  |                                                 |
+\*---------------------------------------------------------------------------*/
+FoamFile
+{
+    version     2.0;
+    format      ascii;
+    class       dictionary;
+    object      decomposeParDict;
+}
+// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
+
+numberOfSubdomains 6;
+
+method          scotch;
+
+// ************************************************************************* //

tutorials/lagrangian/kinematicParcelFoam/drippingChair/system/decomposeParDict-7

@@ -0,0 +1,21 @@
+/*--------------------------------*- C++ -*----------------------------------*\
+| =========                 |                                                 |
+| \\      /  F ield         | OpenFOAM: The Open Source CFD Toolbox           |
+|  \\    /   O peration     | Version:  v2306                                 |
+|   \\  /    A nd           | Website:  www.openfoam.com                      |
+|    \\/     M anipulation  |                                                 |
+\*---------------------------------------------------------------------------*/
+FoamFile
+{
+    version     2.0;
+    format      ascii;
+    class       dictionary;
+    object      decomposeParDict;
+}
+// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
+
+numberOfSubdomains 7;
+
+method          scotch;
+
+// ************************************************************************* //