ENH: improve efficiency of point-cell/cell-point construction (#2715)

- calculate point-cell from point-face if available

applications/utilities/finiteArea/checkFaMesh/faMeshWriteVTK.H

@@ -5,7 +5,7 @@
     \\  /    A nd           | www.openfoam.com
      \\/     M anipulation  |
 -------------------------------------------------------------------------------
-    Copyright (C) 2021-2023 OpenCFD Ltd.
+    Copyright (C) 2021-2022 OpenCFD Ltd.
 -------------------------------------------------------------------------------
 License
     This file is part of OpenFOAM, distributed under GPL-3.0-or-later.
@@ -76,9 +76,9 @@ Description
     writer.beginCellData(4);
     writer.writeProcIDs();
     {
+        // Use primitive patch order
         Field<scalar> fld
         (
-            // Use primitive patch order
             faMeshTools::flattenEdgeField(aMesh.magLe(), true)
         );
         writer.write("magLe", fld);
@@ -93,17 +93,11 @@ Description
 }
 
 {
-    const Field<vector> edgeCentres
-    (
-        // Use primitive patch order
-        faMeshTools::flattenEdgeField(aMesh.edgeCentres(), true)
-    );
-
     // finiteArea - edgeCentres
     // (no other convenient way to display vectors on the edges)
     vtk::lineWriter writer
     (
-        edgeCentres,
+        aMesh.edgeCentres(),
         edgeList::null(),
         // vtk::formatType::INLINE_ASCII,
         fileName
@@ -115,20 +109,19 @@ Description
     writer.writeGeometry();
 
     // PointData
-    writer.beginPointData(3);
-    writer.writeProcIDs();  // Unfortunately cannot threshold on points
+    writer.beginPointData(4);
     {
+        // Use primitive patch order
         Field<vector> fld
         (
-            // Use primitive patch order
             faMeshTools::flattenEdgeField(aMesh.Le(), true)
         );
         writer.write("Le", fld);
     }
     {
+        // Use primitive patch order
         Field<vector> fld
         (
-            // Use primitive patch order
             faMeshTools::flattenEdgeField(aMesh.edgeAreaNormals(), true)
         );
         writer.write("normal", fld);

bin/mpirunDebug

@@ -7,7 +7,7 @@
 #    \\/     M anipulation  |
 #------------------------------------------------------------------------------
 #     Copyright (C) 2011-2015 OpenFOAM Foundation
-#     Copyright (C) 2017-2023 OpenCFD Ltd.
+#     Copyright (C) 2017-2021 OpenCFD Ltd.
 #------------------------------------------------------------------------------
 # License
 #     This file is part of OpenFOAM, distributed under GPL-3.0-or-later.
@@ -45,7 +45,6 @@ options:
   -local        Same as -spawn=1
   -remote       Same as -spawn=2
   -clean        Remove old processor*.{log,sh} files, mpirun.schema etc
-  -decompose-dict=<file>   Specific decomposeParDict name
   -help         Print the usage
 
 Invoke mpirun with separate per-processor log files or running in
@@ -112,7 +111,7 @@ Linux)
 esac
 
 unset appName appArgs nProcs
-unset method spawn optClean optValue
+unset method spawn optClean
 optConfirm=true
 
 decompDict="system/decomposeParDict"
@@ -127,7 +126,7 @@ do
     then
         knownOption=true  # Assume success
         case "$1" in
-        ('') ;;  # Ignore junk
+        '') ;;  # ignore junk
 
         -clean)     optClean=true ;;
         -yes)       unset optConfirm ;;
@@ -174,25 +173,14 @@ do
             shift
             ;;
 
-        (-decompose-dict=*)
-            optValue="${1#*=}"
-            case "$optValue" in
-            ('' | none | false) ;;  ## Ignore
-            (*)
-                decompDict="$optValue"
-                appArgs="${appArgs}${appArgs:+ }-decomposeParDict '$decompDict'"
-                ;;
-            esac
-            ;;
-
         -decomposeParDict)
             # Grab values and add to args immediately
             decompDict="$2"
+            appArgs="${appArgs}${appArgs:+ }$1 '$2'"
             shift
-            appArgs="${appArgs}${appArgs:+ }-decomposeParDict '$decompDict'"
             ;;
 
-        (*)
+        *)
             knownOption=false  # Fallthrough to regular processing
             ;;
         esac
@@ -204,24 +192,23 @@ do
         fi
     fi
 
-    # Processing application arguments
     case "$1" in
-    (-help* | --help*) usage ;;
-    ('') ;;  ## Ignore junk
+    -help* | --help*) usage ;;
+    '') ;;  # ignore junk
 
-    (-np)
+    -np)
         nProcs="$2"
         shift
         ;;
 
-    (-decomposeParDict)
+    -decomposeParDict)
         # Grab values and add to args immediately
         decompDict="$2"
-        appArgs="${appArgs}${appArgs:+ }-decomposeParDict '$decompDict'"
+        appArgs="${appArgs}${appArgs:+ }$1 '$2'"
         shift
         ;;
 
-    (*)
+    *)
         if [ -z "$appName" ]
         then
             appName="$1"

bin/tools/RunFunctions

@@ -6,7 +6,7 @@
 #    \\/     M anipulation  |
 #------------------------------------------------------------------------------
 #     Copyright (C) 2011-2016 OpenFOAM Foundation
-#     Copyright (C) 2015-2023 OpenCFD Ltd.
+#     Copyright (C) 2015-2022 OpenCFD Ltd.
 #------------------------------------------------------------------------------
 # License
 #     This file is part of OpenFOAM, distributed under GPL-3.0-or-later.
@@ -180,16 +180,6 @@ getNumberOfProcessors()
 {
     local dict="${1:-system/decomposeParDict}"
 
-    case "$dict" in
-    (system/*)  # Already qualified
-        ;;
-    (*)
-        # If it does not exist, assume it refers to location in system/
-        [ -f "$dict" ] || dict="system/$dict"
-        ;;
-    esac
-
-
     # Re-use positional parameters for automatic whitespace elimination
     set -- $(foamDictionary -entry numberOfSubdomains -value "$dict" 2>/dev/null)
 
@@ -233,7 +223,7 @@ getApplication()
 #
 runApplication()
 {
-    local appName appRun optValue logFile logMode
+    local appName appRun logFile logMode
 
     # Any additional parsed arguments (eg, decomposeParDict)
     local appArgs
@@ -242,39 +232,25 @@ runApplication()
     while [ "$#" -gt 0 ] && [ -z "$appRun" ]
     do
         case "$1" in
-        ('') ;;  # Ignore junk
-
-        (-a | -append)
-            logMode=append
-            ;;
-        (-o | -overwrite)
-            logMode=overwrite
-            ;;
-        (-s | -suffix)
-            logFile=".$2"
-            shift
-            ;;
-
-        (-decompose-dict=*)
-            optValue="${1#*=}"
-            case "$optValue" in
-            ('' | none | false) ;;  ## Ignore
-            (*) appArgs="$appArgs -decomposeParDict $optValue" ;;
-            esac
-            ;;
-
-        (-decomposeParDict)
-            optValue="$2"
-            shift
-            case "$optValue" in
-            ('' | none | false) ;; ## Ignore
-            (*) appArgs="$appArgs -decomposeParDict $optValue" ;;
-            esac
-            ;;
-
-        (*)
-            appRun="$1"
-            ;;
+            -a | -append)
+                logMode=append
+                ;;
+            -o | -overwrite)
+                logMode=overwrite
+                ;;
+            -s | -suffix)
+                logFile=".$2"
+                shift
+                ;;
+            -decomposeParDict)
+                appArgs="$appArgs $1 $2"
+                shift
+                ;;
+            '')
+                ;;
+            *)
+                appRun="$1"
+                ;;
         esac
         shift
     done
@@ -304,7 +280,7 @@ runApplication()
 #
 runParallel()
 {
-    local appName appRun optValue logFile logMode nProcs
+    local appName appRun logFile logMode nProcs
 
     # Any additional parsed arguments (eg, decomposeParDict)
     local appArgs="-parallel"
@@ -319,46 +295,30 @@ runParallel()
     while [ "$#" -gt 0 ] && [ -z "$appRun" ]
     do
         case "$1" in
-        ('') ;;  # Ignore junk
-
-        (-a | -append)
-            logMode=append
-            ;;
-        (-o | -overwrite)
-            logMode=overwrite
-            ;;
-        (-s | -suffix)
-            logFile=".$2"
-            shift
-            ;;
-        (-n | -np)
-            nProcs="$2"
-            shift
-            ;;
-        (-decompose-dict=*)
-            optValue="${1#*=}"
-            case "$optValue" in
-            ('' | none | false) ;;  ## Ignore
-            (*)
-                appArgs="$appArgs -decomposeParDict $optValue"
-                nProcs="$(getNumberOfProcessors "$optValue")"
+            -a | -append)
+                logMode=append
                 ;;
-            esac
-            ;;
-        (-decomposeParDict)
-            optValue="$2"
-            shift
-            case "$optValue" in
-            ('' | none | false) ;;  ## Ignore
-            (*)
-                appArgs="$appArgs -decomposeParDict $optValue"
-                nProcs="$(getNumberOfProcessors "$optValue")"
+            -o | -overwrite)
+                logMode=overwrite
+                ;;
+            -s | -suffix)
+                logFile=".$2"
+                shift
+                ;;
+            -n | -np)
+                nProcs="$2"
+                shift
+                ;;
+            -decomposeParDict)
+                appArgs="$appArgs $1 $2"
+                nProcs=$(getNumberOfProcessors "$2")
+                shift
+                ;;
+            '')
+                ;;
+            *)
+                appRun="$1"
                 ;;
-            esac
-            ;;
-         (*)
-            appRun="$1"
-            ;;
         esac
         shift
     done

etc/controlDict

@@ -130,9 +130,7 @@ OptimisationSwitches
     // Number processors to change to tree communication
     nProcsSimpleSum 0;
     // Min numProc to use non-blocking exchange algorithm (Hoeffler: NBX)
-    nonBlockingExchange 1;
-    // Use hybrid NBX/PEX for PstreamBuffers
-    pbufs.algorithm 0;
+    nonBlockingExchange 0;
 
     // MPI buffer size (bytes)
     // Can override with the MPI_BUFFER_SIZE env variable.

src/OpenFOAM/db/IOstreams/Pstreams/Pstream.H

@@ -470,7 +470,8 @@ public:
 
     // Exchange
 
-        //- Helper: exchange sizes of sendBufs for specified send/recv ranks
+        //- Helper: exchange sizes of sendBufs for specified
+        //- set of send/receive processes.
         template<class Container>
         static void exchangeSizes
         (
@@ -482,17 +483,6 @@ public:
             const label comm = UPstream::worldComm
         );
 
-        //- Helper: exchange sizes of sendBufs for specified neighbour ranks
-        template<class Container>
-        static void exchangeSizes
-        (
-            const labelUList& neighProcs,
-            const Container& sendBufs,
-            labelList& sizes,
-            const label tag = UPstream::msgType(),
-            const label comm = UPstream::worldComm
-        );
-
         //- Helper: exchange sizes of sendBufs.
         //- The sendBufs is the data per processor (in the communicator).
         //  Returns sizes of sendBufs on the sending processor.

src/OpenFOAM/db/IOstreams/Pstreams/PstreamBuffers.C

@@ -27,30 +27,7 @@ License
 \*---------------------------------------------------------------------------*/
 
 #include "PstreamBuffers.H"
-#include "debug.H"
-#include "registerSwitch.H"
-
-// * * * * * * * * * * * * * * Static Data Members * * * * * * * * * * * * * //
-
-int Foam::PstreamBuffers::algorithm
-(
-    // Not really the most creative name...
-    Foam::debug::optimisationSwitch("pbufs.algorithm", 0)
-);
-registerOptSwitch
-(
-    "pbufs.algorithm",
-    int,
-    Foam::PstreamBuffers::algorithm
-);
-
-
-// Simple enumerations
-// -------------------
-static constexpr int algorithm_PEX_allToAll = -1;  // OpenFOAM 2212 and earlier
-//static constexpr int algorithm_PEX_hybrid = 0;   // New default?
-static constexpr int algorithm_full_NBX = 1;       // Experimental
-
+#include "bitSet.H"
 
 // * * * * * * * * * * * * * Private Member Functions  * * * * * * * * * * * //
 
@@ -66,58 +43,9 @@ void Foam::PstreamBuffers::finalExchange
 
     if (commsType_ == UPstream::commsTypes::nonBlocking)
     {
-        if
-        (
-            wait
-         && (algorithm >= algorithm_full_NBX)
-         && (UPstream::maxCommsSize <= 0)
-        )
-        {
-            // NBX algorithm (nonblocking exchange)
-            // - when requested and waiting, no data chunking etc
+        // Dense storage uses all-to-all
+        Pstream::exchangeSizes(sendBuffers_, recvSizes, comm_);
 
-            PstreamDetail::exchangeConsensus<DynamicList<char>, char>
-            (
-                sendBuffers_,
-                recvBuffers_,
-                recvSizes,
-                (tag_ + 271828),  // some unique tag?
-                comm_,
-                wait
-            );
-
-            return;
-        }
-
-
-        // PEX algorithm with two different flavours of exchanging sizes
-
-        // Assemble the send sizes (cf. Pstream::exchangeSizes)
-        labelList sendSizes(nProcs_);
-        forAll(sendBuffers_, proci)
-        {
-            sendSizes[proci] = sendBuffers_[proci].size();
-        }
-        recvSizes.resize_nocopy(nProcs_);
-
-        if (algorithm == algorithm_PEX_allToAll)
-        {
-            // PEX stage 1: exchange sizes (all-to-all)
-            UPstream::allToAll(sendSizes, recvSizes, comm_);
-        }
-        else
-        {
-            // PEX stage 1: exchange sizes (non-blocking consensus)
-            UPstream::allToAllConsensus
-            (
-                sendSizes,
-                recvSizes,
-                (tag_ + 314159),  // some unique tag?
-                comm_
-            );
-        }
-
-        // PEX stage 2: point-to-point data exchange
         Pstream::exchange<DynamicList<char>, char>
         (
             sendBuffers_,
@@ -133,7 +61,8 @@ void Foam::PstreamBuffers::finalExchange
 
 void Foam::PstreamBuffers::finalExchange
 (
-    const labelUList& neighProcs,
+    const labelUList& sendProcs,
+    const labelUList& recvProcs,
     const bool wait,
     labelList& recvSizes
 )
@@ -144,40 +73,16 @@ void Foam::PstreamBuffers::finalExchange
 
     if (commsType_ == UPstream::commsTypes::nonBlocking)
     {
-        // Preparation. Temporarily abuse recvSizes as logic to clear
-        // send buffers that are not in the neighbourhood connection
-        {
-            recvSizes.resize_nocopy(nProcs_);
-            recvSizes = 0;
-
-            // Preserve self-send, even if not described by neighbourhood
-            recvSizes[UPstream::myProcNo(comm_)] = 1;
-
-            for (const label proci : neighProcs)
-            {
-                recvSizes[proci] = 1;  // Connected
-            }
-
-            for (label proci=0; proci < nProcs_; ++proci)
-            {
-                if (!recvSizes[proci])  // Not connected
-                {
-                    sendBuffers_[proci].clear();
-                }
-            }
-        }
-
-        // PEX stage 1: exchange sizes (limited neighbourhood)
         Pstream::exchangeSizes
         (
-            neighProcs,
+            sendProcs,
+            recvProcs,
             sendBuffers_,
             recvSizes,
             tag_,
             comm_
         );
 
-        // PEX stage 2: point-to-point data exchange
         Pstream::exchange<DynamicList<char>, char>
         (
             sendBuffers_,
@@ -191,10 +96,11 @@ void Foam::PstreamBuffers::finalExchange
 }
 
 
-void Foam::PstreamBuffers::finalGatherScatter
+void Foam::PstreamBuffers::finalExchangeGatherScatter
 (
     const bool isGather,
     const bool wait,
+    const bool needSizes,
     labelList& recvSizes
 )
 {
@@ -227,12 +133,6 @@ void Foam::PstreamBuffers::finalGatherScatter
 
     if (commsType_ == UPstream::commsTypes::nonBlocking)
     {
-        // Use PEX algorithm
-        // - for a non-sparse gather/scatter, it is presumed that
-        // MPI_Gather/MPI_Scatter will be the most efficient way to
-        // communicate the sizes.
-
-        // PEX stage 1: exchange sizes (gather or scatter)
         if (isGather)
         {
             // gather mode (all-to-one): master [0] <- everyone
@@ -266,7 +166,7 @@ void Foam::PstreamBuffers::finalGatherScatter
             recvSizes[0] = myRecv;
         }
 
-        // PEX stage 2: point-to-point data exchange
+
         Pstream::exchange<DynamicList<char>, char>
         (
             sendBuffers_,
@@ -478,7 +378,7 @@ Foam::label Foam::PstreamBuffers::recvDataCount(const label proci) const
 
 Foam::labelList Foam::PstreamBuffers::recvDataCounts() const
 {
-    labelList counts(nProcs_, Zero);
+    labelList counts(recvPositions_.size(), Zero);
 
     if (finishedSendsCalled_)
     {
@@ -504,20 +404,17 @@ Foam::labelList Foam::PstreamBuffers::recvDataCounts() const
 }
 
 
-Foam::label Foam::PstreamBuffers::maxNonLocalRecvCount
-(
-    const label excludeProci
-) const
+Foam::label Foam::PstreamBuffers::maxNonLocalRecvCount(const label proci) const
 {
     label maxLen = 0;
 
     if (finishedSendsCalled_)
     {
-        forAll(recvBuffers_, proci)
+        forAll(recvBuffers_, idx)
         {
-            if (excludeProci != proci)
+            const label len(recvBuffers_[idx].size() - recvPositions_[idx]);
+            if (idx != proci)
             {
-                label len(recvBuffers_[proci].size() - recvPositions_[proci]);
                 maxLen = max(maxLen, len);
             }
         }
@@ -597,9 +494,6 @@ void Foam::PstreamBuffers::finishedSends
     const bool wait
 )
 {
-    // Resize for copying back
-    recvSizes.resize_nocopy(sendBuffers_.size());
-
     finalExchange(wait, recvSizes);
 
     if (commsType_ != UPstream::commsTypes::nonBlocking)
@@ -616,6 +510,110 @@ void Foam::PstreamBuffers::finishedSends
 }
 
 
+void Foam::PstreamBuffers::finishedSends
+(
+    const labelUList& sendProcs,
+    const labelUList& recvProcs,
+    const bool wait
+)
+{
+    labelList recvSizes;
+    finalExchange(sendProcs, recvProcs, wait, recvSizes);
+}
+
+
+void Foam::PstreamBuffers::finishedSends
+(
+    const labelUList& sendProcs,
+    const labelUList& recvProcs,
+    labelList& recvSizes,
+    const bool wait
+)
+{
+    finalExchange(sendProcs, recvProcs, wait, recvSizes);
+
+    if (commsType_ != UPstream::commsTypes::nonBlocking)
+    {
+        FatalErrorInFunction
+            << "Obtaining sizes not supported in "
+            << UPstream::commsTypeNames[commsType_] << endl
+            << " since transfers already in progress. Use non-blocking instead."
+            << exit(FatalError);
+
+        // Note: maybe possible only if using different tag from write started
+        // by ~UOPstream. Needs some work.
+    }
+}
+
+
+bool Foam::PstreamBuffers::finishedSends
+(
+    bitSet& sendConnections,
+    DynamicList<label>& sendProcs,
+    DynamicList<label>& recvProcs,
+    const bool wait
+)
+{
+    bool changed = (sendConnections.size() != nProcs());
+
+    if (changed)
+    {
+        sendConnections.resize(nProcs());
+    }
+
+    // Update send connections
+    // - reasonable to assume there are no self-sends on UPstream::myProcNo
+    forAll(sendBuffers_, proci)
+    {
+        // ie, sendDataCount(proci) != 0
+        if (sendConnections.set(proci, !sendBuffers_[proci].empty()))
+        {
+            // The state changed
+            changed = true;
+        }
+    }
+
+    UPstream::reduceOr(changed, comm_);
+
+    if (changed)
+    {
+        // Create send/recv topology
+
+        // The send ranks
+        sendProcs.clear();
+        forAll(sendBuffers_, proci)
+        {
+            // ie, sendDataCount(proci) != 0
+            if (!sendBuffers_[proci].empty())
+            {
+                sendProcs.push_back(proci);
+            }
+        }
+
+        labelList recvSizes;
+        finishedSends(recvSizes, wait);  // All-to-all
+
+        // The recv ranks
+        recvProcs.clear();
+        forAll(recvSizes, proci)
+        {
+            if (recvSizes[proci] > 0)
+            {
+                recvProcs.push_back(proci);
+            }
+        }
+    }
+    else
+    {
+        // Use existing send/recv ranks
+
+        finishedSends(sendProcs, recvProcs, wait);
+    }
+
+    return changed;
+}
+
+
 void Foam::PstreamBuffers::finishedNeighbourSends
 (
     const labelUList& neighProcs,
@@ -623,7 +621,7 @@ void Foam::PstreamBuffers::finishedNeighbourSends
     const bool wait
 )
 {
-    finalExchange(neighProcs, wait, recvSizes);
+    finishedSends(neighProcs, neighProcs, recvSizes, wait);
 }
 
 
@@ -633,22 +631,21 @@ void Foam::PstreamBuffers::finishedNeighbourSends
     const bool wait
 )
 {
-    labelList recvSizes;
-    finalExchange(neighProcs, wait, recvSizes);
+    finishedSends(neighProcs, neighProcs, wait);
 }
 
 
 void Foam::PstreamBuffers::finishedGathers(const bool wait)
 {
     labelList recvSizes;
-    finalGatherScatter(true, wait, recvSizes);
+    finalExchangeGatherScatter(true, wait, false, recvSizes);
 }
 
 
 void Foam::PstreamBuffers::finishedScatters(const bool wait)
 {
     labelList recvSizes;
-    finalGatherScatter(false, wait, recvSizes);
+    finalExchangeGatherScatter(false, wait, false, recvSizes);
 }
 
 
@@ -658,7 +655,7 @@ void Foam::PstreamBuffers::finishedGathers
     const bool wait
 )
 {
-    finalGatherScatter(true, wait, recvSizes);
+    finalExchangeGatherScatter(true, wait, true, recvSizes);
 
     if (commsType_ != UPstream::commsTypes::nonBlocking)
     {
@@ -680,7 +677,7 @@ void Foam::PstreamBuffers::finishedScatters
     const bool wait
 )
 {
-    finalGatherScatter(false, wait, recvSizes);
+    finalExchangeGatherScatter(false, wait, true, recvSizes);
 
     if (commsType_ != UPstream::commsTypes::nonBlocking)
     {

src/OpenFOAM/db/IOstreams/Pstreams/PstreamBuffers.H

@@ -108,6 +108,9 @@ SourceFiles
 namespace Foam
 {
 
+// Forward Declarations
+class bitSet;
+
 /*---------------------------------------------------------------------------*\
                        Class PstreamBuffers Declaration
 \*---------------------------------------------------------------------------*/
@@ -161,20 +164,22 @@ class PstreamBuffers
         );
 
         //- Mark sends as done.
-        //  Only exchange sizes using the neighbour ranks
+        //  Only exchange sizes using the sendProcs/recvProcs subset
         //  (nonBlocking comms).
         void finalExchange
         (
-            const labelUList& neighProcs,
+            const labelUList& sendProcs,
+            const labelUList& recvProcs,
             const bool wait,
             labelList& recvSizes
         );
 
         //- For all-to-one or one-to-all
-        void finalGatherScatter
+        void finalExchangeGatherScatter
         (
             const bool isGather,
             const bool wait,
+            const bool needSizes,   // If recvSizes needed or scratch
             labelList& recvSizes
         );
 
@@ -197,12 +202,6 @@ class PstreamBuffers
 
 public:
 
-    // Static Data
-
-        //- Preferred exchange algorithm (may change or be removed in future)
-        static int algorithm;
-
-
     // Constructors
 
         //- Construct given communication type (default: nonBlocking), message
@@ -341,7 +340,7 @@ public:
 
         //- Maximum receive size, excluding the specified processor rank
         //- Must call finishedSends() or other finished.. method first!
-        label maxNonLocalRecvCount(const label excludeProci) const;
+        label maxNonLocalRecvCount(const label proci) const;
 
         //- Number of unconsumed receive bytes for the specified processor.
         //- Must call finishedSends() or other finished.. method first!
@@ -398,6 +397,60 @@ public:
 
     // Functions with restricted neighbours
 
+        //- Mark sends as done using subset of send/recv ranks
+        //- to exchange data on.
+        //
+        //  Non-blocking mode: populates receive buffers.
+        //  \param sendProcs ranks used for sends
+        //  \param recvProcs ranks used for recvs
+        //  \param wait wait for requests to complete (in nonBlocking mode)
+        //
+        //  \warning currently only valid for nonBlocking comms.
+        void finishedSends
+        (
+            const labelUList& sendProcs,
+            const labelUList& recvProcs,
+            const bool wait = true
+        );
+
+        //- Mark sends as done using subset of send/recv ranks
+        //- to exchange data on. Recovers the sizes (bytes) received.
+        //
+        //  Non-blocking mode: populates receive buffers.
+        //
+        //  \param sendProcs ranks used for sends
+        //  \param recvProcs ranks used for recvs
+        //  \param[out] recvSizes the sizes (bytes) received
+        //  \param wait wait for requests to complete (in nonBlocking mode)
+        //
+        //  \warning currently only valid for nonBlocking comms.
+        void finishedSends
+        (
+            const labelUList& sendProcs,
+            const labelUList& recvProcs,
+            labelList& recvSizes,
+            const bool wait = true
+        );
+
+        //- A caching version that uses a limited send/recv connectivity.
+        //
+        //  Non-blocking mode: populates receive buffers.
+        //  \param sendConnections on/off for sending ranks
+        //  \param sendProcs ranks used for sends
+        //  \param recvProcs ranks used for recvs
+        //  \param wait wait for requests to complete (in nonBlocking mode)
+        //
+        //  \return True if the send/recv connectivity changed
+        //
+        //  \warning currently only valid for nonBlocking comms.
+        bool finishedSends
+        (
+            bitSet& sendConnections,
+            DynamicList<label>& sendProcs,
+            DynamicList<label>& recvProcs,
+            const bool wait = true
+        );
+
         //- Mark sends as done using subset of send/recv ranks
         //- and recover the sizes (bytes) received.
         //

src/OpenFOAM/db/IOstreams/Pstreams/PstreamExchange.C

@@ -746,89 +746,58 @@ void Foam::Pstream::exchangeSizes
     const label comm
 )
 {
-    const label myProci = UPstream::myProcNo(comm);
-    const label numProcs = UPstream::nProcs(comm);
+    //const label myProci = UPstream::myProcNo(comm);
 
-    if (sendBufs.size() != numProcs)
+    if (sendBufs.size() != UPstream::nProcs(comm))
     {
         FatalErrorInFunction
             << "Size of container " << sendBufs.size()
-            << " does not equal the number of processors " << numProcs
+            << " does not equal the number of processors "
+            << UPstream::nProcs(comm)
             << Foam::abort(FatalError);
     }
 
-    labelList sendSizes(numProcs);
-    for (label proci = 0; proci < numProcs; ++proci)
+    labelList sendSizes(sendProcs.size());
+    forAll(sendProcs, i)
     {
-        sendSizes[proci] = sendBufs[proci].size();
+        sendSizes[i] = sendBufs[sendProcs[i]].size();
     }
 
-    recvSizes.resize_nocopy(numProcs);
+    recvSizes.resize_nocopy(sendBufs.size());
     recvSizes = 0;  // Ensure non-received entries are properly zeroed
 
-    // Preserve self-send, even if not described by neighbourhood
-    recvSizes[myProci] = sendSizes[myProci];
-
     const label startOfRequests = UPstream::nRequests();
 
     for (const label proci : recvProcs)
     {
-        if (proci != myProci)
-        {
-            UIPstream::read
-            (
-                UPstream::commsTypes::nonBlocking,
-                proci,
-                reinterpret_cast<char*>(&recvSizes[proci]),
-                sizeof(label),
-                tag,
-                comm
-            );
-        }
+        UIPstream::read
+        (
+            UPstream::commsTypes::nonBlocking,
+            proci,
+            reinterpret_cast<char*>(&recvSizes[proci]),
+            sizeof(label),
+            tag,
+            comm
+        );
     }
 
-    for (const label proci : sendProcs)
+    forAll(sendProcs, i)
     {
-        if (proci != myProci)
-        {
-            UOPstream::write
-            (
-                UPstream::commsTypes::nonBlocking,
-                proci,
-                reinterpret_cast<char*>(&sendSizes[proci]),
-                sizeof(label),
-                tag,
-                comm
-            );
-        }
+        UOPstream::write
+        (
+            UPstream::commsTypes::nonBlocking,
+            sendProcs[i],
+            reinterpret_cast<char*>(&sendSizes[i]),
+            sizeof(label),
+            tag,
+            comm
+        );
     }
 
     UPstream::waitRequests(startOfRequests);
 }
 
 
-template<class Container>
-void Foam::Pstream::exchangeSizes
-(
-    const labelUList& neighProcs,
-    const Container& sendBufs,
-    labelList& recvSizes,
-    const label tag,
-    const label comm
-)
-{
-    Pstream::exchangeSizes<Container>
-    (
-        neighProcs,  // send
-        neighProcs,  // recv
-        sendBufs,
-        recvSizes,
-        tag,
-        comm
-    );
-}
-
-
 // Sparse sending
 template<class Container>
 void Foam::Pstream::exchangeSizes

src/OpenFOAM/global/argList/argList.C

@@ -1720,8 +1720,6 @@ void Foam::argList::parse
                 << "    nProcsSimpleSum    : " << Pstream::nProcsSimpleSum << nl
                 << "    nonBlockingExchange: "
                 << Pstream::nProcsNonblockingExchange << nl
-                << "    exchange algorithm : "
-                << PstreamBuffers::algorithm << nl
                 << "    commsType          : "
                 << Pstream::commsTypeNames[Pstream::defaultCommsType] << nl
                 << "    polling iterations : " << Pstream::nPollProcInterfaces

src/OpenFOAM/meshes/primitiveMesh/primitiveMeshCellPoints.C

@@ -6,7 +6,7 @@
      \\/     M anipulation  |
 -------------------------------------------------------------------------------
     Copyright (C) 2011-2016 OpenFOAM Foundation
-    Copyright (C) 2018 OpenCFD Ltd.
+    Copyright (C) 2018-2023 OpenCFD Ltd.
 -------------------------------------------------------------------------------
 License
     This file is part of OpenFOAM.
@@ -29,6 +29,7 @@ License
 #include "primitiveMesh.H"
 #include "cell.H"
 #include "bitSet.H"
+#include "DynamicList.H"
 #include "ListOps.H"
 
 // * * * * * * * * * * * * * Private Member Functions  * * * * * * * * * * * //
@@ -57,12 +58,49 @@ void Foam::primitiveMesh::calcCellPoints() const
             << "cellPoints already calculated"
             << abort(FatalError);
     }
-    else
+    else if (hasPointCells())
     {
         // Invert pointCells
         cpPtr_ = new labelListList(nCells());
         invertManyToMany(nCells(), pointCells(), *cpPtr_);
     }
+    else
+    {
+        // Calculate cell-point topology
+
+        cpPtr_ = new labelListList(nCells());
+        auto& cellPointAddr = *cpPtr_;
+
+        const cellList& cellLst = cells();
+        const faceList& faceLst = faces();
+
+        // Vertex labels for the current cell
+        DynamicList<label> vertices(256);
+
+
+        const label loopLen = nCells();
+
+        for (label celli = 0; celli < loopLen; ++celli)
+        {
+            // Clear any previous contents
+            vertices.clear();
+
+            for (const label facei : cellLst[celli])
+            {
+                for (const label pointi : faceLst[facei])
+                {
+                    vertices.push_back(pointi);
+                }
+            }
+
+            // Sort + unique to eliminate duplicates
+            std::sort(vertices.begin(), vertices.end());
+            auto last = std::unique(vertices.begin(), vertices.end());
+            vertices.resize(label(last - vertices.begin()));
+
+            cellPointAddr[celli] = vertices;
+        }
+    }
 }
 
 

src/OpenFOAM/meshes/primitiveMesh/primitiveMeshPointCells.C

@@ -6,6 +6,7 @@
      \\/     M anipulation  |
 -------------------------------------------------------------------------------
     Copyright (C) 2011-2016 OpenFOAM Foundation
+    Copyright (C) 2023 OpenCFD Ltd.
 -------------------------------------------------------------------------------
 License
     This file is part of OpenFOAM.
@@ -28,14 +29,13 @@ License
 #include "primitiveMesh.H"
 #include "cell.H"
 #include "bitSet.H"
+#include "DynamicList.H"
 #include "ListOps.H"
 
 // * * * * * * * * * * * * * Private Member Functions  * * * * * * * * * * * //
 
 void Foam::primitiveMesh::calcPointCells() const
 {
-    // Loop through cells and mark up points
-
     if (debug)
     {
         Pout<< "primitiveMesh::calcPointCells() : "
@@ -59,48 +59,121 @@ void Foam::primitiveMesh::calcPointCells() const
             << "pointCells already calculated"
             << abort(FatalError);
     }
+    else if (hasCellPoints())
+    {
+        // Invert cellPoints
+        pcPtr_ = new labelListList(nPoints());
+        invertManyToMany(nPoints(), cellPoints(), *pcPtr_);
+    }
+    else if (hasPointFaces())
+    {
+        // Calculate point-cell from point-face information
+
+        const labelList& own = faceOwner();
+        const labelList& nei = faceNeighbour();
+        const labelListList& pFaces = pointFaces();
+
+        const label loopLen = nPoints();
+
+        pcPtr_ = new labelListList(nPoints());
+        auto& pointCellAddr = *pcPtr_;
+
+        DynamicList<label> storage(256);
+
+        for (label pointi = 0; pointi < loopLen; ++pointi)
+        {
+            // Clear any previous contents
+            storage.clear();
+
+            for (const label facei : pFaces[pointi])
+            {
+                // Owner cell
+                storage.push_back(own[facei]);
+
+                // Neighbour cell
+                if (facei < nInternalFaces())
+                {
+                    storage.push_back(nei[facei]);
+                }
+            }
+
+            // Sort + unique to eliminate duplicates
+            std::sort(storage.begin(), storage.end());
+            auto last = std::unique(storage.begin(), storage.end());
+            storage.resize(label(last - storage.begin()));
+
+            pointCellAddr[pointi] = storage;
+        }
+    }
     else
     {
-        const cellList& cf = cells();
+        // Calculate point-cell topology
 
-        // Count number of cells per point
+        const cellList& cellLst = cells();
+        const faceList& faceLst = faces();
 
-        labelList npc(nPoints(), Zero);
+        // Vertex labels for the current cell
+        // Note: since the number of points per cell is relatively small,
+        // using a bitSet to track does not help much.
+        // Using a labelHashSet to track has too many allocations.
+        DynamicList<label> vertices(256);
 
-        forAll(cf, celli)
+
+        const label loopLen = nCells();
+
+        // Step 1: count number of cells per point
+
+        labelList pointCount(nPoints(), Zero);
+
+        for (label celli = 0; celli < loopLen; ++celli)
         {
-            const labelList curPoints = cf[celli].labels(faces());
+            // Clear any previous contents
+            vertices.clear();
 
-            forAll(curPoints, pointi)
+            for (const label facei : cellLst[celli])
             {
-                label ptI = curPoints[pointi];
-
-                npc[ptI]++;
+                for (const label pointi : faceLst[facei])
+                {
+                    // Only once for each point id
+                    if (!vertices.contains(pointi))
+                    {
+                        vertices.push_back(pointi);
+                        ++pointCount[pointi];
+                    }
+                }
             }
         }
 
 
-        // Size and fill cells per point
+        // Step 2: set sizing, reset counters
 
-        pcPtr_ = new labelListList(npc.size());
-        labelListList& pointCellAddr = *pcPtr_;
+        pcPtr_ = new labelListList(nPoints());
+        auto& pointCellAddr = *pcPtr_;
 
         forAll(pointCellAddr, pointi)
         {
-            pointCellAddr[pointi].setSize(npc[pointi]);
+            pointCellAddr[pointi].resize_nocopy(pointCount[pointi]);
+            pointCount[pointi] = 0;
         }
-        npc = 0;
 
 
-        forAll(cf, celli)
+        // Step 3: fill in values. Logic as per step 1
+        for (label celli = 0; celli < loopLen; ++celli)
         {
-            const labelList curPoints = cf[celli].labels(faces());
+            // Clear any previous contents
+            vertices.clear();
 
-            forAll(curPoints, pointi)
+            for (const label facei : cellLst[celli])
             {
-                label ptI = curPoints[pointi];
-
-                pointCellAddr[ptI][npc[ptI]++] = celli;
+                for (const label pointi : faceLst[facei])
+                {
+                    // Only once for each point id
+                    if (!vertices.contains(pointi))
+                    {
+                        vertices.push_back(pointi);
+                        pointCellAddr[pointi][pointCount[pointi]++] = celli;
+                    }
+                }
             }
         }
     }

src/OpenFOAM/parallel/globalIndex/globalIndex.C

@@ -214,7 +214,7 @@ void Foam::globalIndex::reset
             // Non-parallel branch: use localSize on-proc, zero elsewhere
 
             localLens.resize(len, Zero);
-            localLens[UPstream::myProcNo(comm)] = localSize;
+            localLens[Pstream::myProcNo(comm)] = localSize;
         }
 
         reset(localLens, true);  // checkOverflow = true

src/OpenFOAM/parallel/globalIndex/globalIndex.H

@@ -519,9 +519,7 @@ public:
         {
             List<Type> allData;
             gather(offsets, comm, procIDs, fld, allData, tag, ct);
-
-            const int masterProci = procIDs.size() ? procIDs[0] : 0;
-            if (UPstream::myProcNo(comm) == masterProci)
+            if (Pstream::myProcNo(comm) == procIDs[0])
             {
                 fld.transfer(allData);
             }

src/OpenFOAM/parallel/globalIndex/globalIndexI.H

@@ -137,14 +137,14 @@ inline Foam::globalIndex::globalIndex
 
 inline bool Foam::globalIndex::empty() const
 {
-    return offsets_.empty() || offsets_.back() == 0;
+    return offsets_.empty() || offsets_.last() == 0;
 }
 
 
 inline Foam::label Foam::globalIndex::totalSize() const
 {
     const label len = (offsets_.size() - 1);
-    return (len < 1) ? 0 : offsets_[len];
+    return (len < 1) ? static_cast<label>(0) : offsets_[len];
 }
 
 
@@ -163,7 +163,7 @@ inline Foam::labelList Foam::globalIndex::sizes() const
 inline Foam::label Foam::globalIndex::nProcs() const noexcept
 {
     const label len = (offsets_.size() - 1);
-    return (len < 1) ? 0 : len;
+    return (len < 1) ? static_cast<label>(0) : len;
 }
 
 
@@ -219,7 +219,7 @@ inline Foam::label Foam::globalIndex::localStart(const label proci) const
 
 inline Foam::label Foam::globalIndex::localStart() const
 {
-    return localStart(UPstream::myProcNo());
+    return localStart(Pstream::myProcNo());
 }
 
 
@@ -231,7 +231,7 @@ inline Foam::label Foam::globalIndex::localSize(const label proci) const
 
 inline Foam::label Foam::globalIndex::localSize() const
 {
-    return localSize(UPstream::myProcNo());
+    return localSize(Pstream::myProcNo());
 }
 
 
@@ -244,7 +244,7 @@ inline Foam::label Foam::globalIndex::maxSize() const
 
 inline Foam::label Foam::globalIndex::maxNonLocalSize() const
 {
-    return maxNonLocalSize(UPstream::myProcNo());
+    return maxNonLocalSize(Pstream::myProcNo());
 }
 
 
@@ -256,7 +256,7 @@ inline Foam::labelRange Foam::globalIndex::range(const label proci) const
 
 inline Foam::labelRange Foam::globalIndex::range() const
 {
-    return range(UPstream::myProcNo());
+    return range(Pstream::myProcNo());
 }
 
 
@@ -268,7 +268,7 @@ inline bool Foam::globalIndex::isLocal(const label proci, const label i) const
 
 inline bool Foam::globalIndex::isLocal(const label i) const
 {
-    return isLocal(UPstream::myProcNo(), i);
+    return isLocal(Pstream::myProcNo(), i);
 }
 
 
@@ -284,7 +284,7 @@ inline Foam::label Foam::globalIndex::toGlobal
 
 inline Foam::label Foam::globalIndex::toGlobal(const label i) const
 {
-    return toGlobal(UPstream::myProcNo(), i);
+    return toGlobal(Pstream::myProcNo(), i);
 }
 
 
@@ -306,7 +306,7 @@ inline Foam::labelList Foam::globalIndex::toGlobal
     const labelUList& labels
 ) const
 {
-    return toGlobal(UPstream::myProcNo(), labels);
+    return toGlobal(Pstream::myProcNo(), labels);
 }
 
 
@@ -327,7 +327,7 @@ inline void Foam::globalIndex::inplaceToGlobal
 
 inline void Foam::globalIndex::inplaceToGlobal(labelUList& labels) const
 {
-    inplaceToGlobal(UPstream::myProcNo(), labels);
+    inplaceToGlobal(Pstream::myProcNo(), labels);
 }
 
 
@@ -349,7 +349,7 @@ Foam::globalIndex::toLocal(const label proci, const label i) const
 
 inline Foam::label Foam::globalIndex::toLocal(const label i) const
 {
-    return toLocal(UPstream::myProcNo(), i);
+    return toLocal(Pstream::myProcNo(), i);
 }
 
 
@@ -363,7 +363,7 @@ inline Foam::label Foam::globalIndex::whichProcID(const label i) const
             << abort(FatalError);
     }
 
-    const int proci = UPstream::myProcNo();
+    const label proci(Pstream::myProcNo());
 
     return isLocal(proci, i) ? proci : findLower(offsets_, i+1);
 }

src/OpenFOAM/parallel/globalIndex/globalIndexTemplates.C

@@ -91,9 +91,7 @@ void Foam::globalIndex::gatherValues
 
     const label startOfRequests = UPstream::nRequests();
 
-    const int masterProci = procIDs.size() ? procIDs[0] : 0;
-
-    if (UPstream::myProcNo(comm) == masterProci)
+    if (UPstream::myProcNo(comm) == procIDs[0])
     {
         allValues.resize_nocopy(procIDs.size());
         allValues[0] = localValue;
@@ -128,7 +126,7 @@ void Foam::globalIndex::gatherValues
             UOPstream::write
             (
                 commsType,
-                masterProci,
+                procIDs[0],
                 reinterpret_cast<const char*>(&localValue),
                 sizeof(Type),
                 tag,
@@ -137,7 +135,7 @@ void Foam::globalIndex::gatherValues
         }
         else
         {
-            OPstream toMaster(commsType, masterProci, 0, tag, comm);
+            OPstream toMaster(commsType, procIDs[0], 0, tag, comm);
             toMaster << localValue;
         }
     }
@@ -178,11 +176,9 @@ void Foam::globalIndex::gather
 
     const label startOfRequests = UPstream::nRequests();
 
-    const int masterProci = procIDs.size() ? procIDs[0] : 0;
-
-    if (UPstream::myProcNo(comm) == masterProci)
+    if (Pstream::myProcNo(comm) == procIDs[0])
     {
-        allFld.resize_nocopy(off.back());  // == totalSize()
+        allFld.resize_nocopy(off.last());  // == totalSize()
 
         // Assign my local data - respect offset information
         // so that we can request 0 entries to be copied.
@@ -230,7 +226,7 @@ void Foam::globalIndex::gather
             UOPstream::write
             (
                 commsType,
-                masterProci,
+                procIDs[0],
                 fld.cdata_bytes(),
                 fld.size_bytes(),
                 tag,
@@ -239,7 +235,7 @@ void Foam::globalIndex::gather
         }
         else
         {
-            OPstream toMaster(commsType, masterProci, 0, tag, comm);
+            OPstream toMaster(commsType, procIDs[0], 0, tag, comm);
             toMaster << fld;
         }
     }
@@ -298,11 +294,9 @@ void Foam::globalIndex::gather
 
     const label startOfRequests = UPstream::nRequests();
 
-    const int masterProci = procIDs.size() ? procIDs[0] : 0;
-
-    if (UPstream::myProcNo(comm) == masterProci)
+    if (Pstream::myProcNo(comm) == procIDs[0])
     {
-        allFld.resize_nocopy(off.back());  // == totalSize()
+        allFld.resize_nocopy(off.last());  // == totalSize()
 
         // Assign my local data - respect offset information
         // so that we can request 0 entries to be copied
@@ -337,7 +331,7 @@ void Foam::globalIndex::gather
         }
         else
         {
-            OPstream toMaster(commsType, masterProci, 0, tag, comm);
+            OPstream toMaster(commsType, procIDs[0], 0, tag, comm);
             toMaster << fld;
         }
     }
@@ -883,9 +877,7 @@ void Foam::globalIndex::scatter
 
     const label startOfRequests = UPstream::nRequests();
 
-    const int masterProci = procIDs.size() ? procIDs[0] : 0;
-
-    if (UPstream::myProcNo(comm) == masterProci)
+    if (Pstream::myProcNo(comm) == procIDs[0])
     {
         for (label i = 1; i < procIDs.size(); ++i)
         {
@@ -937,7 +929,7 @@ void Foam::globalIndex::scatter
             UIPstream::read
             (
                 commsType,
-                masterProci,
+                procIDs[0],
                 fld.data_bytes(),
                 fld.size_bytes(),
                 tag,
@@ -946,7 +938,7 @@ void Foam::globalIndex::scatter
         }
         else
         {
-            IPstream fromMaster(commsType, masterProci, 0, tag, comm);
+            IPstream fromMaster(commsType, procIDs[0], 0, tag, comm);
             fromMaster >> fld;
         }
     }

src/fileFormats/vtk/file/foamVtkFileWriter.C

@@ -5,7 +5,7 @@
     \\  /    A nd           | www.openfoam.com
      \\/     M anipulation  |
 -------------------------------------------------------------------------------
-    Copyright (C) 2018-2023 OpenCFD Ltd.
+    Copyright (C) 2018-2022 OpenCFD Ltd.
 -------------------------------------------------------------------------------
 License
     This file is part of OpenFOAM.
@@ -76,11 +76,11 @@ Foam::Ostream& Foam::vtk::fileWriter::reportBadState
 Foam::Ostream& Foam::vtk::fileWriter::reportBadState
 (
     Ostream& os,
-    outputState expected1,
+    outputState expected,
     outputState expected2
 ) const
 {
-    reportBadState(os, expected1)
+    reportBadState(os, expected)
         << " or (" << stateNames[expected2] << ')';
     return os;
 }
@@ -554,18 +554,10 @@ bool Foam::vtk::fileWriter::writeProcIDs(const label nValues)
     {
         ++nCellData_;
     }
-    else if (isState(outputState::POINT_DATA))
-    {
-        ++nPointData_;
-    }
     else
     {
-        reportBadState
-        (
-            FatalErrorInFunction,
-            outputState::CELL_DATA,
-            outputState::POINT_DATA
-        )   << " for procID field" << nl << endl
+        reportBadState(FatalErrorInFunction, outputState::CELL_DATA)
+            << " for procID field" << nl << endl
             << exit(FatalError);
 
         return false;

src/fileFormats/vtk/file/foamVtkFileWriter.H

@@ -5,7 +5,7 @@
     \\  /    A nd           | www.openfoam.com
      \\/     M anipulation  |
 -------------------------------------------------------------------------------
-    Copyright (C) 2018-2023 OpenCFD Ltd.
+    Copyright (C) 2018-2022 OpenCFD Ltd.
 -------------------------------------------------------------------------------
 License
     This file is part of OpenFOAM.
@@ -201,8 +201,7 @@ protected:
             const UList<Type>& field
         );
 
-        //- Write nValues of processor ids as CellData or PointData
-        //- (no-op in serial)
+        //- Write nValues of processor ids as CellData (no-op in serial)
         bool writeProcIDs(const label nValues);
 
 
@@ -306,16 +305,10 @@ public:
         //  \return True if the state changed
         virtual bool beginPointData(label nFields = 0) = 0;
 
-        //- True if output state corresponds to CELL_DATA
-        inline bool isCellData() const noexcept;
-
-        //- True if output state corresponds to POINT_DATA
-        inline bool isPointData() const noexcept;
-
-        //- The number of CellData written for the Piece thus far.
+        //- Return the number of CellData written for the Piece thus far.
         inline label nCellData() const noexcept;
 
-        //- The number of PointData written for the Piece thus far.
+        //- Return the number of PointData written for the Piece thus far.
         inline label nPointData() const noexcept;
 
 

src/fileFormats/vtk/file/foamVtkFileWriterI.H

@@ -5,7 +5,7 @@
     \\  /    A nd           | www.openfoam.com
      \\/     M anipulation  |
 -------------------------------------------------------------------------------
-    Copyright (C) 2018-2023 OpenCFD Ltd.
+    Copyright (C) 2018-2022 OpenCFD Ltd.
 -------------------------------------------------------------------------------
 License
     This file is part of OpenFOAM.
@@ -107,16 +107,4 @@ inline Foam::label Foam::vtk::fileWriter::nPointData() const noexcept
 }
 
 
-inline bool Foam::vtk::fileWriter::isCellData() const noexcept
-{
-    return (outputState::CELL_DATA == state_);
-}
-
-
-inline bool Foam::vtk::fileWriter::isPointData() const noexcept
-{
-    return (outputState::POINT_DATA == state_);
-}
-
-
 // ************************************************************************* //

src/fileFormats/vtk/write/foamVtkLineWriter.C

@@ -5,7 +5,7 @@
     \\  /    A nd           | www.openfoam.com
      \\/     M anipulation  |
 -------------------------------------------------------------------------------
-    Copyright (C) 2021-2023 OpenCFD Ltd.
+    Copyright (C) 2021 OpenCFD Ltd.
 -------------------------------------------------------------------------------
 License
     This file is part of OpenFOAM.
@@ -131,10 +131,6 @@ void Foam::vtk::lineWriter::piece
 
 bool Foam::vtk::lineWriter::writeProcIDs()
 {
-    if (this->isPointData())
-    {
-        return vtk::fileWriter::writeProcIDs(nLocalPoints_);
-    }
     return vtk::fileWriter::writeProcIDs(nLocalLines_);
 }
 

src/fileFormats/vtk/write/foamVtkLineWriter.H

@@ -5,7 +5,7 @@
     \\  /    A nd           | www.openfoam.com
      \\/     M anipulation  |
 -------------------------------------------------------------------------------
-    Copyright (C) 2021-2023 OpenCFD Ltd.
+    Copyright (C) 2021 OpenCFD Ltd.
 -------------------------------------------------------------------------------
 License
     This file is part of OpenFOAM.
@@ -155,8 +155,8 @@ public:
         void piece(const pointField& points, const edgeList& edges);
 
 
-        //- Write processor ids for each line as CellData or for each point
-        //- as PointData, depending on isPointData() state. No-op in serial.
+        //- Write processor ids for each line as CellData
+        //- (no-op in serial)
         bool writeProcIDs();
 
         //- Write a uniform field of Cell (Line) or Point values

src/fileFormats/vtk/write/foamVtkSurfaceWriter.C

@@ -5,7 +5,7 @@
     \\  /    A nd           | www.openfoam.com
      \\/     M anipulation  |
 -------------------------------------------------------------------------------
-    Copyright (C) 2018-2023 OpenCFD Ltd.
+    Copyright (C) 2018-2021 OpenCFD Ltd.
 -------------------------------------------------------------------------------
 License
     This file is part of OpenFOAM.
@@ -131,10 +131,6 @@ void Foam::vtk::surfaceWriter::piece
 
 bool Foam::vtk::surfaceWriter::writeProcIDs()
 {
-    if (this->isPointData())
-    {
-        return vtk::fileWriter::writeProcIDs(nLocalPoints_);
-    }
     return vtk::fileWriter::writeProcIDs(nLocalPolys_);
 }
 

src/fileFormats/vtk/write/foamVtkSurfaceWriter.H

@@ -5,7 +5,7 @@
     \\  /    A nd           | www.openfoam.com
      \\/     M anipulation  |
 -------------------------------------------------------------------------------
-    Copyright (C) 2018-2023 OpenCFD Ltd.
+    Copyright (C) 2018-2021 OpenCFD Ltd.
 -------------------------------------------------------------------------------
 License
     This file is part of OpenFOAM.
@@ -155,8 +155,8 @@ public:
         void piece(const pointField& points, const faceList& faces);
 
 
-        //- Write processor ids for each poly as CellData or for each point
-        //- as PointData, depending on isPointData() state. No-op in serial.
+        //- Write processor ids for each poly as CellData
+        //- (no-op in serial)
         bool writeProcIDs();
 
         //- Write a uniform field of Cell (Poly) or Point values

src/finiteVolume/fvMesh/zoneDistribute/zoneDistribute.C

@@ -109,22 +109,22 @@ void Foam::zoneDistribute::setUpCommforZone
             }
         }
 
-        // Stream the send data into PstreamBuffers
+        // Stream the send data into PstreamBuffers,
+        // which we also use to track the current topology.
+
         pBufs_.clear();
 
-        for (const int proci : pBufs_.allProcs())
+        for (const int proci : UPstream::allProcs())
         {
-            const auto& indices = needed[proci];
-
-            if (proci != UPstream::myProcNo() && !indices.empty())
+            if (proci != UPstream::myProcNo() && !needed[proci].empty())
             {
                 // Serialize as List
                 UOPstream toProc(proci, pBufs_);
-                toProc << indices.sortedToc();
+                toProc << needed[proci].sortedToc();
             }
         }
 
-        pBufs_.finishedSends();
+        pBufs_.finishedSends(sendConnections_, sendProcs_, recvProcs_);
 
         for (const int proci : pBufs_.allProcs())
         {

src/finiteVolume/fvMesh/zoneDistribute/zoneDistribute.H

@@ -93,6 +93,15 @@ class zoneDistribute
         //- Global cell/face index to send for processor-to-processor comms
         List<labelList> send_;
 
+        //- Parallel [cache]: send connectivity (true/false)
+        bitSet sendConnections_;
+
+        //- Parallel [cache]: send data to these ranks
+        DynamicList<label> sendProcs_;
+
+        //- Parallel [cache]: recv data from these ranks
+        DynamicList<label> recvProcs_;
+
         //- Persistent set of exchange buffers
         PstreamBuffers pBufs_;
 

src/finiteVolume/fvMesh/zoneDistribute/zoneDistributeI.H

@@ -6,7 +6,7 @@
      \\/     M anipulation  |
 -------------------------------------------------------------------------------
     Copyright (C) 2019-2020 DLR
-    Copyright (C) 2020-2023 OpenCFD Ltd.
+    Copyright (C) 2020-2022 OpenCFD Ltd.
 -------------------------------------------------------------------------------
 License
     This file is part of OpenFOAM.
@@ -161,18 +161,28 @@ Foam::Map<Type> Foam::zoneDistribute::getDatafromOtherProc
 
     if (UPstream::parRun())
     {
+        if (sendConnections_.empty())
+        {
+            WarningInFunction
+                << "The send/recv connections not initialized - "
+                << "likely that setUpCommforZone() was not called"
+                << endl;
+            // But don't exit/abort for now
+        }
+
+        // Stream the send data into PstreamBuffers,
+        // which we also use to track the current topology.
+
         pBufs_.clear();
 
-        for (const int proci : pBufs_.allProcs())
+        for (const int proci : UPstream::allProcs())
         {
-            const auto& indices = send_[proci];
-
-            if (proci != UPstream::myProcNo() && !indices.empty())
+            if (proci != UPstream::myProcNo() && !send_[proci].empty())
             {
                 // Serialize as Map
-                Map<Type> sendValues(2*indices.size());
+                Map<Type> sendValues(2*send_[proci].size());
 
-                for (const label sendIdx : indices)
+                for (const label sendIdx : send_[proci])
                 {
                     sendValues.insert
                     (
@@ -186,7 +196,7 @@ Foam::Map<Type> Foam::zoneDistribute::getDatafromOtherProc
             }
         }
 
-        pBufs_.finishedSends();
+        pBufs_.finishedSends(sendConnections_, sendProcs_, recvProcs_);
 
         for (const int proci : pBufs_.allProcs())
         {

src/meshTools/coordSet/writers/vtk/foamVtkCoordSetWriter.H

@@ -201,9 +201,8 @@ public:
         void piece(const UPtrList<const pointField>& points);
 
 
-        //- Write processor ids for each line as CellData or for each point
-        //- as PointData, depending on isPointData() state. No-op in serial.
-        //  Not implemented.
+        //- Write processor ids for each line as CellData
+        //- (no-op in serial)
         bool writeProcIDs();
 
 

src/meshTools/output/vtk/patch/foamVtkGenericPatchWriter.H

@@ -5,7 +5,7 @@
     \\  /    A nd           | www.openfoam.com
      \\/     M anipulation  |
 -------------------------------------------------------------------------------
-    Copyright (C) 2018-2023 OpenCFD Ltd.
+    Copyright (C) 2018-2021 OpenCFD Ltd.
 -------------------------------------------------------------------------------
 License
     This file is part of OpenFOAM.
@@ -159,10 +159,6 @@ public:
         //- Write processor ids for each poly as CellData
         bool writeProcIDs()
         {
-            if (this->isPointData())
-            {
-                return vtk::fileWriter::writeProcIDs(nLocalPoints_);
-            }
             return vtk::polyWriter::writeProcIDs(nLocalPolys_);
         }
 };

src/meshTools/output/vtk/patch/foamVtkPatchMeshWriter.C

@@ -5,7 +5,7 @@
     \\  /    A nd           | www.openfoam.com
      \\/     M anipulation  |
 -------------------------------------------------------------------------------
-    Copyright (C) 2016-2023 OpenCFD Ltd.
+    Copyright (C) 2016-2022 OpenCFD Ltd.
 -------------------------------------------------------------------------------
 License
     This file is part of OpenFOAM.
@@ -585,10 +585,6 @@ void Foam::vtk::patchMeshWriter::writePatchIDs()
 
 bool Foam::vtk::patchMeshWriter::writeProcIDs()
 {
-    if (this->isPointData())
-    {
-        return vtk::fileWriter::writeProcIDs(nLocalPoints_);
-    }
     return vtk::fileWriter::writeProcIDs(nLocalPolys_);
 }
 

src/meshTools/output/vtk/patch/foamVtkPatchMeshWriter.H

@@ -5,7 +5,7 @@
     \\  /    A nd           | www.openfoam.com
      \\/     M anipulation  |
 -------------------------------------------------------------------------------
-    Copyright (C) 2016-2023 OpenCFD Ltd.
+    Copyright (C) 2016-2022 OpenCFD Ltd.
 -------------------------------------------------------------------------------
 License
     This file is part of OpenFOAM.
@@ -209,8 +209,8 @@ public:
         //  Must be called within the CELL_DATA state.
         void writePatchIDs();
 
-        //- Write processor ids for each line as CellData or for each point
-        //- as PointData, depending on isPointData() state. No-op in serial.
+        //- Write processor ids as CellData. This is no-op in serial.
+        //  Must be called within the CELL_DATA state.
         bool writeProcIDs();
 
         //- Write processor neighbour ids as CellData. This is no-op in serial.

tutorials/lagrangian/kinematicParcelFoam/drippingChair/Allrun

@@ -3,30 +3,19 @@ cd "${0%/*}" || exit                                # Run from this directory
 . ${WM_PROJECT_DIR:?}/bin/tools/RunFunctions        # Tutorial run functions
 #------------------------------------------------------------------------------
 
-unset decompDict
-# decompDict=system/decomposeParDict-6
-# decompDict=system/decomposeParDict-7
-
 runApplication blockMesh
 
 runApplication makeFaMesh
 
-runApplication -decompose-dict="$decompDict" decomposePar
+runApplication decomposePar
 
-# For ids and processor partitioning
-runParallel -s volume -decompose-dict="$decompDict" \
-    foamToVTK -name VTK-parallel -time 0 \
-    -no-finite-area -no-internal -no-lagrangian -no-fields -with-ids \
-    -patches filmWalls
+# For the processor partitioning
+runParallel -s finiteVolume foamToVTK -name VTK-parallel -time 0 \
+    -no-finite-area -no-internal -no-lagrangian -no-fields -with-ids
 
-runParallel -decompose-dict="$decompDict" $(getApplication)
+runParallel $(getApplication)
 
-if false
-then
-    # Not usually needed - uses areaWrite
-    runParallel -s area -decompose-dict="$decompDict" \
-        foamToVTK -name VTK-parallel \
-        -no-boundary -no-internal -no-lagrangian -with-ids
-fi
+runParallel -s finiteArea foamToVTK -name VTK-parallel \
+    -no-boundary -no-internal -no-lagrangian
 
 #------------------------------------------------------------------------------

tutorials/lagrangian/kinematicParcelFoam/drippingChair/Allrun-serial

@@ -7,18 +7,13 @@ runApplication blockMesh
 
 runApplication makeFaMesh
 
-# For ids and processor partitioning
-runApplication -s volume.serial foamToVTK -name VTK-serial -time 0 \
-    -no-finite-area -no-internal -no-lagrangian -no-fields -with-ids \
-    -patches filmWalls
+# For the cell ids etc
+runApplication -s finiteVolume.serial foamToVTK -name VTK-serial -time 0 \
+    -no-finite-area -no-internal -no-lagrangian -no-fields -with-ids
 
 runApplication $(getApplication)
 
-if false
-then
-    # Not usually needed - uses areaWrite
-    runApplication -s area.serial foamToVTK -name VTK-serial \
-        -no-boundary -no-internal -no-lagrangian
-fi
+runApplication -s finiteArea.serial foamToVTK -name VTK-serial \
+    -no-boundary -no-internal -no-lagrangian
 
 #------------------------------------------------------------------------------

tutorials/lagrangian/kinematicParcelFoam/drippingChair/system/areaWrite

@@ -1,25 +0,0 @@
-// -*- C++ -*-
-// Use the areaWrite function object
-
-areaWrite
-{
-    type    areaWrite;
-    libs    (utilityFunctionObjects);
-    log     true;
-
-    writeControl    writeTime;
-    writeInterval   1;
-
-    // Fields to output (words or regex)
-    fields  (Uf_film hf_film pf_film);
-
-    surfaceFormat ensight;
-
-    formatOptions
-    {
-        default { format binary; }
-    }
-}
-
-
-// ************************************************************************* //

tutorials/lagrangian/kinematicParcelFoam/drippingChair/system/controlDict

@@ -52,10 +52,5 @@ regionFaMaxCo   1;
 
 maxDeltaT       0.1;
 
-functions
-{
-    #include "areaWrite"
-}
-
 
 // ************************************************************************* //

tutorials/lagrangian/kinematicParcelFoam/drippingChair/system/decomposeParDict-6

@@ -1,21 +0,0 @@
-/*--------------------------------*- C++ -*----------------------------------*\
-| =========                 |                                                 |
-| \\      /  F ield         | OpenFOAM: The Open Source CFD Toolbox           |
-|  \\    /   O peration     | Version:  v2306                                 |
-|   \\  /    A nd           | Website:  www.openfoam.com                      |
-|    \\/     M anipulation  |                                                 |
-\*---------------------------------------------------------------------------*/
-FoamFile
-{
-    version     2.0;
-    format      ascii;
-    class       dictionary;
-    object      decomposeParDict;
-}
-// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
-
-numberOfSubdomains 6;
-
-method          scotch;
-
-// ************************************************************************* //

tutorials/lagrangian/kinematicParcelFoam/drippingChair/system/decomposeParDict-7

@@ -1,21 +0,0 @@
-/*--------------------------------*- C++ -*----------------------------------*\
-| =========                 |                                                 |
-| \\      /  F ield         | OpenFOAM: The Open Source CFD Toolbox           |
-|  \\    /   O peration     | Version:  v2306                                 |
-|   \\  /    A nd           | Website:  www.openfoam.com                      |
-|    \\/     M anipulation  |                                                 |
-\*---------------------------------------------------------------------------*/
-FoamFile
-{
-    version     2.0;
-    format      ascii;
-    class       dictionary;
-    object      decomposeParDict;
-}
-// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
-
-numberOfSubdomains 7;
-
-method          scotch;
-
-// ************************************************************************* //