From 048166c3d8126903a404a5cdbb36fc259d07f56f Mon Sep 17 00:00:00 2001
From: Mark Olesen <Mark.Olesen@esi-group.com>
Date: Fri, 29 Oct 2021 14:17:56 +0200
Subject: [PATCH] ENH: reduces buffer use / blocking in Ensight output

- reuse single component buffer within an Ensight output method.
  Use direct UPstream read/write to avoid Pstream char buffers

- replace blocking transfer with scheduled for Ensight cloud output
---
 .../ensight/output/ensightOutput.H            |  17 +-
 .../ensight/output/ensightOutputTemplates.C   | 185 +++++++++---------
 .../conversion/ensight/ensightOutputCloud.C   | 142 ++++++++------
 .../conversion/ensight/ensightOutputCloud.H   |  36 +++-
 .../ensight/ensightOutputCloudTemplates.C     | 138 ++++++++-----
 5 files changed, 302 insertions(+), 216 deletions(-)

diff --git a/src/fileFormats/ensight/output/ensightOutput.H b/src/fileFormats/ensight/output/ensightOutput.H
index fefe06a535..eeb34f3aca 100644
--- a/src/fileFormats/ensight/output/ensightOutput.H
+++ b/src/fileFormats/ensight/output/ensightOutput.H
@@ -50,6 +50,7 @@ SourceFiles
 #include "ListOps.H"
 #include "ListListOps.H"
 #include "IndirectList.H"
+#include "DynamicList.H"
 
 // * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
 
@@ -260,17 +261,27 @@ labelList getPolysNFaces(const polyMesh& mesh, const labelUList& addr);
 labelList getPolysNPointsPerFace(const polyMesh& mesh, const labelUList& addr);
 
 
-//- Copy specified field component into a scalarField
-//  works for various lists types
+//- Copy specified field component into a scalar buffer
+//- works for various lists types. Must be adequately sized before calling
 template<template<typename> class FieldContainer, class Type>
 void copyComponent
 (
-    scalarField& res,
+    List<scalar>& cmptBuffer,
     const FieldContainer<Type>& input,
     const direction cmpt
 );
 
 
+//- Write field content (component-wise)
+template<template<typename> class FieldContainer, class Type>
+void writeFieldContent
+(
+    ensightFile& os,
+    const FieldContainer<Type>& fld,
+    bool parallel  //!< Collective write?
+);
+
+
 //- Write coordinates (component-wise) for the given part
 template<template<typename> class FieldContainer>
 bool writeCoordinates
diff --git a/src/fileFormats/ensight/output/ensightOutputTemplates.C b/src/fileFormats/ensight/output/ensightOutputTemplates.C
index 3c408d6fc2..0dd52790c8 100644
--- a/src/fileFormats/ensight/output/ensightOutputTemplates.C
+++ b/src/fileFormats/ensight/output/ensightOutputTemplates.C
@@ -27,20 +27,27 @@ License
 
 #include "ensightOutput.H"
 #include "ensightPTraits.H"
+#include "globalIndex.H"
 
 // * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
 
 template<template<typename> class FieldContainer, class Type>
 void Foam::ensightOutput::Detail::copyComponent
 (
-    scalarField& res,
+    List<scalar>& cmptBuffer,
     const FieldContainer<Type>& input,
     const direction cmpt
 )
 {
-    res.resize(input.size());
+    if (cmptBuffer.size() < input.size())
+    {
+        FatalErrorInFunction
+            << "Component buffer too small: "
+            << cmptBuffer.size() << " < " << input.size() << nl
+            << exit(FatalError);
+    }
 
-    auto iter = res.begin();
+    auto iter = cmptBuffer.begin();
 
     for (const Type& val : input)
     {
@@ -50,6 +57,80 @@ void Foam::ensightOutput::Detail::copyComponent
 }
 
 
+template<template<typename> class FieldContainer, class Type>
+void Foam::ensightOutput::Detail::writeFieldContent
+(
+    ensightFile& os,
+    const FieldContainer<Type>& fld,
+    bool parallel
+)
+{
+    // already checked prior to calling, but extra safety
+    parallel = parallel && Pstream::parRun();
+
+    // Size information (offsets are irrelevant)
+    globalIndex procAddr;
+    if (parallel)
+    {
+        procAddr.reset(UPstream::listGatherValues<label>(fld.size()));
+    }
+    else
+    {
+        // Master size
+        procAddr.reset(labelList(Foam::one{}, fld.size()));
+    }
+
+
+    if (Pstream::master())
+    {
+        DynamicList<scalar> cmptBuffer(procAddr.maxSize());
+
+        for (direction d=0; d < pTraits<Type>::nComponents; ++d)
+        {
+            const direction cmpt = ensightPTraits<Type>::componentOrder[d];
+
+            // Write master data
+            cmptBuffer.resize_nocopy(procAddr.localSize(0));
+            copyComponent(cmptBuffer, fld, cmpt);
+            os.writeList(cmptBuffer);
+
+            // Receive and write
+            for (const label proci : procAddr.subProcs())
+            {
+                cmptBuffer.resize_nocopy(procAddr.localSize(proci));
+                UIPstream::read
+                (
+                    UPstream::commsTypes::scheduled,
+                    proci,
+                    cmptBuffer.data_bytes(),
+                    cmptBuffer.size_bytes()
+                );
+                os.writeList(cmptBuffer);
+            }
+        }
+    }
+    else if (parallel)
+    {
+        // Send
+        List<scalar> cmptBuffer(fld.size());
+
+        for (direction d=0; d < pTraits<Type>::nComponents; ++d)
+        {
+            const direction cmpt = ensightPTraits<Type>::componentOrder[d];
+
+            copyComponent(cmptBuffer, fld, cmpt);
+            UOPstream::write
+            (
+                UPstream::commsTypes::scheduled,
+                Pstream::masterNo(),
+                cmptBuffer.cdata_bytes(),
+                cmptBuffer.size_bytes()
+            );
+        }
+    }
+}
+
+
 template<template<typename> class FieldContainer>
 bool Foam::ensightOutput::Detail::writeCoordinates
 (
@@ -63,57 +144,13 @@ bool Foam::ensightOutput::Detail::writeCoordinates
 {
     parallel = parallel && Pstream::parRun();
 
-    const IntRange<int> senders =
-    (
-        parallel
-      ? Pstream::subProcs()
-      : IntRange<int>()
-    );
-
-
-    // Using manual copyComponent(...) instead of fld.component() to support
-    // indirect lists etc.
-
-    scalarField send(fld.size());
-
     if (Pstream::master())
     {
         os.beginPart(partId, partName);
         os.beginCoordinates(nPoints);
-
-        for (direction cmpt=0; cmpt < point::nComponents; ++cmpt)
-        {
-            // Main
-            copyComponent(send, fld, cmpt);
-            os.writeList(send);
-
-            // Others
-            for (const int proci : senders)
-            {
-                IPstream fromOther(Pstream::commsTypes::scheduled, proci);
-                scalarField recv(fromOther);
-
-                os.writeList(recv);
-            }
-        }
     }
-    else if (senders)
-    {
-        // Send from other (parallel)
 
-        for (direction cmpt=0; cmpt < point::nComponents; ++cmpt)
-        {
-            copyComponent(send, fld, cmpt);
-
-            OPstream toMaster
-            (
-                Pstream::commsTypes::scheduled,
-                Pstream::masterNo()
-            );
-
-            toMaster << send;
-        }
-    }
+    ensightOutput::Detail::writeFieldContent(os, fld, parallel);
 
     return true;
 }
@@ -130,14 +167,6 @@ bool Foam::ensightOutput::Detail::writeFieldComponents
 {
     parallel = parallel && Pstream::parRun();
 
-    const IntRange<int> senders =
-    (
-        parallel
-      ? Pstream::subProcs()
-      : IntRange<int>()
-    );
-
-
     // Preliminary checks
     {
         bool hasField = !fld.empty();
@@ -152,52 +181,12 @@ bool Foam::ensightOutput::Detail::writeFieldComponents
     }
 
 
-    // Using manual copyComponent(...) instead of fld.component() to support
-    // indirect lists etc.
-
-    scalarField send(fld.size());
-
     if (Pstream::master())
     {
         os.writeKeyword(key);
-
-        for (direction d=0; d < pTraits<Type>::nComponents; ++d)
-        {
-            const direction cmpt = ensightPTraits<Type>::componentOrder[d];
-
-            // Main
-            copyComponent(send, fld, cmpt);
-            os.writeList(send);
-
-            // Others
-            for (const int proci : senders)
-            {
-                IPstream fromOther(Pstream::commsTypes::scheduled, proci);
-                scalarField recv(fromOther);
-
-                os.writeList(recv);
-            }
-        }
     }
-    else if (senders)
-    {
-        // Send from other (parallel)
 
-        for (direction d=0; d < pTraits<Type>::nComponents; ++d)
-        {
-            const direction cmpt = ensightPTraits<Type>::componentOrder[d];
-
-            copyComponent(send, fld, cmpt);
-
-            OPstream toMaster
-            (
-                Pstream::commsTypes::scheduled,
-                Pstream::masterNo()
-            );
-
-            toMaster << send;
-        }
-    }
+    ensightOutput::Detail::writeFieldContent(os, fld, parallel);
 
     return true;
 }
@@ -294,6 +283,7 @@ bool Foam::ensightOutput::Detail::writeFaceLocalField
             << exit(FatalError);
     }
 
+
     if (Pstream::master())
     {
         os.beginPart(part.index());
@@ -345,6 +335,7 @@ bool Foam::ensightOutput::writeField
         if (!hasField) return false;
     }
 
+
     if (Pstream::master())
     {
         os.beginPart(part.index());
diff --git a/src/lagrangian/intermediate/conversion/ensight/ensightOutputCloud.C b/src/lagrangian/intermediate/conversion/ensight/ensightOutputCloud.C
index 28c512a8b4..dad05382e0 100644
--- a/src/lagrangian/intermediate/conversion/ensight/ensightOutputCloud.C
+++ b/src/lagrangian/intermediate/conversion/ensight/ensightOutputCloud.C
@@ -5,7 +5,7 @@
     \\  /    A nd           | www.openfoam.com
      \\/     M anipulation  |
 -------------------------------------------------------------------------------
-    Copyright (C) 2016-2020 OpenCFD Ltd.
+    Copyright (C) 2016-2021 OpenCFD Ltd.
 -------------------------------------------------------------------------------
 License
     This file is part of OpenFOAM.
@@ -29,17 +29,17 @@ License
 #include "fvMesh.H"
 #include "Cloud.H"
 #include "passiveParticle.H"
-#include "pointField.H"
+#include "globalIndex.H"
 
 // * * * * * * * * * * * * * * * Local Functions * * * * * * * * * * * * * * //
 
 namespace Foam
 {
     //- Binary output
-    static inline void writeMeasured
+    static inline void writeMeasured_binary
     (
         ensightFile& os,
-        const pointField& points
+        const UList<point>& points
     )
     {
         for (const point& p : points)
@@ -51,11 +51,11 @@ namespace Foam
     }
 
     //- ASCII output. Id + position together
-    static inline label writeMeasured
+    static inline label writeMeasured_ascii
     (
         ensightFile& os,
         label pointId,
-        const pointField& points
+        const UList<point>& points
     )
     {
         for (const point& p : points)
@@ -80,17 +80,50 @@ bool Foam::ensightOutput::writeCloudPositions
     const fvMesh& mesh,
     const word& cloudName,
     bool exists,
-    autoPtr<ensightFile>& output,
-    Pstream::commsTypes comm
+    autoPtr<ensightFile>& output
 )
 {
-    pointField positions;
+    label nLocalParcels(0);
+    autoPtr<Cloud<passiveParticle>> parcelsPtr;
 
     if (exists)
     {
-        Cloud<passiveParticle> parcels(mesh, cloudName, false);
+        parcelsPtr.reset(new Cloud<passiveParticle>(mesh, cloudName, false));
+        nLocalParcels = parcelsPtr().size();
+    }
 
-        positions.resize(parcels.size());
+    // Total number of parcels on all processes
+    const label nTotParcels = returnReduce(nLocalParcels, sumOp<label>());
+
+    if (Pstream::master())
+    {
+        ensightFile& os = output();
+        os.beginParticleCoordinates(nTotParcels);
+    }
+
+    if (!nTotParcels)
+    {
+        return false;  // DONE
+    }
+
+
+    // Size information (offsets are irrelevant)
+    const globalIndex procAddr
+    (
+        UPstream::listGatherValues<label>(nLocalParcels),
+        globalIndex::SIZES
+    );
+
+
+    DynamicList<point> positions;
+    positions.reserve(Pstream::master() ? procAddr.maxSize() : nLocalParcels);
+
+    // Extract positions
+    if (parcelsPtr)
+    {
+        const auto& parcels = *parcelsPtr;
+
+        positions.resize_nocopy(parcels.size());  // same as nLocalParcels
 
         auto outIter = positions.begin();
 
@@ -99,75 +132,74 @@ bool Foam::ensightOutput::writeCloudPositions
             *outIter = p.position();
             ++outIter;
         }
+
+        parcelsPtr.reset(nullptr);
     }
 
-
-    // Total number of parcels on all processes
-    const label nTotParcels = returnReduce(positions.size(), sumOp<label>());
-
-    // Update the exists/not exists information (for return value)
-    exists = nTotParcels;
-
     if (Pstream::master())
     {
         ensightFile& os = output();
+        const bool isBinaryOutput = (os.format() == IOstream::BINARY);
 
-        os.beginParticleCoordinates(nTotParcels);
-        if (!exists)
-        {
-            return exists;  // DONE
-        }
+        label parcelId = 0;
 
-        if (os.format() == IOstream::BINARY)
+        if (isBinaryOutput)
         {
-            // binary write is Ensight6 - first ids, then positions
+            // NB: binary write is Ensight6 - first ids, then positions
 
             // 1-index
-            for (label parcelId = 1; parcelId <= nTotParcels; ++parcelId)
+            for (label id = 1; id <= nTotParcels; ++id)
             {
-                os.write(parcelId);
+                os.write(id);
             }
 
-            // Master
-            writeMeasured(os, positions);
-
-            // Slaves
-            for (const int slave : Pstream::subProcs())
-            {
-                IPstream fromSlave(comm, slave);
-                pointField recv(fromSlave);
-
-                writeMeasured(os, recv);
-            }
+            // Write master data
+            writeMeasured_binary(os, positions);
         }
         else
         {
-            // ASCII id + position together
-            label parcelId = 0;
+            // NB: ascii write is (id + position) together
 
-            // Master
-            parcelId = writeMeasured(os, parcelId, positions);
+            // Write master data
+            parcelId = writeMeasured_ascii(os, parcelId, positions);
+        }
 
-            // Slaves
-            for (const int slave : Pstream::subProcs())
+
+        // Receive and write
+        for (const label proci : procAddr.subProcs())
+        {
+            positions.resize_nocopy(procAddr.localSize(proci));
+            UIPstream::read
+            (
+                UPstream::commsTypes::scheduled,
+                proci,
+                positions.data_bytes(),
+                positions.size_bytes()
+            );
+
+            if (isBinaryOutput)
             {
-                IPstream fromSlave(comm, slave);
-                pointField recv(fromSlave);
-
-                parcelId = writeMeasured(os, parcelId, recv);
+                writeMeasured_binary(os, positions);
+            }
+            else
+            {
+                parcelId = writeMeasured_ascii(os, parcelId, positions);
             }
         }
     }
-    else if (nTotParcels)
+    else
     {
-        // SLAVE, and data exist
-        OPstream toMaster(comm, Pstream::masterNo());
-
-        toMaster
-            << positions;
+        // Send
+        UOPstream::write
+        (
+            UPstream::commsTypes::scheduled,
+            Pstream::masterNo(),
+            positions.cdata_bytes(),
+            positions.size_bytes()
+        );
     }
 
-    return exists;
+    return true;
 }
 
 
diff --git a/src/lagrangian/intermediate/conversion/ensight/ensightOutputCloud.H b/src/lagrangian/intermediate/conversion/ensight/ensightOutputCloud.H
index 21a3f196c6..894a6aa21c 100644
--- a/src/lagrangian/intermediate/conversion/ensight/ensightOutputCloud.H
+++ b/src/lagrangian/intermediate/conversion/ensight/ensightOutputCloud.H
@@ -5,7 +5,7 @@
     \\  /    A nd           | www.openfoam.com
      \\/     M anipulation  |
 -------------------------------------------------------------------------------
-    Copyright (C) 2016-2019 OpenCFD Ltd.
+    Copyright (C) 2016-2021 OpenCFD Ltd.
 -------------------------------------------------------------------------------
 License
     This file is part of OpenFOAM.
@@ -47,7 +47,7 @@ SourceFiles
 namespace Foam
 {
 
-// Forward declarations
+// Forward Declarations
 class fvMesh;
 
 namespace ensightOutput
@@ -62,8 +62,7 @@ bool writeCloudPositions
     const fvMesh& mesh,
     const word& cloudName,
     bool exists,
-    autoPtr<ensightFile>& output,
-    Pstream::commsTypes comm = Pstream::commsTypes::blocking
+    autoPtr<ensightFile>& output
 );
 
 
@@ -72,8 +71,7 @@ template<class Type>
 bool writeCloudField
 (
     const IOField<Type>& field,
-    ensightFile& os,
-    Pstream::commsTypes comm = Pstream::commsTypes::blocking
+    ensightFile& os
 );
 
 
@@ -84,11 +82,33 @@ bool writeCloudField
 (
     const IOobject& io,
     bool exists,
-    autoPtr<ensightFile>& output,
-    Pstream::commsTypes comm = Pstream::commsTypes::blocking
+    autoPtr<ensightFile>& output
 );
 
 
+/*---------------------------------------------------------------------------*\
+                        Namespace ensightOutput::Detail
+\*---------------------------------------------------------------------------*/
+
+namespace Detail
+{
+
+//- Write cloud field data (serial) with rounding and newlines.
+//  \return the current output count
+template<class Type>
+label writeCloudFieldContent
+(
+    ensightFile& os,
+    const UList<Type>& fld,
+    label count = 0     //!< The current output count
+);
+
+
+} // End namespace Detail
+
+
+// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
+
 } // End namespace ensightOutput
 } // End namespace Foam
 
diff --git a/src/lagrangian/intermediate/conversion/ensight/ensightOutputCloudTemplates.C b/src/lagrangian/intermediate/conversion/ensight/ensightOutputCloudTemplates.C
index 26f3840b7e..08478a8d59 100644
--- a/src/lagrangian/intermediate/conversion/ensight/ensightOutputCloudTemplates.C
+++ b/src/lagrangian/intermediate/conversion/ensight/ensightOutputCloudTemplates.C
@@ -27,6 +27,42 @@ License
 
 #include "ensightOutputCloud.H"
 #include "ensightPTraits.H"
+#include "globalIndex.H"
+
+// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
+
+template<class Type>
+Foam::label Foam::ensightOutput::Detail::writeCloudFieldContent
+(
+    ensightFile& os,
+    const UList<Type>& field,
+    label count
+)
+{
+    // Write master data
+    for (Type val : field)          // <-- working on a copy!
+    {
+        if (mag(val) < 1e-90)       // approximately root(ROOTVSMALL)
+        {
+            val = Zero;
+        }
+
+        for (direction d=0; d < pTraits<Type>::nComponents; ++d)
+        {
+            const direction cmpt = ensightPTraits<Type>::componentOrder[d];
+
+            os.write(component(val, cmpt));
+
+            if (++count % 6 == 0)
+            {
+                os.newline();
+            }
+        }
+    }
+
+    return count;
+}
+
 
 // * * * * * * * * * * * * * * * Global Functions  * * * * * * * * * * * * * //
 
@@ -34,8 +70,7 @@ template<class Type>
 bool Foam::ensightOutput::writeCloudField
 (
     const IOField<Type>& field,
-    ensightFile& os,
-    Pstream::commsTypes comm
+    ensightFile& os
 )
 {
     if (returnReduce(field.empty(), andOp<bool>()))
@@ -43,59 +78,51 @@ bool Foam::ensightOutput::writeCloudField
         return false;
     }
 
+    // Size information (offsets are irrelevant)
+    globalIndex procAddr;
+    if (Pstream::parRun())
+    {
+        procAddr.reset(UPstream::listGatherValues<label>(field.size()));
+    }
+    else
+    {
+        procAddr.reset(labelList(Foam::one{}, field.size()));
+    }
+
+
     if (Pstream::master())
     {
         // 6 values per line
         label count = 0;
 
-        // Master
-        for (Type val : field)          // <-- working on a copy
+        // Write master data
+        count = ensightOutput::Detail::writeCloudFieldContent
+        (
+            os,
+            field,
+            count
+        );
+
+        // Receive and write
+        DynamicList<Type> recvData(procAddr.maxNonLocalSize());
+
+        for (const label proci : procAddr.subProcs())
         {
-            if (mag(val) < 1e-90)       // approximately root(ROOTVSMALL)
-            {
-                val = Zero;
-            }
+            recvData.resize_nocopy(procAddr.localSize(proci));
+            UIPstream::read
+            (
+                UPstream::commsTypes::scheduled,
+                proci,
+                recvData.data_bytes(),
+                recvData.size_bytes()
+            );
 
-            for (direction d=0; d < pTraits<Type>::nComponents; ++d)
-            {
-                const direction cmpt =
-                    ensightPTraits<Type>::componentOrder[d];
-
-                os.write(component(val, cmpt));
-
-                if (++count % 6 == 0)
-                {
-                    os.newline();
-                }
-            }
-        }
-
-        // Slaves
-        for (const int slave : Pstream::subProcs())
-        {
-            IPstream fromSlave(comm, slave);
-            Field<Type> recv(fromSlave);
-
-            for (Type val : recv)       // <-- working on a copy
-            {
-                if (mag(val) < 1e-90)   // approximately root(ROOTVSMALL)
-                {
-                    val = Zero;
-                }
-
-                for (direction d=0; d < pTraits<Type>::nComponents; ++d)
-                {
-                    const direction cmpt =
-                        ensightPTraits<Type>::componentOrder[d];
-
-                    os.write(component(val, cmpt));
-
-                    if (++count % 6 == 0)
-                    {
-                        os.newline();
-                    }
-                }
-            }
+            count = ensightOutput::Detail::writeCloudFieldContent
+            (
+                os,
+                recvData,
+                count
+            );
         }
 
         // Add final newline if required
@@ -106,8 +133,14 @@ bool Foam::ensightOutput::writeCloudField
     }
     else
     {
-        OPstream toMaster(comm, Pstream::masterNo());
-        toMaster << field;
+        // Send
+        UOPstream::write
+        (
+            UPstream::commsTypes::scheduled,
+            Pstream::masterNo(),
+            field.cdata_bytes(),
+            field.size_bytes()
+        );
     }
 
     return true;
@@ -119,8 +152,7 @@ bool Foam::ensightOutput::writeCloudField
 (
     const IOobject& io,
     const bool exists,
-    autoPtr<ensightFile>& output,
-    Pstream::commsTypes comm
+    autoPtr<ensightFile>& output
 )
 {
     if (exists)
@@ -134,7 +166,7 @@ bool Foam::ensightOutput::writeCloudField
 
         IOField<Type> field(fieldObj);
 
-        writeCloudField(field, output.ref(), comm);
+        writeCloudField(field, output.ref());
     }
 
     return true;