ENH: improve handling of database mapping for multi-world

- cleanup initialization (to avoid stealing tokens)

src/TurbulenceModels/compressible/turbulentFluidThermoModels/derivedFvPatchFields/turbulentTemperatureCoupledBaffleMixed/turbulentTemperatureCoupledBaffleMixedFvPatchScalarField.C

@@ -6,7 +6,7 @@
      \\/     M anipulation  |
 -------------------------------------------------------------------------------
     Copyright (C) 2011-2016 OpenFOAM Foundation
-    Copyright (C) 2019-2020 OpenCFD Ltd.
+    Copyright (C) 2019-2021 OpenCFD Ltd.
 -------------------------------------------------------------------------------
 License
     This file is part of OpenFOAM.
@@ -150,17 +150,19 @@ turbulentTemperatureCoupledBaffleMixedFvPatchScalarField
         valueFraction() = 1.0;
     }
 
-    // Store patch value as initial guess when running in database mode
+// This blocks (crashes) with more than two worlds!
-    mappedPatchFieldBase<scalar>::initRetrieveField
+//
-    (
+///    // Store patch value as initial guess when running in database mode
-        this->internalField().name(),
+///    mappedPatchFieldBase<scalar>::initRetrieveField
-        *this
+///    (
-    );
+///        this->internalField().name(),
-    mappedPatchFieldBase<scalar>::initRetrieveField
+///        *this
-    (
+///    );
-        this->internalField().name() + "_weights",
+///    mappedPatchFieldBase<scalar>::initRetrieveField
-        this->patch().deltaCoeffs()
+///    (
-    );
+///        this->internalField().name() + "_weights",
+///        this->patch().deltaCoeffs()
+///    );
 }
 
 

src/finiteVolume/fields/fvPatchFields/derived/mappedField/mappedMixedFieldFvPatchField/mappedMixedFieldFvPatchField.C

@@ -5,7 +5,7 @@
     \\  /    A nd           | www.openfoam.com
      \\/     M anipulation  |
 -------------------------------------------------------------------------------
-    Copyright (C) 2019-2020 OpenCFD Ltd.
+    Copyright (C) 2019-2021 OpenCFD Ltd.
 -------------------------------------------------------------------------------
 License
     This file is part of OpenFOAM.
@@ -61,39 +61,7 @@ Foam::mappedMixedFieldFvPatchField<Type>::mappedMixedFieldFvPatchField
     mappedPatchBase(p.patch(), dict),
     mappedPatchFieldBase<Type>(*this, *this, dict),
     weightFieldName_(dict.getOrDefault<word>("weightField", word::null))
-{
+{}
-    mixedFvPatchField<Type>::operator=
-    (
-        Field<Type>("value", dict, p.size())
-    );
-
-    if (dict.found("refValue"))
-    {
-        // Full restart
-        this->refValue() = Field<Type>("refValue", dict, p.size());
-        this->refGrad() = Field<Type>("refGradient", dict, p.size());
-        this->valueFraction() = scalarField("valueFraction", dict, p.size());
-    }
-    else
-    {
-        // Start from user entered data. Assume fixedValue.
-        this->refValue() = *this;
-        this->refGrad() = Zero;
-        this->valueFraction() = 1.0;
-    }
-
-    // Store patch value as initial guess when running in database mode
-    mappedPatchFieldBase<Type>::initRetrieveField
-    (
-        this->internalField().name(),
-        *this
-    );
-    mappedPatchFieldBase<Type>::initRetrieveField
-    (
-        this->internalField().name() + "_weights",
-        this->patch().deltaCoeffs()
-    );
-}
 
 
 template<class Type>

src/finiteVolume/fields/fvPatchFields/derived/mappedField/mappedPatchFieldBase.C

@@ -55,29 +55,34 @@ void Foam::mappedPatchFieldBase<Type>::storeField
     const objectRegistry& obr,
     const word& region,
     const word& patch,
+    const label myComm,
     const labelListList& procToMap,
     const word& fieldName,
     const Field<T>& fld
 ) const
 {
     // Store my data onto database
-    //const label myRank = Pstream::myProcNo(0);  // comm_
-    const label nProcs = Pstream::nProcs(0);    // comm_
 
-    for (label domain = 0; domain < nProcs; domain++)
+    const auto& procIDs = UPstream::procID(myComm);
+
+    forAll(procToMap, ranki)
     {
-        const labelList& map = procToMap[domain];
+        const labelList& map = procToMap[ranki];
+        const label proci = procIDs[ranki];
 
         if (map.size())
         {
             const Field<T> subFld(fld, map);
 
-            const objectRegistry& subObr = mappedPatchBase::subRegistry
+            auto& subObr = const_cast<objectRegistry&>
             (
-                obr,
+                mappedPatchBase::subRegistry
-                mapper_.sendPath(domain)
+                (
-              / region
+                    obr,
-              / patch
+                    mapper_.sendPath(proci)
+                  / region
+                  / patch
+                )
             );
 
             if (fvPatchField<Type>::debug)
@@ -88,12 +93,7 @@ void Foam::mappedPatchFieldBase<Type>::storeField
                     << " as:" << subObr.objectPath() << endl;
             }
 
-            mappedPatchBase::storeField
+            mappedPatchBase::storeField(subObr, fieldName, subFld);
-            (
-                const_cast<objectRegistry&>(subObr),
-                fieldName,
-                subFld
-            );
         }
     }
 }
@@ -107,27 +107,32 @@ bool Foam::mappedPatchFieldBase<Type>::retrieveField
     const objectRegistry& obr,
     const word& region,
     const word& patch,
+    const label myComm,
     const labelListList& procToMap,
     const word& fieldName,
     Field<T>& fld
 ) const
 {
-    // Store my data onto database
+    const auto& procIDs = UPstream::procID(myComm);
-    const label nProcs = Pstream::nProcs(0);    // comm_
 
     bool ok = true;
 
-    for (label domain = 0; domain < nProcs; domain++)
+    forAll(procToMap, ranki)
     {
-        const labelList& map = procToMap[domain];
+        const labelList& map = procToMap[ranki];
+        const label proci = procIDs[ranki];
+
         if (map.size())
         {
-            const objectRegistry& subObr = mappedPatchBase::subRegistry
+            auto& subObr = const_cast<objectRegistry&>
             (
-                obr,
+                mappedPatchBase::subRegistry
-                mapper_.receivePath(domain)
+                (
-              / region
+                    obr,
-              / patch
+                    mapper_.receivePath(proci)
+                  / region
+                  / patch
+                )
             );
 
             const IOField<T>* subFldPtr = subObr.getObjectPtr<IOField<T>>
@@ -174,12 +179,7 @@ bool Foam::mappedPatchFieldBase<Type>::retrieveField
                 // detect it if necessary.
                 const Field<T> dummyFld(0);
 
-                mappedPatchBase::storeField
+                mappedPatchBase::storeField(subObr, fieldName, dummyFld);
-                (
-                    const_cast<objectRegistry&>(subObr),
-                    fieldName,
-                    dummyFld
-                );
 
                 ok = false;
             }
@@ -207,6 +207,8 @@ void Foam::mappedPatchFieldBase<Type>::initRetrieveField
     const Field<T>& fld
 ) const
 {
+    // Old code. Likely not quite correct...
+
     // Store my data onto database
     const label nProcs = Pstream::nProcs(0);    // comm_
 
@@ -215,12 +217,15 @@ void Foam::mappedPatchFieldBase<Type>::initRetrieveField
         const labelList& constructMap = map[domain];
         if (constructMap.size())
         {
-            const objectRegistry& subObr = mappedPatchBase::subRegistry
+            auto& subObr = const_cast<objectRegistry&>
             (
-                obr,
+                mappedPatchBase::subRegistry
-                mapper_.receivePath(domain)
+                (
-              / region
+                    obr,
-              / patch
+                    mapper_.receivePath(domain)
+                  / region
+                  / patch
+                )
             );
 
             const Field<T> receiveFld(fld, constructMap);
@@ -235,12 +240,7 @@ void Foam::mappedPatchFieldBase<Type>::initRetrieveField
                     << " as:" << subObr.objectPath() << endl;
             }
 
-            mappedPatchBase::storeField
+            mappedPatchBase::storeField(subObr, fieldName, receiveFld);
-            (
-                const_cast<objectRegistry&>(subObr),
-                fieldName,
-                receiveFld
-            );
         }
     }
 }
@@ -251,6 +251,7 @@ template<class T>
 bool Foam::mappedPatchFieldBase<Type>::storeAndRetrieveField
 (
     const word& fieldName,
+    const label myComm,
     const labelListList& subMap,
     const label constructSize,
     const labelListList& constructMap,
@@ -264,6 +265,7 @@ bool Foam::mappedPatchFieldBase<Type>::storeAndRetrieveField
         patchField_.internalField().time(),
         patchField_.patch().boundaryMesh().mesh().name(),
         patchField_.patch().name(),
+        myComm,
         subMap,
         fieldName,
         fld
@@ -276,6 +278,7 @@ bool Foam::mappedPatchFieldBase<Type>::storeAndRetrieveField
         patchField_.internalField().time(),
         mapper_.sampleRegion(),
         mapper_.samplePatch(),
+        myComm,
         constructMap,
         fieldName,
         work
@@ -525,6 +528,8 @@ void Foam::mappedPatchFieldBase<Type>::distribute
 {
     if (mapper_.sampleDatabase())
     {
+        const label myComm = mapper_.getCommunicator();  // Get or create
+
         if (mapper_.mode() != mappedPatchBase::NEARESTPATCHFACEAMI)
         {
             // Store my data on send buffers
@@ -533,6 +538,7 @@ void Foam::mappedPatchFieldBase<Type>::distribute
                 patchField_.internalField().time(),
                 patchField_.patch().boundaryMesh().mesh().name(),
                 patchField_.patch().name(),
+                myComm,
                 mapper_.map().subMap(),
                 fieldName,
                 fld
@@ -545,6 +551,7 @@ void Foam::mappedPatchFieldBase<Type>::distribute
                 patchField_.internalField().time(),
                 mapper_.sampleRegion(),
                 mapper_.samplePatch(),
+                myComm,
                 mapper_.map().constructMap(),
                 fieldName,
                 fld
@@ -566,6 +573,7 @@ void Foam::mappedPatchFieldBase<Type>::distribute
                 storeAndRetrieveField
                 (
                     fieldName,
+                    myComm,
                     AMI.srcMap().subMap(),
                     AMI.tgtMap().constructSize(),
                     AMI.tgtMap().constructMap(),
@@ -581,6 +589,7 @@ void Foam::mappedPatchFieldBase<Type>::distribute
                 storeAndRetrieveField
                 (
                     fieldName,
+                    myComm,
                     AMI.tgtMap().subMap(),
                     AMI.srcMap().constructSize(),
                     AMI.srcMap().constructMap(),

src/finiteVolume/fields/fvPatchFields/derived/mappedField/mappedPatchFieldBase.H

@@ -67,9 +67,9 @@ namespace Foam
 {
 
 // Forward Declarations
-class mappedPatchBase;
 template<class> class interpolation;
 class mapDistribute;
+class mappedPatchBase;
 
 /*---------------------------------------------------------------------------*\
                     Class mappedPatchFieldBase Declaration
@@ -86,7 +86,7 @@ class mappedPatchFieldBase
 
 protected:
 
-    // Protected data
+    // Protected Data
 
         //- Mapping engine
         const mappedPatchBase& mapper_;
@@ -129,6 +129,7 @@ protected:
         bool storeAndRetrieveField
         (
             const word& fieldName,
+            const label myComm,
             const labelListList& subMap,
             const label constructSize,
             const labelListList& constructMap,
@@ -138,6 +139,44 @@ protected:
         ) const;
 
 
+        //- Initialise field to retrieve (e.g. when value read from dictionary)
+        template<class T>
+        void initRetrieveField
+        (
+            const word& fieldName,
+            const Field<T>& fld
+        ) const;
+
+
+        //- Store elements of field onto (sub) registry
+        template<class T>
+        void storeField
+        (
+            const objectRegistry& obr,
+            const word& region,
+            const word& patch,
+            const label myComm,
+            const labelListList& procToMap,
+            const word& fieldName,
+            const Field<T>& fld
+        ) const;
+
+        //- Construct field from registered elements.
+        //  \return true if successful
+        template<class T>
+        bool retrieveField
+        (
+            const bool allowUnset,
+            const objectRegistry& obr,
+            const word& region,
+            const word& patch,
+            const label myComm,
+            const labelListList& procToMap,
+            const word& fieldName,
+            Field<T>& fld
+        ) const;
+
+
 public:
 
     // Constructors
@@ -230,54 +269,19 @@ public:
         virtual void write(Ostream& os) const;
 
 
-        // Helpers
+    // Helpers
 
-            //- Check that patch is of correct type
+        //- Check that patch is of correct type
-            static const mappedPatchBase& mapper
+        static const mappedPatchBase& mapper
-            (
+        (
-                const fvPatch& p,
+            const fvPatch& p,
-                const DimensionedField<Type, volMesh>& iF
+            const DimensionedField<Type, volMesh>& iF
-            );
+        );
 
-            //- Initialise field to retrieve (used e.g. when value read from
+        //- Wrapper for mapDistribute::distribute that knows about
-            //  dictionary)
+        //- dabase mapping
-            template<class T>
+        template<class T>
-            void initRetrieveField
+        void distribute(const word& fieldName, Field<T>& newValues) const;
-            (
-                const word& fieldName,
-                const Field<T>& fld
-            ) const;
-
-            //- Store elements of field onto (sub) registry
-            template<class T>
-            void storeField
-            (
-                const objectRegistry& obr,
-                const word& region,
-                const word& patch,
-                const labelListList& procToMap,
-                const word& fieldName,
-                const Field<T>& fld
-            ) const;
-
-            //- Construct field from registered elements. Return true if
-            //  successful
-            template<class T>
-            bool retrieveField
-            (
-                const bool allowUnset,
-                const objectRegistry& obr,
-                const word& region,
-                const word& patch,
-                const labelListList& procToMap,
-                const word& fieldName,
-                Field<T>& fld
-            ) const;
-
-            //- Wrapper for mapDistribute::distribute that knows about
-            //- dabase mapping
-            template<class T>
-            void distribute(const word& fieldName, Field<T>& newValues) const;
 };
 
 

src/finiteVolume/fields/fvPatchFields/derived/mappedMixed/mappedMixedFvPatchField.C

@@ -5,7 +5,7 @@
     \\  /    A nd           | www.openfoam.com
      \\/     M anipulation  |
 -------------------------------------------------------------------------------
-    Copyright (C) 2020 OpenCFD Ltd.
+    Copyright (C) 2020-2021 OpenCFD Ltd.
 -------------------------------------------------------------------------------
 License
     This file is part of OpenFOAM.
@@ -90,17 +90,19 @@ Foam::mappedMixedFvPatchField<Type>::mappedMixedFvPatchField
         this->valueFraction() = 1.0;
     }
 
-    // Store patch value as initial guess when running in database mode
+// This blocks (crashes) with more than two worlds!
-    mappedPatchFieldBase<Type>::initRetrieveField
+//
-    (
+///    // Store patch value as initial guess when running in database mode
-        this->internalField().name(),
+///    mappedPatchFieldBase<Type>::initRetrieveField
-        *this
+///    (
-    );
+///        this->internalField().name(),
-    mappedPatchFieldBase<Type>::initRetrieveField
+///        *this
-    (
+///    );
-        this->internalField().name() + "_weights",
+///    mappedPatchFieldBase<Type>::initRetrieveField
-        this->patch().deltaCoeffs()
+///    (
-    );
+///        this->internalField().name() + "_weights",
+///        this->patch().deltaCoeffs()
+///    );
 }
 
 

src/functionObjects/utilities/syncObjects/syncObjects.C

@@ -5,7 +5,7 @@
     \\  /    A nd           | www.openfoam.com
      \\/     M anipulation  |
 -------------------------------------------------------------------------------
-    Copyright (C) 2020 OpenCFD Ltd.
+    Copyright (C) 2020-2021 OpenCFD Ltd.
 -------------------------------------------------------------------------------
 License
     This file is part of OpenFOAM.
@@ -39,7 +39,6 @@ namespace Foam
 namespace functionObjects
 {
     defineTypeNameAndDebug(syncObjects, 0);
-
     addToRunTimeSelectionTable
     (
         functionObject,
@@ -50,9 +49,6 @@ namespace functionObjects
 }
 
 
-// * * * * * * * * * * * * * Private Member Functions  * * * * * * * * * * * //
-
-
 // * * * * * * * * * * * * * * * * Constructors  * * * * * * * * * * * * * * //
 
 Foam::functionObjects::syncObjects::syncObjects
@@ -63,14 +59,7 @@ Foam::functionObjects::syncObjects::syncObjects
 )
 :
     functionObject(name),
-    obr_
+    obr_(runTime)
-    (
-        //runTime.lookupObject<objectRegistry>
-        //(
-        //    dict.lookupOrDefault("region", polyMesh::defaultRegion)
-        //)
-        runTime
-    )
 {
     read(dict);
 }
@@ -121,9 +110,12 @@ void Foam::functionObjects::syncObjects::sync()
         dictionary sendDataDict;
         mappedPatchBase::writeDict(sendObr, sendDataDict);
 
-        //Pout<< "** to processor " << proci
+        if (debug & 2)
-        //    << " sendObr:" << sendObr.objectPath()
+        {
-        //    << " sending dictionary:" << sendDataDict << endl;
+            Pout<< "** to processor " << proci
+                << " sendObr:" << sendObr.objectPath()
+                << " sending dictionary:" << sendDataDict << endl;
+        }
         UOPstream os(proci, pBufs);
         os << sendDataDict;
     }
@@ -139,14 +131,14 @@ void Foam::functionObjects::syncObjects::sync()
             obr_,
             mappedPatchBase::receivePath(root_, proci)
         );
-        //Pout<< "** from processor " << proci
-        //        << " receiveObr:" << receiveObr.objectPath()
-        //    << " receiving dictionary" << endl;
         UIPstream is(proci, pBufs);
         const dictionary fromProcDict(is);
-        //Pout<< "** from processor " << proci
+        if (debug & 2)
-        //    << " received dictionary:" << fromProcDict << endl;
+        {
-
+            Pout<< "** from processor " << proci
+                << " receiveObr:" << receiveObr.objectPath()
+                << " received dictionary:" << fromProcDict << endl;
+        }
         mappedPatchBase::readDict
         (
             fromProcDict,