diff --git a/src/finiteArea/faMesh/faBoundaryMesh/faBoundaryMesh.C b/src/finiteArea/faMesh/faBoundaryMesh/faBoundaryMesh.C index f94d5fa4bf..fbae5aedd6 100644 --- a/src/finiteArea/faMesh/faBoundaryMesh/faBoundaryMesh.C +++ b/src/finiteArea/faMesh/faBoundaryMesh/faBoundaryMesh.C @@ -174,14 +174,46 @@ void Foam::faBoundaryMesh::calcGeometry() // force construction. (void)mesh_.pointAreaNormals(); - forAll(*this, patchi) - { - operator[](patchi).initGeometry(); - } + PstreamBuffers pBufs(Pstream::defaultCommsType); - forAll(*this, patchi) + if + ( + pBufs.commsType() == Pstream::commsTypes::blocking + || pBufs.commsType() == Pstream::commsTypes::nonBlocking + ) { - operator[](patchi).calcGeometry(); + forAll(*this, patchi) + { + operator[](patchi).initGeometry(pBufs); + } + + pBufs.finishedSends(); + + forAll(*this, patchi) + { + operator[](patchi).calcGeometry(pBufs); + } + } + else if (pBufs.commsType() == Pstream::commsTypes::scheduled) + { + const lduSchedule& patchSchedule = mesh().globalData().patchSchedule(); + + // Dummy. + pBufs.finishedSends(); + + for (const auto& patchEval : patchSchedule) + { + const label patchi = patchEval.patch; + + if (patchEval.init) + { + operator[](patchi).initGeometry(pBufs); + } + else + { + operator[](patchi).calcGeometry(pBufs); + } + } } } @@ -633,32 +665,92 @@ void Foam::faBoundaryMesh::movePoints(const pointField& p) // force construction. (void)mesh_.pointAreaNormals(); - faPatchList& patches = *this; + PstreamBuffers pBufs(Pstream::defaultCommsType); - forAll(patches, patchi) + if + ( + pBufs.commsType() == Pstream::commsTypes::blocking + || pBufs.commsType() == Pstream::commsTypes::nonBlocking + ) { - patches[patchi].initMovePoints(p); + forAll(*this, patchi) + { + operator[](patchi).initMovePoints(pBufs, p); + } + + pBufs.finishedSends(); + + forAll(*this, patchi) + { + operator[](patchi).movePoints(pBufs, p); + } } - - forAll(patches, patchi) + else if (pBufs.commsType() == Pstream::commsTypes::scheduled) { - patches[patchi].movePoints(p); + const lduSchedule& patchSchedule = mesh().globalData().patchSchedule(); + + // Dummy. + pBufs.finishedSends(); + + for (const auto& schedEval : patchSchedule) + { + const label patchi = schedEval.patch; + + if (schedEval.init) + { + operator[](patchi).initMovePoints(pBufs, p); + } + else + { + operator[](patchi).movePoints(pBufs, p); + } + } } } void Foam::faBoundaryMesh::updateMesh() { - faPatchList& patches = *this; + PstreamBuffers pBufs(Pstream::defaultCommsType); - forAll(patches, patchi) + if + ( + pBufs.commsType() == Pstream::commsTypes::blocking + || pBufs.commsType() == Pstream::commsTypes::nonBlocking + ) { - patches[patchi].initUpdateMesh(); + forAll(*this, patchi) + { + operator[](patchi).initUpdateMesh(pBufs); + } + + pBufs.finishedSends(); + + forAll(*this, patchi) + { + operator[](patchi).updateMesh(pBufs); + } } - - forAll(patches, patchi) + else if (pBufs.commsType() == Pstream::commsTypes::scheduled) { - patches[patchi].updateMesh(); + const lduSchedule& patchSchedule = mesh().globalData().patchSchedule(); + + // Dummy. + pBufs.finishedSends(); + + for (const auto& schedEval : patchSchedule) + { + const label patchi = schedEval.patch; + + if (schedEval.init) + { + operator[](patchi).initUpdateMesh(pBufs); + } + else + { + operator[](patchi).updateMesh(pBufs); + } + } } } diff --git a/src/finiteArea/faMesh/faMeshDemandDrivenData.C b/src/finiteArea/faMesh/faMeshDemandDrivenData.C index f69bd76aaf..e56b1c13c2 100644 --- a/src/finiteArea/faMesh/faMeshDemandDrivenData.C +++ b/src/finiteArea/faMesh/faMeshDemandDrivenData.C @@ -6,7 +6,7 @@ \\/ M anipulation | ------------------------------------------------------------------------------- Copyright (C) 2016-2017 Wikki Ltd - Copyright (C) 2018-2021 OpenCFD Ltd. + Copyright (C) 2018-2022 OpenCFD Ltd. ------------------------------------------------------------------------------- License This file is part of OpenFOAM. @@ -1090,7 +1090,7 @@ void Foam::faMesh::calcPointAreaNormals_orig(vectorField& result) const Pstream::commsTypes::blocking, procPatch.neighbProcNo(), patchPointNormals.cdata_bytes(), - patchPointNormals.byteSize() + patchPointNormals.size_bytes() ); } @@ -1106,7 +1106,7 @@ void Foam::faMesh::calcPointAreaNormals_orig(vectorField& result) const Pstream::commsTypes::blocking, procPatch.neighbProcNo(), ngbPatchPointNormals.data_bytes(), - ngbPatchPointNormals.byteSize() + ngbPatchPointNormals.size_bytes() ); } @@ -1330,7 +1330,7 @@ void Foam::faMesh::calcPointAreaNormals(vectorField& result) const ( Pstream::commsTypes::blocking, procPatch.neighbProcNo(), - reinterpret_cast(patchPointNormals.cdata()), + patchPointNormals.cdata_bytes(), patchPointNormals.size_bytes() ); } @@ -1343,7 +1343,7 @@ void Foam::faMesh::calcPointAreaNormals(vectorField& result) const ( Pstream::commsTypes::blocking, procPatch.neighbProcNo(), - reinterpret_cast(patchPointNormals.data()), + patchPointNormals.data_bytes(), patchPointNormals.size_bytes() ); } @@ -1656,8 +1656,8 @@ void Foam::faMesh::calcPointAreaNormalsByQuadricsFit(vectorField& result) const ( Pstream::commsTypes::blocking, procPatch.neighbProcNo(), - toNgbProcLsPoints.byteSize() - + toNgbProcLsPointStarts.byteSize() + toNgbProcLsPoints.size_bytes() + + toNgbProcLsPointStarts.size_bytes() + 10*sizeof(label) ); @@ -1686,7 +1686,7 @@ void Foam::faMesh::calcPointAreaNormalsByQuadricsFit(vectorField& result) const Pstream::commsTypes::blocking, procPatch.neighbProcNo(), 10*patchPointLabels.size()*sizeof(vector) - + fromNgbProcLsPointStarts.byteSize() + + fromNgbProcLsPointStarts.size_bytes() + 10*sizeof(label) ); diff --git a/src/finiteArea/faMesh/faPatches/basic/coupled/coupledFaPatch.H b/src/finiteArea/faMesh/faPatches/basic/coupled/coupledFaPatch.H index ed36517d76..47e59cbe5f 100644 --- a/src/finiteArea/faMesh/faPatches/basic/coupled/coupledFaPatch.H +++ b/src/finiteArea/faMesh/faPatches/basic/coupled/coupledFaPatch.H @@ -6,6 +6,7 @@ \\/ M anipulation | ------------------------------------------------------------------------------- Copyright (C) 2016-2017 Wikki Ltd + Copyright (C) 2022 OpenCFD Ltd. ------------------------------------------------------------------------------- License This file is part of OpenFOAM. @@ -150,7 +151,16 @@ public: return true; } - //- Are the coupled planes separated + //- Does this side own the patch ? + virtual bool owner() const = 0; + + //- Does the coupled side own the patch ? + virtual bool neighbour() const + { + return !owner(); + } + + //- Are the coupled planes separated? bool separated() const { return separation_.size(); @@ -204,16 +214,16 @@ public: //- Initialise the calculation of the patch geometry - virtual void initGeometry() = 0; + virtual void initGeometry(PstreamBuffers&) = 0; //- Calculate the patch geometry - virtual void calcGeometry() = 0; + virtual void calcGeometry(PstreamBuffers&) = 0; //- Initialise the patches for moving points - virtual void initMovePoints(const pointField&) = 0; + virtual void initMovePoints(PstreamBuffers&, const pointField&) = 0; //- Correct patches after moving points - virtual void movePoints(const pointField&) = 0; + virtual void movePoints(PstreamBuffers&, const pointField&) = 0; // Access functions for demand driven data diff --git a/src/finiteArea/faMesh/faPatches/constraint/cyclic/cyclicFaPatch.C b/src/finiteArea/faMesh/faPatches/constraint/cyclic/cyclicFaPatch.C index 5af8a2ae23..cb3612260f 100644 --- a/src/finiteArea/faMesh/faPatches/constraint/cyclic/cyclicFaPatch.C +++ b/src/finiteArea/faMesh/faPatches/constraint/cyclic/cyclicFaPatch.C @@ -236,28 +236,36 @@ void Foam::cyclicFaPatch::makeDeltaCoeffs(scalarField& dc) const } -void Foam::cyclicFaPatch::initGeometry() +void Foam::cyclicFaPatch::initGeometry(PstreamBuffers& pBufs) { - faPatch::initGeometry(); + faPatch::initGeometry(pBufs); } -void Foam::cyclicFaPatch::calcGeometry() +void Foam::cyclicFaPatch::calcGeometry(PstreamBuffers& pBufs) { - faPatch::calcGeometry(); + faPatch::calcGeometry(pBufs); calcTransforms(); } -void Foam::cyclicFaPatch::initMovePoints(const pointField& p) +void Foam::cyclicFaPatch::initMovePoints +( + PstreamBuffers& pBufs, + const pointField& p +) { - faPatch::initMovePoints(p); + faPatch::initMovePoints(pBufs, p); } -void Foam::cyclicFaPatch::movePoints(const pointField& p) +void Foam::cyclicFaPatch::movePoints +( + PstreamBuffers& pBufs, + const pointField& p +) { - faPatch::movePoints(p); + faPatch::movePoints(pBufs, p); calcTransforms(); } diff --git a/src/finiteArea/faMesh/faPatches/constraint/cyclic/cyclicFaPatch.H b/src/finiteArea/faMesh/faPatches/constraint/cyclic/cyclicFaPatch.H index cb170ce658..9e4059fdff 100644 --- a/src/finiteArea/faMesh/faPatches/constraint/cyclic/cyclicFaPatch.H +++ b/src/finiteArea/faMesh/faPatches/constraint/cyclic/cyclicFaPatch.H @@ -109,8 +109,9 @@ public: // Access - //- Is this the master side? Yes: it contains both sets of faces - virtual bool master() const + //- Does this side own the patch ? + //- Yes: it contains both sets of faces + virtual bool owner() const { return true; } @@ -122,11 +123,6 @@ public: return index(); } - virtual bool owner() const - { - return master(); - } - //- Return processor number virtual const cyclicLduInterface& neighbPatch() const { @@ -147,16 +143,16 @@ public: } //- Initialise the calculation of the patch geometry - virtual void initGeometry(); + virtual void initGeometry(PstreamBuffers&); //- Calculate the patch geometry - virtual void calcGeometry(); + virtual void calcGeometry(PstreamBuffers&); //- Initialise the patches for moving points - virtual void initMovePoints(const pointField&); + virtual void initMovePoints(PstreamBuffers&, const pointField&); //- Correct patches after moving points - virtual void movePoints(const pointField&); + virtual void movePoints(PstreamBuffers&, const pointField&); //- Return delta (P to N) vectors across coupled patch virtual tmp delta() const; diff --git a/src/finiteArea/faMesh/faPatches/constraint/processor/processorFaPatch.C b/src/finiteArea/faMesh/faPatches/constraint/processor/processorFaPatch.C index e8d20553c6..15f2eb9933 100644 --- a/src/finiteArea/faMesh/faPatches/constraint/processor/processorFaPatch.C +++ b/src/finiteArea/faMesh/faPatches/constraint/processor/processorFaPatch.C @@ -128,12 +128,6 @@ Foam::label Foam::processorFaPatch::comm() const } -int Foam::processorFaPatch::tag() const -{ - return Pstream::msgType(); -} - - void Foam::processorFaPatch::makeNonGlobalPatchPoints() const { // If it is not running parallel or there are no global points @@ -181,16 +175,21 @@ void Foam::processorFaPatch::makeNonGlobalPatchPoints() const } -void Foam::processorFaPatch::initGeometry() +void Foam::processorFaPatch::initGeometry(PstreamBuffers& pBufs) { if (Pstream::parRun()) { - OPstream toNeighbProc - ( - Pstream::commsTypes::blocking, - neighbProcNo(), - 3*(sizeof(label) + size()*sizeof(vector)) - ); + if (neighbProcNo() >= Pstream::nProcs(pBufs.comm())) + { + FatalErrorInFunction + << "On patch " << name() + << " trying to access out of range neighbour processor " + << neighbProcNo() << ". This can happen if" << nl + << " trying to run on an incorrect number of processors" + << exit(FatalError); + } + + UOPstream toNeighbProc(neighbProcNo(), pBufs); toNeighbProc << edgeCentres() @@ -200,17 +199,13 @@ void Foam::processorFaPatch::initGeometry() } -void Foam::processorFaPatch::calcGeometry() +void Foam::processorFaPatch::calcGeometry(PstreamBuffers& pBufs) { if (Pstream::parRun()) { { - IPstream fromNeighbProc - ( - Pstream::commsTypes::blocking, - neighbProcNo(), - 3*(sizeof(label) + size()*sizeof(vector)) - ); + UIPstream fromNeighbProc(neighbProcNo(), pBufs); + fromNeighbProc >> neighbEdgeCentres_ >> neighbEdgeLengths_ @@ -246,28 +241,46 @@ void Foam::processorFaPatch::calcGeometry() } -void Foam::processorFaPatch::initMovePoints(const pointField& p) +void Foam::processorFaPatch::initMovePoints +( + PstreamBuffers& pBufs, + const pointField& p +) { - faPatch::movePoints(p); - initGeometry(); + faPatch::movePoints(pBufs, p); + initGeometry(pBufs); } -void Foam::processorFaPatch::movePoints(const pointField&) +void Foam::processorFaPatch::movePoints +( + PstreamBuffers& pBufs, + const pointField& +) { - calcGeometry(); + processorFaPatch::calcGeometry(pBufs); } -void Foam::processorFaPatch::initUpdateMesh() +void Foam::processorFaPatch::initUpdateMesh(PstreamBuffers& pBufs) { // For completeness - faPatch::initUpdateMesh(); + faPatch::initUpdateMesh(pBufs); neighbPointsPtr_.clear(); if (Pstream::parRun()) { + if (neighbProcNo() >= Pstream::nProcs(pBufs.comm())) + { + FatalErrorInFunction + << "On patch " << name() + << " trying to access out of range neighbour processor " + << neighbProcNo() << ". This can happen if" << nl + << " trying to run on an incorrect number of processors" + << exit(FatalError); + } + // Express all points as patch edge and index in edge. labelList patchEdge(nPoints()); labelList indexInEdge(nPoints()); @@ -288,12 +301,7 @@ void Foam::processorFaPatch::initUpdateMesh() indexInEdge[patchPointI] = e.find(pointLabels()[patchPointI]); } - OPstream toNeighbProc - ( - Pstream::commsTypes::blocking, - neighbProcNo(), - 2*sizeof(label) + 2*nPoints()*sizeof(label) - ); + UOPstream toNeighbProc(neighbProcNo(), pBufs); toNeighbProc << patchEdge @@ -302,10 +310,12 @@ void Foam::processorFaPatch::initUpdateMesh() } -void Foam::processorFaPatch::updateMesh() +void Foam::processorFaPatch::updateMesh(PstreamBuffers& pBufs) { // For completeness - faPatch::updateMesh(); + faPatch::updateMesh(pBufs); + + neighbPointsPtr_.clear(); if (Pstream::parRun()) { @@ -315,11 +325,8 @@ void Foam::processorFaPatch::updateMesh() { // Note cannot predict exact size since edgeList not (yet) sent as // binary entity but as List of edges. - IPstream fromNeighbProc - ( - Pstream::commsTypes::blocking, - neighbProcNo() - ); + + UIPstream fromNeighbProc(neighbProcNo(), pBufs); fromNeighbProc >> nbrPatchEdge @@ -352,7 +359,6 @@ void Foam::processorFaPatch::updateMesh() { // Differing number of points. Probably patch includes // part of a cyclic. - neighbPointsPtr_.clear(); } } } diff --git a/src/finiteArea/faMesh/faPatches/constraint/processor/processorFaPatch.H b/src/finiteArea/faMesh/faPatches/constraint/processor/processorFaPatch.H index 89875c4434..8a15653964 100644 --- a/src/finiteArea/faMesh/faPatches/constraint/processor/processorFaPatch.H +++ b/src/finiteArea/faMesh/faPatches/constraint/processor/processorFaPatch.H @@ -115,22 +115,22 @@ protected: // Geometry Functions //- Initialise the calculation of the patch geometry - void initGeometry(); + void initGeometry(PstreamBuffers&); //- Calculate the patch geometry - void calcGeometry(); + void calcGeometry(PstreamBuffers&); //- Initialise the patches for moving points - void initMovePoints(const pointField&); + void initMovePoints(PstreamBuffers&, const pointField&); //- Correct patches after moving points - void movePoints(const pointField&); + void movePoints(PstreamBuffers&, const pointField&); //- Initialise the update of the patch topology - virtual void initUpdateMesh(); + virtual void initUpdateMesh(PstreamBuffers&); //- Update of the patch topology - virtual void updateMesh(); + virtual void updateMesh(PstreamBuffers&); public: @@ -194,20 +194,21 @@ public: return Pstream::parRun(); } - //- Is this the master side? - virtual bool master() const noexcept + //- Does this side own the patch ? + virtual bool owner() const noexcept { return (myProcNo_ < neighbProcNo_); } - // Communications support + //- The message tag to use for communication + virtual int tag() const + { + return UPstream::msgType(); + } - //- Return communicator used for communication - virtual label comm() const; - - //- Return message tag to use for communication - virtual int tag() const; + //- Return communicator used for communication + virtual label comm() const; //- Return face transformation tensor diff --git a/src/finiteArea/faMesh/faPatches/faPatch/faPatch.C b/src/finiteArea/faMesh/faPatches/faPatch/faPatch.C index 5a5ce3888d..f87e82d8aa 100644 --- a/src/finiteArea/faMesh/faPatches/faPatch/faPatch.C +++ b/src/finiteArea/faMesh/faPatches/faPatch/faPatch.C @@ -531,7 +531,7 @@ const Foam::scalarField& Foam::faPatch::weights() const } -void Foam::faPatch::movePoints(const pointField& points) +void Foam::faPatch::movePoints(PstreamBuffers&, const pointField&) {} diff --git a/src/finiteArea/faMesh/faPatches/faPatch/faPatch.H b/src/finiteArea/faMesh/faPatches/faPatch/faPatch.H index 11cc5bcaac..2090cbe650 100644 --- a/src/finiteArea/faMesh/faPatches/faPatch/faPatch.H +++ b/src/finiteArea/faMesh/faPatches/faPatch/faPatch.H @@ -116,26 +116,26 @@ protected: void calcPointEdges() const; //- Initialise the calculation of the patch geometry - virtual void initGeometry() + virtual void initGeometry(PstreamBuffers&) {} //- Calculate the patch geometry - virtual void calcGeometry() + virtual void calcGeometry(PstreamBuffers&) {} //- Initialise the patches for moving points - virtual void initMovePoints(const pointField&) + virtual void initMovePoints(PstreamBuffers&, const pointField&) {} //- Correct patch after moving points - virtual void movePoints(const pointField&); + virtual void movePoints(PstreamBuffers&, const pointField&); //- Initialise the update of the patch topology - virtual void initUpdateMesh() + virtual void initUpdateMesh(PstreamBuffers&) {} //- Update of the patch topology - virtual void updateMesh() + virtual void updateMesh(PstreamBuffers&) {}