From a1345b7e831639b95888955fad92d2b5efaab87f Mon Sep 17 00:00:00 2001 From: Mark Olesen Date: Wed, 1 Aug 2018 21:55:45 +0200 Subject: [PATCH] ENH: improve syncTools handling of PackedList and bitSet - create a subset copy for sending on the processor patches instead of a List of unsigned ints. Reduces memory overhead and data transfer amount. --- .../polyMesh/globalMeshData/globalMeshData.H | 4 +- .../globalMeshData/globalMeshDataTemplates.C | 114 +++++++-------- .../meshes/polyMesh/syncTools/syncTools.H | 134 +++++++++++------- .../polyMesh/syncTools/syncToolsTemplates.C | 30 ++-- 4 files changed, 147 insertions(+), 135 deletions(-) diff --git a/src/OpenFOAM/meshes/polyMesh/globalMeshData/globalMeshData.H b/src/OpenFOAM/meshes/polyMesh/globalMeshData/globalMeshData.H index 5b0db8f9ca..d17a5f7dba 100644 --- a/src/OpenFOAM/meshes/polyMesh/globalMeshData/globalMeshData.H +++ b/src/OpenFOAM/meshes/polyMesh/globalMeshData/globalMeshData.H @@ -499,7 +499,7 @@ public: template static void syncData ( - List& pointData, + List& elems, const labelListList& slaves, const labelListList& transformedSlaves, const mapDistribute& slavesMap, @@ -512,7 +512,7 @@ public: template static void syncData ( - List& pointData, + List& elems, const labelListList& slaves, const labelListList& transformedSlaves, const mapDistribute& slavesMap, diff --git a/src/OpenFOAM/meshes/polyMesh/globalMeshData/globalMeshDataTemplates.C b/src/OpenFOAM/meshes/polyMesh/globalMeshData/globalMeshDataTemplates.C index 01ed808f25..f70eddefd4 100644 --- a/src/OpenFOAM/meshes/polyMesh/globalMeshData/globalMeshDataTemplates.C +++ b/src/OpenFOAM/meshes/polyMesh/globalMeshData/globalMeshDataTemplates.C @@ -3,7 +3,7 @@ \\ / F ield | OpenFOAM: The Open Source CFD Toolbox \\ / O peration | \\ / A nd | Copyright (C) 2011-2015 OpenFOAM Foundation - \\/ M anipulation | + \\/ M anipulation | Copyright (C) 2018 OpenCFD Ltd. ------------------------------------------------------------------------------- License This file is part of OpenFOAM. @@ -50,44 +50,36 @@ void Foam::globalMeshData::syncData Type& elem = elems[i]; const labelList& slavePoints = slaves[i]; - label nTransformSlavePoints = + + const labelList& transformSlavePoints = ( - transformedSlaves.size() == 0 - ? 0 - : transformedSlaves[i].size() + transformedSlaves.empty() + ? Foam::emptyLabelList + : transformedSlaves[i] ); - if (slavePoints.size()+nTransformSlavePoints > 0) + + // Combine master with untransformed slave data + for (const label pointi : slavePoints) { - // Combine master with untransformed slave data - forAll(slavePoints, j) - { - cop(elem, elems[slavePoints[j]]); - } + cop(elem, elems[pointi]); + } - // Combine master with transformed slave data - if (nTransformSlavePoints) - { - const labelList& transformSlavePoints = transformedSlaves[i]; - forAll(transformSlavePoints, j) - { - cop(elem, elems[transformSlavePoints[j]]); - } - } + // Combine master with transformed slave data + for (const label pointi : transformSlavePoints) + { + cop(elem, elems[pointi]); + } - // Copy result back to slave slots - forAll(slavePoints, j) - { - elems[slavePoints[j]] = elem; - } - if (nTransformSlavePoints) - { - const labelList& transformSlavePoints = transformedSlaves[i]; - forAll(transformSlavePoints, j) - { - elems[transformSlavePoints[j]] = elem; - } - } + // Copy result back to slave slots + for (const label pointi : slavePoints) + { + elems[pointi] = elem; + } + + for (const label pointi : transformSlavePoints) + { + elems[pointi] = elem; } } @@ -121,44 +113,36 @@ void Foam::globalMeshData::syncData Type& elem = elems[i]; const labelList& slavePoints = slaves[i]; - label nTransformSlavePoints = + + const labelList& transformSlavePoints = ( - transformedSlaves.size() == 0 - ? 0 - : transformedSlaves[i].size() + transformedSlaves.empty() + ? Foam::emptyLabelList + : transformedSlaves[i] ); - if (slavePoints.size()+nTransformSlavePoints > 0) + + // Combine master with untransformed slave data + for (const label pointi : slavePoints) { - // Combine master with untransformed slave data - forAll(slavePoints, j) - { - cop(elem, elems[slavePoints[j]]); - } + cop(elem, elems[pointi]); + } - // Combine master with transformed slave data - if (nTransformSlavePoints) - { - const labelList& transformSlavePoints = transformedSlaves[i]; - forAll(transformSlavePoints, j) - { - cop(elem, elems[transformSlavePoints[j]]); - } - } + // Combine master with transformed slave data + for (const label pointi : transformSlavePoints) + { + cop(elem, elems[pointi]); + } - // Copy result back to slave slots - forAll(slavePoints, j) - { - elems[slavePoints[j]] = elem; - } - if (nTransformSlavePoints) - { - const labelList& transformSlavePoints = transformedSlaves[i]; - forAll(transformSlavePoints, j) - { - elems[transformSlavePoints[j]] = elem; - } - } + // Copy result back to slave slots + for (const label pointi : slavePoints) + { + elems[pointi] = elem; + } + + for (const label pointi : transformSlavePoints) + { + elems[pointi] = elem; } } diff --git a/src/OpenFOAM/meshes/polyMesh/syncTools/syncTools.H b/src/OpenFOAM/meshes/polyMesh/syncTools/syncTools.H index 1d5f8282ff..3ab9dd3bf3 100644 --- a/src/OpenFOAM/meshes/polyMesh/syncTools/syncTools.H +++ b/src/OpenFOAM/meshes/polyMesh/syncTools/syncTools.H @@ -96,7 +96,7 @@ public: template static void syncPointMap ( - const polyMesh&, + const polyMesh& mesh, Map& pointValues, const CombineOp& cop, const TransformOp& top @@ -106,7 +106,7 @@ public: template static void syncEdgeMap ( - const polyMesh&, + const polyMesh& mesh, EdgeMap& edgeValues, const CombineOp& cop, const TransformOp& top @@ -116,8 +116,8 @@ public: template static void syncPointList ( - const polyMesh&, - List&, + const polyMesh& mesh, + List& pointValues, const CombineOp& cop, const T& nullValue, const TransformOp& top @@ -127,9 +127,9 @@ public: template static void syncPointList ( - const polyMesh&, - const labelList& meshPoints, - List&, + const polyMesh& mesh, + const labelUList& meshPoints, + List& pointValues, const CombineOp& cop, const T& nullValue, const TransformOp& top @@ -139,8 +139,8 @@ public: template static void syncEdgeList ( - const polyMesh&, - List&, + const polyMesh& mesh, + List& edgeValues, const CombineOp& cop, const T& nullValue, const TransformOp& top @@ -150,9 +150,9 @@ public: template static void syncEdgeList ( - const polyMesh&, + const polyMesh& mesh, const labelList& meshEdges, - List&, + List& edgeValues, const CombineOp& cop, const T& nullValue, const TransformOp& top @@ -162,8 +162,8 @@ public: template static void syncBoundaryFaceList ( - const polyMesh&, - UList&, + const polyMesh& mesh, + UList& faceValues, const CombineOp& cop, const TransformOp& top, const bool parRun = Pstream::parRun() @@ -177,7 +177,7 @@ public: static void syncPointList ( const polyMesh& mesh, - List& l, + List& pointValues, const CombineOp& cop, const T& nullValue ) @@ -185,7 +185,7 @@ public: syncPointList ( mesh, - l, + pointValues, cop, nullValue, mapDistribute::transform() @@ -197,7 +197,7 @@ public: static void syncPointPositions ( const polyMesh& mesh, - List& l, + List& positions, const CombineOp& cop, const point& nullValue ) @@ -205,7 +205,7 @@ public: syncPointList ( mesh, - l, + positions, cop, nullValue, mapDistribute::transformPosition() @@ -218,7 +218,7 @@ public: ( const polyMesh& mesh, const labelList& meshPoints, - List& l, + List& pointValues, const CombineOp& cop, const T& nullValue ) @@ -227,7 +227,7 @@ public: ( mesh, meshPoints, - l, + pointValues, cop, nullValue, mapDistribute::transform() @@ -240,7 +240,7 @@ public: ( const polyMesh& mesh, const labelList& meshPoints, - List& l, + List& positions, const CombineOp& cop, const point& nullValue ) @@ -249,7 +249,7 @@ public: ( mesh, meshPoints, - l, + positions, cop, nullValue, mapDistribute::transformPosition() @@ -264,7 +264,7 @@ public: static void syncEdgeList ( const polyMesh& mesh, - List& l, + List& edgeValues, const CombineOp& cop, const T& nullValue ) @@ -272,7 +272,7 @@ public: syncEdgeList ( mesh, - l, + edgeValues, cop, nullValue, mapDistribute::transform() @@ -284,7 +284,7 @@ public: static void syncEdgePositions ( const polyMesh& mesh, - List& l, + List& positions, const CombineOp& cop, const point& nullValue ) @@ -292,7 +292,7 @@ public: syncEdgeList ( mesh, - l, + positions, cop, nullValue, mapDistribute::transformPosition() @@ -305,7 +305,7 @@ public: ( const polyMesh& mesh, const labelList& meshEdges, - List& l, + List& edgeValues, const CombineOp& cop, const T& nullValue ) @@ -314,7 +314,7 @@ public: ( mesh, meshEdges, - l, + edgeValues, cop, nullValue, mapDistribute::transform() @@ -327,7 +327,7 @@ public: ( const polyMesh& mesh, const labelList& meshEdges, - List& l, + List& positions, const CombineOp& cop, const point& nullValue ) @@ -336,7 +336,7 @@ public: ( mesh, meshEdges, - l, + positions, cop, nullValue, mapDistribute::transformPosition() @@ -352,11 +352,17 @@ public: static void syncBoundaryFaceList ( const polyMesh& mesh, - UList& l, + UList& faceValues, const CombineOp& cop ) { - syncBoundaryFaceList(mesh, l, cop, mapDistribute::transform()); + syncBoundaryFaceList + ( + mesh, + faceValues, + cop, + mapDistribute::transform() + ); } //- Synchronize locations on boundary faces only. @@ -364,14 +370,14 @@ public: static void syncBoundaryFacePositions ( const polyMesh& mesh, - UList& l, + UList& positions, const CombineOp& cop ) { syncBoundaryFaceList ( mesh, - l, + positions, cop, mapDistribute::transformPosition() ); @@ -382,13 +388,13 @@ public: static void syncFaceList ( const polyMesh& mesh, - UList& l, + UList& faceValues, const CombineOp& cop ) { SubList bndValues ( - l, + faceValues, mesh.nFaces()-mesh.nInternalFaces(), mesh.nInternalFaces() ); @@ -407,13 +413,13 @@ public: static void syncFacePositions ( const polyMesh& mesh, - UList& l, + UList& positions, const CombineOp& cop ) { SubList bndValues ( - l, + positions, mesh.nFaces()-mesh.nInternalFaces(), mesh.nInternalFaces() ); @@ -431,29 +437,29 @@ public: static void swapBoundaryFaceList ( const polyMesh& mesh, - UList& l + UList& faceValues ) { syncBoundaryFaceList ( mesh, - l, + faceValues, eqOp(), mapDistribute::transform() ); } - //- Swap coupled positions. + //- Swap coupled positions. static void swapBoundaryFacePositions ( const polyMesh& mesh, - UList& l + UList& positions ) { syncBoundaryFaceList ( mesh, - l, + positions, eqOp(), mapDistribute::transformPosition() ); @@ -464,12 +470,12 @@ public: static void swapFaceList ( const polyMesh& mesh, - UList& l + UList& faceValues ) { SubList bndValues ( - l, + faceValues, mesh.nFaces()-mesh.nInternalFaces(), mesh.nInternalFaces() ); @@ -506,11 +512,17 @@ public: static void syncPointMap ( const polyMesh& mesh, - Map& l, + Map& pointValues, const CombineOp& cop ) { - syncPointMap(mesh, l, cop, mapDistribute::transform()); + syncPointMap + ( + mesh, + pointValues, + cop, + mapDistribute::transform() + ); } //- Synchronize locations on selected points. @@ -518,11 +530,17 @@ public: static void syncPointPositions ( const polyMesh& mesh, - Map& l, + Map& positions, const CombineOp& cop ) { - syncPointMap(mesh, l, cop, mapDistribute::transformPosition()); + syncPointMap + ( + mesh, + positions, + cop, + mapDistribute::transformPosition() + ); } //- Synchronize values on selected edges. Edges are represented @@ -532,11 +550,17 @@ public: static void syncEdgeMap ( const polyMesh& mesh, - EdgeMap& l, + EdgeMap& edgeValues, const CombineOp& cop ) { - syncEdgeMap(mesh, l, cop, mapDistribute::transform()); + syncEdgeMap + ( + mesh, + edgeValues, + cop, + mapDistribute::transform() + ); } //- Synchronize locations on selected edges. @@ -544,11 +568,17 @@ public: static void syncEdgePositions ( const polyMesh& mesh, - EdgeMap& l, + EdgeMap& edgePositions, const CombineOp& cop ) { - syncEdgeMap(mesh, l, cop, mapDistribute::transformPosition()); + syncEdgeMap + ( + mesh, + edgePositions, + cop, + mapDistribute::transformPosition() + ); } // PackedList versions diff --git a/src/OpenFOAM/meshes/polyMesh/syncTools/syncToolsTemplates.C b/src/OpenFOAM/meshes/polyMesh/syncTools/syncToolsTemplates.C index 36386b3cda..e96be7fe10 100644 --- a/src/OpenFOAM/meshes/polyMesh/syncTools/syncToolsTemplates.C +++ b/src/OpenFOAM/meshes/polyMesh/syncTools/syncToolsTemplates.C @@ -766,7 +766,7 @@ template void Foam::syncTools::syncPointList ( const polyMesh& mesh, - const labelList& meshPoints, + const labelUList& meshPoints, List& pointValues, const CombineOp& cop, const T& nullValue, @@ -1061,38 +1061,35 @@ void Foam::syncTools::syncFaceList for (const polyPatch& pp : patches) { - if (isA(pp) && pp.size() > 0) + if (isA(pp) && pp.size()) { const processorPolyPatch& procPatch = refCast(pp); - List patchInfo(procPatch.size()); - forAll(procPatch, i) - { - patchInfo[i] = faceValues[procPatch.start()+i]; - } - + // Send slice of values on the patch UOPstream toNbr(procPatch.neighbProcNo(), pBufs); - toNbr << patchInfo; + toNbr + << PackedList(faceValues, procPatch.range()); } } - pBufs.finishedSends(); + // Receive and combine. for (const polyPatch& pp : patches) { - if (isA(pp) && pp.size() > 0) + if (isA(pp) && pp.size()) { const processorPolyPatch& procPatch = refCast(pp); - List patchInfo(procPatch.size()); + // Recv slice of values on the patch + PackedList recvInfo(procPatch.size()); { UIPstream fromNbr(procPatch.neighbProcNo(), pBufs); - fromNbr >> patchInfo; + fromNbr >> recvInfo; } // Combine (bitwise) @@ -1100,16 +1097,17 @@ void Foam::syncTools::syncFaceList { const label meshFacei = procPatch.start()+i; - unsigned int patchVal = patchInfo[i]; + unsigned int recvVal = recvInfo[i]; unsigned int faceVal = faceValues[meshFacei]; - cop(faceVal, patchVal); - faceValues[meshFacei] = faceVal; + cop(faceVal, recvVal); + faceValues.set(meshFacei, faceVal); } } } } + // Do the cyclics. for (const polyPatch& pp : patches) {