diff --git a/applications/test/syncTools/Test-syncTools.C b/applications/test/syncTools/Test-syncTools.C index bdfd2007e8..bed6124ca8 100644 --- a/applications/test/syncTools/Test-syncTools.C +++ b/applications/test/syncTools/Test-syncTools.C @@ -3,9 +3,10 @@ \\ / F ield | OpenFOAM: The Open Source CFD Toolbox \\ / O peration | \\ / A nd | www.openfoam.com - \\/ M anipulation | Copyright (C) 2021 OpenCFD Ltd. + \\/ M anipulation | ------------------------------------------------------------------------------- Copyright (C) 2011-2016 OpenFOAM Foundation + Copyright (C) 2021 OpenCFD Ltd. ------------------------------------------------------------------------------- License This file is part of OpenFOAM. @@ -38,6 +39,7 @@ Description #include "Random.H" #include "PackedList.H" #include "flipOp.H" +#include "pointList.H" using namespace Foam; @@ -224,8 +226,6 @@ void testSparseData(const polyMesh& mesh, Random& rndGen) fullData[meshPointi] = pt; } - //Pout<< "sparseData:" << sparseData << endl; - syncTools::syncPointMap ( mesh, @@ -310,8 +310,6 @@ void testSparseData(const polyMesh& mesh, Random& rndGen) fullData[meshEdgeI] = pt; } - //Pout<< "sparseData:" << sparseData << endl; - syncTools::syncEdgeMap ( mesh, @@ -585,7 +583,7 @@ public: }; void testEdgeFlip2(const polyMesh& mesh, Random& rndGen) { - Pout<< nl << "Testing edge-wise (oriented) data synchronisation." << endl; + Info<< nl << "Testing edge-wise (oriented) data synchronisation." << endl; const edgeList& edges = mesh.edges(); const pointField& points = mesh.points(); @@ -766,6 +764,64 @@ void testEdgeFlip(const polyMesh& mesh, Random& rndGen) } +class pointListOps +{ +public: + + void operator()(pointList& lhs, const pointList& rhs) const + { + forAll(lhs, i) + { + point& l = lhs[i]; + const point& r = rhs[i]; + maxMagSqrEqOp()(l, r); + } + } + + void operator() + ( + const vectorTensorTransform& vt, + const bool forward, + List& fld + ) const + { + if (forward) + { + for (auto& elems : fld) + { + for (auto& elem : elems) + { + elem = vt.transformPosition(elem); + } + } + } + else + { + for (auto& elems : fld) + { + for (auto& elem : elems) + { + elem = vt.invTransformPosition(elem); + } + } + } + } + + //- Transform patch-based field + void operator()(const coupledPolyPatch& cpp, List& fld) const + { + forAll(fld, facei) + { + pointList& pts = fld[facei]; + for (auto& pt : pts) + { + cpp.transformPosition(pt, facei); + } + } + } +}; + + void testFaceSync(const polyMesh& mesh, Random& rndGen) { Info<< nl << "Testing face-wise data synchronisation." << endl; @@ -795,6 +851,52 @@ void testFaceSync(const polyMesh& mesh, Random& rndGen) } } + + // Test non-contiguous data (uses streaming) + + { + List syncedFc(mesh.nFaces()); + + const pointField& fcs = mesh.faceCentres(); + forAll(fcs, facei) + { + const point& fc = fcs[facei]; + syncedFc[facei].setSize(2, fc); + } + + SubList bndValues + ( + syncedFc, + mesh.nBoundaryFaces(), + mesh.nInternalFaces() + ); + syncTools::syncBoundaryFaceList + ( + mesh, + bndValues, + pointListOps(), //does maxMagSqrEqOp() + pointListOps() //transforms all + ); + + forAll(syncedFc, facei) + { + const point& fc = fcs[facei]; + const pointList& fld = syncedFc[facei]; + forAll(fld, i) + { + if (mag(fld[i] - fc) > SMALL) + { + FatalErrorInFunction + << "Face " << facei + << " original centre " << fc + << " synced centre " << fld[i] + << exit(FatalError); + } + } + } + } + + // Test masterFaces { diff --git a/src/OpenFOAM/meshes/polyMesh/syncTools/syncToolsTemplates.C b/src/OpenFOAM/meshes/polyMesh/syncTools/syncToolsTemplates.C index 54be93cee7..32b0afc76a 100644 --- a/src/OpenFOAM/meshes/polyMesh/syncTools/syncToolsTemplates.C +++ b/src/OpenFOAM/meshes/polyMesh/syncTools/syncToolsTemplates.C @@ -1022,50 +1022,147 @@ void Foam::syncTools::syncBoundaryFaceList if (parRun) { - PstreamBuffers pBufs(Pstream::commsTypes::nonBlocking); - - // Send - - for (const polyPatch& pp : patches) + if + ( + is_contiguous::value + && Pstream::defaultCommsType == Pstream::commsTypes::nonBlocking + ) { - if (isA(pp) && pp.size() > 0) + const label startRequest = UPstream::nRequests(); + + // Receive buffer + List receivedValues(mesh.nBoundaryFaces()); + + // Set up reads + for (const polyPatch& pp : patches) { - const processorPolyPatch& procPatch = - refCast(pp); + if (isA(pp) && pp.size() > 0) + { + const processorPolyPatch& procPatch = + refCast(pp); - const label patchStart = procPatch.start()-boundaryOffset; + SubList fld + ( + receivedValues, + procPatch.size(), + procPatch.offset() + ); - // Send slice of values on the patch - UOPstream toNbr(procPatch.neighbProcNo(), pBufs); - toNbr<< SubList(faceValues, procPatch.size(), patchStart); + IPstream::read + ( + Pstream::commsTypes::nonBlocking, + procPatch.neighbProcNo(), + reinterpret_cast(fld.data()), + fld.byteSize() + ); + } + } + + // Set up writes + for (const polyPatch& pp : patches) + { + if (isA(pp) && pp.size() > 0) + { + const processorPolyPatch& procPatch = + refCast(pp); + + const SubList fld + ( + faceValues, + procPatch.size(), + procPatch.offset() + ); + + OPstream::write + ( + Pstream::commsTypes::nonBlocking, + procPatch.neighbProcNo(), + reinterpret_cast(fld.cdata()), + fld.byteSize() + ); + } + } + + // Wait for all comms to finish + Pstream::waitRequests(startRequest); + + // Combine with existing data + for (const polyPatch& pp : patches) + { + if (isA(pp) && pp.size() > 0) + { + const processorPolyPatch& procPatch = + refCast(pp); + SubList recvFld + ( + receivedValues, + procPatch.size(), + procPatch.offset() + ); + const List& fakeList = recvFld; + top(procPatch, const_cast&>(fakeList)); + + SubList patchValues + ( + faceValues, + procPatch.size(), + procPatch.offset() + ); + forAll(patchValues, i) + { + cop(patchValues[i], recvFld[i]); + } + } } } - - - pBufs.finishedSends(); - - - // Receive and combine. - - for (const polyPatch& pp : patches) + else { - if (isA(pp) && pp.size() > 0) + PstreamBuffers pBufs(Pstream::commsTypes::nonBlocking); + + // Send + + for (const polyPatch& pp : patches) { - const processorPolyPatch& procPatch = - refCast(pp); - - Field nbrVals(procPatch.size()); - - UIPstream fromNbr(procPatch.neighbProcNo(), pBufs); - fromNbr >> nbrVals; - - top(procPatch, nbrVals); - - label bFacei = procPatch.start()-boundaryOffset; - - for (T& nbrVal : nbrVals) + if (isA(pp) && pp.size() > 0) { - cop(faceValues[bFacei++], nbrVal); + const processorPolyPatch& procPatch = + refCast(pp); + + const label patchStart = procPatch.start()-boundaryOffset; + + // Send slice of values on the patch + UOPstream toNbr(procPatch.neighbProcNo(), pBufs); + toNbr << + SubList(faceValues, procPatch.size(), patchStart); + } + } + + + pBufs.finishedSends(); + + + // Receive and combine. + + for (const polyPatch& pp : patches) + { + if (isA(pp) && pp.size() > 0) + { + const processorPolyPatch& procPatch = + refCast(pp); + + List nbrVals(procPatch.size()); + + UIPstream fromNbr(procPatch.neighbProcNo(), pBufs); + fromNbr >> nbrVals; + + top(procPatch, nbrVals); + + label bFacei = procPatch.start()-boundaryOffset; + + for (const T& nbrVal : nbrVals) + { + cop(faceValues[bFacei++], nbrVal); + } } } } @@ -1089,10 +1186,10 @@ void Foam::syncTools::syncBoundaryFaceList const label nbrStart = nbrPatch.start()-boundaryOffset; // Transform (copy of) data on both sides - Field ownVals(SubList(faceValues, patchSize, ownStart)); + List ownVals(SubList(faceValues, patchSize, ownStart)); top(nbrPatch, ownVals); - Field nbrVals(SubList(faceValues, patchSize, nbrStart)); + List nbrVals(SubList(faceValues, patchSize, nbrStart)); top(cycPatch, nbrVals); label bFacei = ownStart; @@ -1145,10 +1242,39 @@ void Foam::syncTools::syncFaceList if (parRun) { - PstreamBuffers pBufs(Pstream::commsTypes::nonBlocking); + const label startRequest = UPstream::nRequests(); - // Send + // Receive buffers + PtrList> recvInfos(patches.size()); + // Set up reads + for (const polyPatch& pp : patches) + { + if (isA(pp) && pp.size()) + { + const processorPolyPatch& procPatch = + refCast(pp); + + const label patchSize = procPatch.size(); + const label patchi = procPatch.index(); + + recvInfos.set(patchi, new PackedList(patchSize)); + PackedList& recvInfo = recvInfos[patchi]; + + IPstream::read + ( + Pstream::commsTypes::nonBlocking, + procPatch.neighbProcNo(), + reinterpret_cast(recvInfo.storage().data()), + recvInfo.byteSize() + ); + } + } + + // Send buffers + PtrList> sendInfos(patches.size()); + + // Set up writes for (const polyPatch& pp : patches) { if (isA(pp) && pp.size()) @@ -1161,18 +1287,29 @@ void Foam::syncTools::syncFaceList procPatch.start()-boundaryOffset, procPatch.size() ); + const label patchi = procPatch.index(); - // Send slice of values on the patch - UOPstream toNbr(procPatch.neighbProcNo(), pBufs); - toNbr<< PackedList(faceValues, range); + sendInfos.set + ( + patchi, + new PackedList(faceValues, range) + ); + PackedList& sendInfo = sendInfos[patchi]; + + OPstream::write + ( + Pstream::commsTypes::nonBlocking, + procPatch.neighbProcNo(), + reinterpret_cast(sendInfo.storage().cdata()), + sendInfo.byteSize() + ); } } - pBufs.finishedSends(); - - - // Receive and combine. + // Wait for all comms to finish + Pstream::waitRequests(startRequest); + // Combine with existing data for (const polyPatch& pp : patches) { if (isA(pp) && pp.size()) @@ -1181,13 +1318,8 @@ void Foam::syncTools::syncFaceList refCast(pp); const label patchSize = procPatch.size(); - - // Recv slice of values on the patch - PackedList recvInfo(patchSize); - { - UIPstream fromNbr(procPatch.neighbProcNo(), pBufs); - fromNbr >> recvInfo; - } + const label patchi = procPatch.index(); + const PackedList& recvInfo = recvInfos[patchi]; // Combine (bitwise) label bFacei = procPatch.start()-boundaryOffset; @@ -1203,6 +1335,66 @@ void Foam::syncTools::syncFaceList } } } + + + //PstreamBuffers pBufs(Pstream::commsTypes::nonBlocking); + // + //// Send + // + //for (const polyPatch& pp : patches) + //{ + // if (isA(pp) && pp.size()) + // { + // const processorPolyPatch& procPatch = + // refCast(pp); + // + // const labelRange range + // ( + // procPatch.start()-boundaryOffset, + // procPatch.size() + // ); + // + // // Send slice of values on the patch + // UOPstream toNbr(procPatch.neighbProcNo(), pBufs); + // toNbr<< PackedList(faceValues, range); + // } + //} + // + //pBufs.finishedSends(); + // + // + //// Receive and combine. + // + //for (const polyPatch& pp : patches) + //{ + // if (isA(pp) && pp.size()) + // { + // const processorPolyPatch& procPatch = + // refCast(pp); + // + // const label patchSize = procPatch.size(); + // + // // Recv slice of values on the patch + // PackedList recvInfo(patchSize); + // { + // UIPstream fromNbr(procPatch.neighbProcNo(), pBufs); + // fromNbr >> recvInfo; + // } + // + // // Combine (bitwise) + // label bFacei = procPatch.start()-boundaryOffset; + // for (label i = 0; i < patchSize; ++i) + // { + // unsigned int recvVal = recvInfo[i]; + // unsigned int faceVal = faceValues[bFacei]; + // + // cop(faceVal, recvVal); + // faceValues.set(bFacei, faceVal); + // + // ++bFacei; + // } + // } + //} }