diff --git a/applications/test/laplacianFoam-communicators/laplacianFoam.C b/applications/test/laplacianFoam-communicators/laplacianFoam.C index 8d10b56cf9..3f7bf4cf6d 100644 --- a/applications/test/laplacianFoam-communicators/laplacianFoam.C +++ b/applications/test/laplacianFoam-communicators/laplacianFoam.C @@ -31,55 +31,860 @@ Description #include "fvCFD.H" #include "simpleControl.H" +#include "globalIndex.H" +#include "lduPrimitiveMesh.H" +#include "processorGAMGInterface.H" // * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * // -// Determine parent rank -label processorToRank +void checkUpperTriangular ( - const label comm, - const label procI + const label size, + const labelUList& l, + const labelUList& u ) { - const List& parentRanks = UPstream::procID(comm); - label parentComm = UPstream::parent(comm); - - if (parentComm == -1) + forAll(l, faceI) { - return findIndex(parentRanks, procI); + if (u[faceI] < l[faceI]) + { + FatalErrorIn + ( + "checkUpperTriangular" + "(const label, const labelUList&, const labelUList&)" + ) << "Reversed face. Problem at face " << faceI + << " l:" << l[faceI] << " u:" << u[faceI] << abort(FatalError); + } + if (l[faceI] < 0 || u[faceI] < 0 || u[faceI] >= size) + { + FatalErrorIn + ( + "checkUpperTriangular" + "(const label, const labelUList&, const labelUList&)" + ) << "Illegal cell label. Problem at face " << faceI + << " l:" << l[faceI] << " u:" << u[faceI] << abort(FatalError); + } } - else + + for (label faceI=1; faceI < l.size(); faceI++) { - label parentRank = processorToRank(parentComm, procI); - return findIndex(parentRanks, parentRank); + if (l[faceI-1] > l[faceI]) + { + FatalErrorIn + ( + "checkUpperTriangular" + "(const label, const labelUList&, const labelUList&)" + ) << "Lower not in incremental cell order." + << " Problem at face " << faceI + << " l:" << l[faceI] << " u:" << u[faceI] + << " previous l:" << l[faceI-1] << abort(FatalError); + } + else if (l[faceI-1] == l[faceI]) + { + // Same cell. + if (u[faceI-1] > u[faceI]) + { + FatalErrorIn + ( + "checkUpperTriangular" + "(const label, const labelUList&, const labelUList&)" + ) << "Upper not in incremental cell order." + << " Problem at face " << faceI + << " l:" << l[faceI] << " u:" << u[faceI] + << " previous u:" << u[faceI-1] << abort(FatalError); + } + } } } -// Determine rank in new communicator -label rank -( - const label currentComm, - const label currentRank, - const label newComm -) +void print(const string& msg, const lduMesh& mesh) { - // Determine actual processor (walk up the tree to the top) - label procID = currentRank; - label comm = currentComm; + const lduAddressing& addressing = mesh.lduAddr(); + const lduInterfacePtrsList interfaces = mesh.interfaces(); - while (UPstream::parent(comm) != -1) + Pout<< "Mesh:" << msg.c_str() << nl + << " cells:" << addressing.size() << nl + << " faces:" << addressing.lowerAddr().size() << nl + << " patches:" << interfaces.size() << nl; + + + const labelUList& l = addressing.lowerAddr(); + const labelUList& startAddr = addressing.losortStartAddr(); + const labelUList& addr = addressing.losortAddr(); + + forAll(addressing, cellI) { - const List& parentRanks = UPstream::procID(comm); - procID = parentRanks[procID]; - comm = UPstream::parent(comm); + Pout<< " cell:" << cellI << nl; + + label start = startAddr[cellI]; + label end = startAddr[cellI+1]; + + for (label index = start; index < end; index++) + { + Pout<< " nbr:" << l[addr[index]] << nl; + } } - //Pout<< "Walked to top : comm:" << comm << " procID:" << procID - // << endl; + Pout<< "Patches:" << nl; + forAll(interfaces, i) + { + if (interfaces.set(i)) + { + if (isA(interfaces[i])) + { + const processorLduInterface& pldui = + refCast(interfaces[i]); + Pout<< " " << i + << " me:" << pldui.myProcNo() + << " nbr:" << pldui.neighbProcNo() + << " comm:" << pldui.comm() + << " tag:" << pldui.tag() + << nl; + } - // Find out what procID in the newComm is. - return processorToRank(newComm, procID); + { + Pout<< " " << i << " addressing:" << nl; + const labelUList& faceCells = interfaces[i].faceCells(); + forAll(faceCells, i) + { + Pout<< "\t\t" << i << '\t' << faceCells[i] << nl; + } + } + } + } +} + + + +template +lduSchedule nonBlockingSchedule +( + const lduInterfacePtrsList& interfaces +) +{ + lduSchedule schedule(2*interfaces.size()); + label slotI = 0; + + forAll(interfaces, i) + { + if (interfaces.set(i) && !isA(interfaces[i])) + { + schedule[slotI].patch = i; + schedule[slotI].init = true; + slotI++; + schedule[slotI].patch = i; + schedule[slotI].init = false; + slotI++; + } + } + + forAll(interfaces, i) + { + if (interfaces.set(i) && isA(interfaces[i])) + { + schedule[slotI].patch = i; + schedule[slotI].init = true; + slotI++; + } + } + + forAll(interfaces, i) + { + if (interfaces.set(i) && isA(interfaces[i])) + { + schedule[slotI].patch = i; + schedule[slotI].init = false; + slotI++; + } + } + + return schedule; +} + + + +// Say combining procs 1,13,9,4. +// - cells get added in order 1,4,9,13 +// - internal faces get added in order of processor and then order of +// neighbouring processor: +// - internal faces of 1 +// - faces between 1 and 4 (keep orientation) +// - faces between 1 and 9 +// - faces between 1 and 13 +// - interfaces between 1 and other processors +// and then +// - internal faces of 4 +// - faces between 4 and 1 (skip; already added) +// - faces between 4 and 9 +// - faces between 4 and 13 +// - interfaces between 4 and other processors +// etc. +// - this still does not guarantee that upper-triangular order is preserved: +// A cell has two processor faces. These now become internal faces and if +// the cell numbering on the other side is not in the same order as the internal +// faces we have upper-triangular. +autoPtr combineMeshes +( + const label newComm, + const labelList& procIDs, + const PtrList& procMeshes, + + labelList& cellOffsets, // per mesh the starting cell + labelList& faceOffsets, // per mesh the starting face + labelList& interfaceOffsets // per mesh,per interface the starting face +) +{ + // Sanity check. + for (label i = 1; i < procIDs.size(); i++) + { + if (procIDs[i] <= procIDs[i-1]) + { + FatalErrorIn + ( + "combineMeshes(const labelList&, const PtrList&)" + ) << "Processor " << procIDs[i] << " at index " << i + << " should be higher numbered than its predecessor " + << procIDs[i-1] + << exit(FatalError); + } + } + + + label currentComm = procMeshes[0].comm(); + + + // Cells get added in order. + cellOffsets.setSize(procMeshes.size()+1); + cellOffsets[0] = 0; + forAll(procMeshes, procMeshI) + { + const lduMesh& mesh = procMeshes[procMeshI]; + cellOffsets[procMeshI+1] = + cellOffsets[procMeshI] + + mesh.lduAddr().size(); + } + + // Faces initially get added in order, sorted later + labelList internalFaceOffsets(procMeshes.size()+1); + internalFaceOffsets[0] = 0; + forAll(procMeshes, procMeshI) + { + const lduMesh& mesh = procMeshes[procMeshI]; + internalFaceOffsets[procMeshI+1] = + internalFaceOffsets[procMeshI] + + mesh.lduAddr().lowerAddr().size(); + } + + // Count how faces get added. Interfaces inbetween get merged. + + // Merged interfaces: map from two processors back to + // - procMeshes + // - interface in procMesh + EdgeMap mergedMap + ( + 2*procMeshes[0].interfaces().size() + ); + + + // Unmerged interfaces: map from two processors back to + // - procMeshes + // - interface in procMesh + EdgeMap unmergedMap + ( + 2*procMeshes[0].interfaces().size() + ); + // Per interface the size + //DynamicList