diff --git a/CHANGES-splitCyclic b/CHANGES-splitCyclic new file mode 100644 index 0000000000..506d6a76ca --- /dev/null +++ b/CHANGES-splitCyclic @@ -0,0 +1,98 @@ +Short overview of the changes to have cyclics split into two halves. + +Cyclics +------- +The two cyclic halves are now split like processor patches. There should be no +difference in running. Just some of the reasons for switching over from the +old 2-halves in one patch: +- face ordering after topological changes becomes much easier since we +now preserve what half the face comes from. +- cyclic handling becomes more consistent with processor handling and can +quite often be handled in the same condition. +- decomposed cyclics can now be handled properly. It just needs to preserve +the cyclic patch it originates from. +- We can now construct a table of global transformations and handle +points/edges/cells with transformations. +The disadvantages: +- a patch-wise loop now might need to store data to go to the neighbour half +since it is no longer handled in a single patch. +- decomposed cyclics now require overlapping communications so will +only work in non-blocking mode. Also the underlying message passing library +will require overlapping communications with message tags. +- it is quite a code-change and there might be some oversights. + + + +blockMesh +--------- +- blockMeshDict now allows patch definition using the construct-from-dictionary +constructor. This helps defining patches that require additional input e.g. +directMapped and now cyclic: + +boundary +( + sides2_half0 + { + type cyclic; + neighbourPatch sides2_half1; + faces ((2 4 5 3)); + } + +The syntax is - like the polyMesh/boundary file - a list of dictionaries with +one additional entry 'faces' for the block faces. Above shows the new +required entry 'neighbourPatch' for cyclic. + +blockMesh still reads the old format. For a cyclic it will automatically +introduce two patches for the halves, with names xxx_half0 and xxx_half1. + + +decomposePar +------------ +Decomposes cyclics into processorCyclic: + + procBoundary0to1throughsides1_half0 + { + type processorCyclic; + nFaces 1000; + startFace 91350; + myProcNo 0; + neighbProcNo 1; + referPatch sides1_half0; + } + +They have an additional 'referPatch' entry which gives the (cyclic) patch +to use for any transformation. + + +Details +------- +- the cyclic patch dictionary has an entry neighbourPatch. The +patch has new member functions: + + //- Get neighbouring patchID + label neighbPatchID() const + + //- Get neighbouring patch + const cyclicPolyPatch& neighbPatch() + + //- Am I the owner half + bool owner() + +The cyclic still has forward() and reverse() transformations (with +the reverse() equal to the neighbPatch().forward()). + +There is no transformLocalFace anymore - the ordering is the same for +both halves. + + +- 'pure' processor patches now are always coincident - they (should) have no +transformation. As said above cyclics are decomposed into a derived +type 'processorCyclic'. + + +- processor patches use overlapping communication using a different message +tag. This maps straight through into the MPI message tag. + + + + diff --git a/TODO b/TODO index 4931d92f1e..576750da0d 100644 --- a/TODO +++ b/TODO @@ -76,5 +76,5 @@ Tested on movingCone-with-cyclics - isoSurface.C - referredCellList.C - work out scheduled communication? - +- LUscalarMatrix::convert still expects interfaces to be cyclic diff --git a/src/OpenFOAM/matrices/lduMatrix/lduAddressing/lduInterface/lduInterface.H b/src/OpenFOAM/matrices/lduMatrix/lduAddressing/lduInterface/lduInterface.H index 745733284e..60ca7372d0 100644 --- a/src/OpenFOAM/matrices/lduMatrix/lduAddressing/lduInterface/lduInterface.H +++ b/src/OpenFOAM/matrices/lduMatrix/lduAddressing/lduInterface/lduInterface.H @@ -96,21 +96,6 @@ public: const unallocLabelList& internalData ) const = 0; -// //- Initialise interface data transfer -// virtual void initTransfer -// ( -// const Pstream::commsTypes commsType, -// const unallocLabelList& interfaceData -// ) const -// {} -// -// //- Transfer and return neighbour field -// virtual tmp transfer -// ( -// const Pstream::commsTypes commsType, -// const unallocLabelList& interfaceData -// ) const = 0; - //- Initialise transfer of internal field adjacent to the interface virtual void initInternalFieldTransfer ( diff --git a/src/OpenFOAM/matrices/lduMatrix/lduAddressing/lduInterface/processorLduInterfaceTemplates.C b/src/OpenFOAM/matrices/lduMatrix/lduAddressing/lduInterface/processorLduInterfaceTemplates.C index cfa1716f68..450c30f884 100644 --- a/src/OpenFOAM/matrices/lduMatrix/lduAddressing/lduInterface/processorLduInterfaceTemplates.C +++ b/src/OpenFOAM/matrices/lduMatrix/lduAddressing/lduInterface/processorLduInterfaceTemplates.C @@ -29,49 +29,6 @@ License // * * * * * * * * * * * * * * * Member Functions * * * * * * * * * * * * * // -// template -// Foam::List& Foam::processorLduInterface::setSendBuf(const label nElems) -// const -// { -// if (!contiguous()) -// { -// FatalErrorIn("processorLduInterface::setSendBuf(const label) const") -// << "Cannot return the binary size of a list of " -// "non-primitive elements" -// << abort(FatalError); -// } -// -// label nBytes = nElems*sizeof(Type); -// sendBuf_.setSize(nBytes); -// -// return reinterpret_cast&>(sendBuf_); -// } -// -// -// template -// Foam::List& Foam::processorLduInterface::setReceiveBuf -// ( -// const label nElems -// ) const -// { -// if (!contiguous()) -// { -// FatalErrorIn("processorLduInterface::setReceiveBuf(const label) const") -// << "Cannot return the binary size of a list of " -// "non-primitive elements" -// << abort(FatalError); -// } -// -// label nBytes = nElems*sizeof(Type); -// -// //receiveBuf_.setSize(nBytes, '\0'); // necessary because of expanding -// // compression? -// receiveBuf_.setSize(nBytes); -// -// return reinterpret_cast&>(receiveBuf_); -// } - - template void Foam::processorLduInterface::send ( @@ -92,7 +49,6 @@ void Foam::processorLduInterface::send } else if (commsType == Pstream::nonBlocking) { - //setReceiveBuf(f.size()); resizeBuf(receiveBuf_, f.size()*sizeof(Type)); IPstream::read @@ -104,7 +60,6 @@ void Foam::processorLduInterface::send tag() ); - //setSendBuf(f.size()); resizeBuf(sendBuf_, f.byteSize()); memcpy(sendBuf_.begin(), f.begin(), f.byteSize()); @@ -187,7 +142,6 @@ void Foam::processorLduInterface::compressedSend const scalar *sArray = reinterpret_cast(f.begin()); const scalar *slast = &sArray[nm1]; - //setSendBuf(nFloats); resizeBuf(sendBuf_, nBytes); float *fArray = reinterpret_cast(sendBuf_.begin()); @@ -211,7 +165,6 @@ void Foam::processorLduInterface::compressedSend } else if (commsType == Pstream::nonBlocking) { - //setReceiveBuf(nFloats); resizeBuf(receiveBuf_, nBytes); IPstream::read @@ -262,7 +215,6 @@ void Foam::processorLduInterface::compressedReceive if (commsType == Pstream::blocking || commsType == Pstream::scheduled) { - //setReceiveBuf(nFloats); resizeBuf(receiveBuf_, nBytes); IPstream::read @@ -311,160 +263,4 @@ Foam::tmp > Foam::processorLduInterface::compressedReceive } -// template -// void Foam::processorLduInterface::compressedBufferSend -// ( -// const Pstream::commsTypes commsType -// ) const -// { -// // Optionally inline compress sendBuf -// if -// ( -// sizeof(scalar) > sizeof(float) -// && sendBuf_.size() -// && Pstream::floatTransfer -// ) -// { -// const List& f = reinterpret_cast&>(sendBuf_); -// label fSize = f.size()/sizeof(Type); -// -// // Inplace compress -// static const label nCmpts = sizeof(Type)/sizeof(scalar); -// label nm1 = (fSize - 1)*nCmpts; -// label nlast = sizeof(Type)/sizeof(float); -// label nFloats = nm1 + nlast; -// -// const scalar *sArray = reinterpret_cast(f.begin()); -// const scalar *slast = &sArray[nm1]; -// float *fArray = reinterpret_cast(sendBuf_.begin()); -// -// for (register label i=0; i(fArray[nm1]) = f[fSize - 1]; -// -// // Trim -// setSendBuf(nFloats); -// } -// -// // Send sendBuf -// if (commsType == Pstream::blocking || commsType == Pstream::scheduled) -// { -// OPstream::write -// ( -// commsType, -// neighbProcNo(), -// sendBuf_.begin(), -// sendBuf_.size(), -// tag() -// ); -// } -// else if (commsType == Pstream::nonBlocking) -// { -// setReceiveBuf(sendBuf_.size()); -// -// IPstream::read -// ( -// commsType, -// neighbProcNo(), -// receiveBuf_.begin(), -// receiveBuf_.size(), -// tag() -// ); -// -// OPstream::write -// ( -// commsType, -// neighbProcNo(), -// sendBuf_.begin(), -// sendBuf_.size(), -// tag() -// ); -// } -// else -// { -// FatalErrorIn("processorLduInterface::compressedBufferSend") -// << "Unsupported communications type " << commsType -// << exit(FatalError); -// } -// } -// -// -// template -// const Foam::List& Foam::processorLduInterface::compressedBufferReceive -// ( -// const Pstream::commsTypes commsType, -// const label size -// ) const -// { -// if (sizeof(scalar) > sizeof(float) && size && Pstream::floatTransfer) -// { -// static const label nCmpts = sizeof(Type)/sizeof(scalar); -// label nm1 = (size - 1)*nCmpts; -// label nlast = sizeof(Type)/sizeof(float); -// label nFloats = nm1 + nlast; -// -// if (commsType == Pstream::blocking || commsType == Pstream::scheduled) -// { -// setReceiveBuf(nFloats); -// -// IPstream::read -// ( -// commsType, -// neighbProcNo(), -// receiveBuf_.begin(), -// receiveBuf_.size(), -// tag() -// ); -// } -// else if (commsType != Pstream::nonBlocking) -// { -// FatalErrorIn("processorLduInterface::compressedBufferReceive") -// << "Unsupported communications type " << commsType -// << exit(FatalError); -// } -// -// // Inline expand -// List& f = setReceiveBuf(size); -// label fSize = f.size()/sizeof(Type); -// -// const float *fArray = -// reinterpret_cast(receiveBuf_.begin()); -// f[fSize - 1] = reinterpret_cast(fArray[nm1]); -// scalar *sArray = reinterpret_cast(f.begin()); -// const scalar *slast = &sArray[nm1]; -// -// for (register label i=0; i(size); -// -// IPstream::read -// ( -// commsType, -// neighbProcNo(), -// receiveBuf_.begin(), -// receiveBuf_.size(), -// tag() -// ); -// } -// else if (commsType != Pstream::nonBlocking) -// { -// FatalErrorIn("processorLduInterface::compressedBufferReceive") -// << "Unsupported communications type " << commsType -// << exit(FatalError); -// } -// } -// return reinterpret_cast&>(receiveBuf_); -// } - - // ************************************************************************* // diff --git a/src/OpenFOAM/matrices/lduMatrix/solvers/GAMG/interfaces/GAMGInterface/GAMGInterface.H b/src/OpenFOAM/matrices/lduMatrix/solvers/GAMG/interfaces/GAMGInterface/GAMGInterface.H index 39351c2ca2..baad87d6d8 100644 --- a/src/OpenFOAM/matrices/lduMatrix/solvers/GAMG/interfaces/GAMGInterface/GAMGInterface.H +++ b/src/OpenFOAM/matrices/lduMatrix/solvers/GAMG/interfaces/GAMGInterface/GAMGInterface.H @@ -36,7 +36,6 @@ SourceFiles #ifndef GAMGInterface_H #define GAMGInterface_H -//#include "lduInterface.H" #include "autoPtr.H" #include "lduInterfacePtrsList.H" diff --git a/src/OpenFOAM/matrices/lduMatrix/solvers/GAMG/interfaces/cyclicGAMGInterface/cyclicGAMGInterface.C b/src/OpenFOAM/matrices/lduMatrix/solvers/GAMG/interfaces/cyclicGAMGInterface/cyclicGAMGInterface.C index 8148ffc063..2aac532ce1 100644 --- a/src/OpenFOAM/matrices/lduMatrix/solvers/GAMG/interfaces/cyclicGAMGInterface/cyclicGAMGInterface.C +++ b/src/OpenFOAM/matrices/lduMatrix/solvers/GAMG/interfaces/cyclicGAMGInterface/cyclicGAMGInterface.C @@ -245,30 +245,6 @@ Foam::cyclicGAMGInterface::~cyclicGAMGInterface() // * * * * * * * * * * * * * * * Member Functions * * * * * * * * * * * * * // -//Foam::tmp Foam::cyclicGAMGInterface::transfer -//( -// const Pstream::commsTypes, -// const unallocLabelList& interfaceData -//) const -//{ -// notImplemented("cyclicGAMGInterface::transfer(..)"); -// -////XXXXX to be done -// tmp tpnf(new labelField(size())); -// labelField& pnf = tpnf(); -// -// label sizeby2 = size()/2; -// -// for (label facei=0; facei Foam::cyclicGAMGInterface::internalFieldTransfer ( const Pstream::commsTypes, diff --git a/src/OpenFOAM/matrices/lduMatrix/solvers/GAMG/interfaces/cyclicGAMGInterface/cyclicGAMGInterface.H b/src/OpenFOAM/matrices/lduMatrix/solvers/GAMG/interfaces/cyclicGAMGInterface/cyclicGAMGInterface.H index 36ba2974b1..c24d5fdca7 100644 --- a/src/OpenFOAM/matrices/lduMatrix/solvers/GAMG/interfaces/cyclicGAMGInterface/cyclicGAMGInterface.H +++ b/src/OpenFOAM/matrices/lduMatrix/solvers/GAMG/interfaces/cyclicGAMGInterface/cyclicGAMGInterface.H @@ -96,13 +96,6 @@ public: // Interface transfer functions -// //- Transfer and return neighbour field -// virtual tmp transfer -// ( -// const Pstream::commsTypes commsType, -// const unallocLabelList& interfaceData -// ) const; - //- Transfer and return internal field adjacent to the interface virtual tmp internalFieldTransfer ( @@ -113,12 +106,6 @@ public: //- Cyclic interface functions -// //- Return my index -// virtual label patchID() const -// { -// return fineCyclicInterface_.patchID(); -// } - //- Return neigbour processor number virtual label neighbPatchID() const { diff --git a/src/OpenFOAM/matrices/lduMatrix/solvers/GAMG/interfaces/processorCyclicGAMGInterface/processorCyclicGAMGInterface.C b/src/OpenFOAM/matrices/lduMatrix/solvers/GAMG/interfaces/processorCyclicGAMGInterface/processorCyclicGAMGInterface.C index df050adebc..0455998058 100644 --- a/src/OpenFOAM/matrices/lduMatrix/solvers/GAMG/interfaces/processorCyclicGAMGInterface/processorCyclicGAMGInterface.C +++ b/src/OpenFOAM/matrices/lduMatrix/solvers/GAMG/interfaces/processorCyclicGAMGInterface/processorCyclicGAMGInterface.C @@ -70,46 +70,4 @@ Foam::processorCyclicGAMGInterface::~processorCyclicGAMGInterface() {} -// * * * * * * * * * * * * * * * Member Functions * * * * * * * * * * * * * // - -// void Foam::processorCyclicGAMGInterface::initTransfer -// ( -// const Pstream::commsTypes commsType, -// const unallocLabelList& interfaceData -// ) const -// { -// send(commsType, interfaceData); -// } -// -// -// Foam::tmp Foam::processorCyclicGAMGInterface::transfer -// ( -// const Pstream::commsTypes commsType, -// const unallocLabelList& interfaceData -// ) const -// { -// return receive