diff --git a/src/OpenFOAM/meshes/polyMesh/mapPolyMesh/mapDistribute/mapDistributeTemplates.C b/src/OpenFOAM/meshes/polyMesh/mapPolyMesh/mapDistribute/mapDistributeTemplates.C index 6ad85c7971..86a0a618c1 100644 --- a/src/OpenFOAM/meshes/polyMesh/mapPolyMesh/mapDistribute/mapDistributeTemplates.C +++ b/src/OpenFOAM/meshes/polyMesh/mapPolyMesh/mapDistribute/mapDistributeTemplates.C @@ -84,16 +84,32 @@ void Foam::mapDistribute::distribute // Receive sub field from neighbour for (label domain = 0; domain < Pstream::nProcs(); domain++) { - if - ( - domain != Pstream::myProcNo() - && constructMap[domain].size() > 0 - ) + const labelList& map = constructMap[domain]; + + if (domain != Pstream::myProcNo() && map.size() > 0) { IPstream fromNbr(Pstream::blocking, domain); List subField(fromNbr); - const labelList& map = constructMap[domain]; + if (subField.size() != map.size()) + { + FatalErrorIn + ( + "template\n" + "void mapDistribute::distribute\n" + "(\n" + " const Pstream::commsTypes commsType,\n" + " const List& schedule,\n" + " const label constructSize,\n" + " const labelListList& subMap,\n" + " const labelListList& constructMap,\n" + " List& field\n" + ")\n" + ) << "Expected from processor " << domain + << " " << map.size() << " but received " + << subField.size() << " elements." + << abort(FatalError); + } forAll(map, i) { @@ -126,6 +142,7 @@ void Foam::mapDistribute::distribute newField[map[i]] = subField[i]; } + // Schedule will already have pruned 0-sized comms forAll(schedule, i) { const labelPair& twoProcs = schedule[i]; @@ -154,6 +171,26 @@ void Foam::mapDistribute::distribute const labelList& map = constructMap[sendProc]; + if (subField.size() != map.size()) + { + FatalErrorIn + ( + "template\n" + "void mapDistribute::distribute\n" + "(\n" + " const Pstream::commsTypes commsType,\n" + " const List& schedule,\n" + " const label constructSize,\n" + " const labelListList& subMap,\n" + " const labelListList& constructMap,\n" + " List& field\n" + ")\n" + ) << "Expected from processor " << sendProc + << " " << map.size() << " but received " + << subField.size() << " elements." + << abort(FatalError); + } + forAll(map, i) { newField[map[i]] = subField[i]; @@ -164,7 +201,74 @@ void Foam::mapDistribute::distribute } else if (commsType == Pstream::nonBlocking) { - List newField(constructSize); + if (!contiguous()) + { + FatalErrorIn + ( + "template\n" + "void mapDistribute::distribute\n" + "(\n" + " const Pstream::commsTypes commsType,\n" + " const List& schedule,\n" + " const label constructSize,\n" + " const labelListList& subMap,\n" + " const labelListList& constructMap,\n" + " List& field\n" + ")\n" + ) << "Non-blocking only supported for contiguous data." + << exit(FatalError); + } + + // Set up sends to neighbours + + List > sendFields(Pstream::nProcs()); + + for (label domain = 0; domain < Pstream::nProcs(); domain++) + { + const labelList& map = subMap[domain]; + + if (domain != Pstream::myProcNo() && map.size() > 0) + { + List& subField = sendFields[domain]; + subField.setSize(map.size()); + forAll(map, i) + { + subField[i] = field[map[i]]; + } + + OPstream::write + ( + Pstream::nonBlocking, + domain, + reinterpret_cast(subField.begin()), + subField.size() + ); + } + } + + // Set up receives from neighbours + + List > recvFields(Pstream::nProcs()); + + for (label domain = 0; domain < Pstream::nProcs(); domain++) + { + const labelList& map = constructMap[domain]; + + if (domain != Pstream::myProcNo() && map.size() > 0) + { + recvFields[domain].setSize(map.size()); + IPstream::read + ( + Pstream::nonBlocking, + domain, + reinterpret_cast(recvFields[domain].begin()), + recvFields[domain].size() + ); + } + } + + + // Combine bits. Note that can reuse field storage // Subset myself const labelList& mySubMap = subMap[Pstream::myProcNo()]; @@ -175,54 +279,56 @@ void Foam::mapDistribute::distribute subField[i] = field[mySubMap[i]]; } + field.setSize(constructSize); + // Receive sub field from myself (subField) const labelList& map = constructMap[Pstream::myProcNo()]; forAll(map, i) { - newField[map[i]] = subField[i]; - } - - // Send sub field to neighbour - for (label domain = 0; domain < Pstream::nProcs(); domain++) - { - const labelList& map = subMap[domain]; - - if (domain != Pstream::myProcNo() && map.size() > 0) - { - - List subField(map.size()); - forAll(map, i) - { - subField[i] = field[map[i]]; - } - - OPstream toNbr(Pstream::nonBlocking, domain); - toNbr << subField; - } + field[map[i]] = subField[i]; } - // Receive sub field from neighbour + // Wait for all to finish + + OPstream::waitRequests(); + IPstream::waitRequests(); + + // Collect neighbour fields + for (label domain = 0; domain < Pstream::nProcs(); domain++) { const labelList& map = constructMap[domain]; if (domain != Pstream::myProcNo() && map.size() > 0) { - IPstream fromNbr(Pstream::nonBlocking, domain); - List subField(fromNbr); + if (recvFields[domain].size() != map.size()) + { + FatalErrorIn + ( + "template\n" + "void mapDistribute::distribute\n" + "(\n" + " const Pstream::commsTypes commsType,\n" + " const List& schedule,\n" + " const label constructSize,\n" + " const labelListList& subMap,\n" + " const labelListList& constructMap,\n" + " List& field\n" + ")\n" + ) << "Expected from processor " << domain + << " " << map.size() << " but received " + << recvFields[domain].size() << " elements." + << abort(FatalError); + } forAll(map, i) { - newField[map[i]] = subField[i]; + field[map[i]] = recvFields[domain][i]; } } } - OPstream::waitRequests(); - IPstream::waitRequests(); - - field.transfer(newField); } else {