ENH: globalIndex gather ops with reduced communication (#2332)

- for contiguous data, added mpiGatherOp() to complement the
  gatherOp() static method

- the gather ops (static methods) populate the globalIndex on the
  master only (not needed on other procs) for reduced communication

- rename inplace gather methods to include 'inplace' in their name.
  Regular gather methods return the gathered data directly, which
  allows the following:

      const scalarField mergedWeights(globalFaces().gather(wghtSum));

  vs.
      scalarField mergedWeights;
      globalFaces().gather(wghtSum, mergedWeights());

  or even:

      scalarField mergedWeights;
      List<scalarField> allWeights(Pstream::nProcs());
      allWeights[Pstream::myProcNo()] = wghtSum;
      Pstream::gatherList(allWeights);
      if (Pstream::master())
      {
          mergedWeights =
              ListListOps::combine<scalarField>
              (
                  allWeights, accessOp<scalarField>()
              );
       }

- add parRun guards on various globalIndex gather methods
  (simple copies or no-ops in serial) to simplify the effort for callers.
This commit is contained in:
Mark Olesen
2022-01-19 10:33:19 +01:00
parent 6b99fea4e7
commit e1f06bf38e
5 changed files with 656 additions and 360 deletions

View File

@ -6,7 +6,7 @@
\\/ M anipulation |
-------------------------------------------------------------------------------
Copyright (C) 2011-2016 OpenFOAM Foundation
Copyright (C) 2021 OpenCFD Ltd.
Copyright (C) 2021-2022 OpenCFD Ltd.
-------------------------------------------------------------------------------
License
This file is part of OpenFOAM.
@ -198,15 +198,15 @@ int main(int argc, char *argv[])
// Do the brute-force method as well : collect all cell centres on all
// processors
pointField allCcs(globalNumbering.size());
globalNumbering.gather
(
Pstream::worldComm,
Pstream::procID(Pstream::worldComm),
mesh.cellCentres(),
allCcs
);
Info<< "Gathered/scattered cell centres:" << endl;
labelPair inOut;
pointField allCcs(globalNumbering.gather(mesh.cellCentres()));
inOut[0] = allCcs.size();
Pstream::scatter(allCcs);
inOut[1] = allCcs.size();
Pout<< " " << inOut << endl;
// Compare
forAll(ccs, i)
@ -239,10 +239,13 @@ int main(int argc, char *argv[])
Info<< "local-sizes: " << globalPointsPtr().sizes() << nl;
UIndirectList<point> procPoints(mesh.points(), uniqueMeshPointLabels);
pointField patchPoints;
globalPointsPtr().gather(procPoints, patchPoints);
pointField patchPoints
(
globalPointsPtr().gather
(
UIndirectList<point>(mesh.points(), uniqueMeshPointLabels)
)
);
Info<< "gathered point field = " << patchPoints.size() << " points\n";
}