From 927fc0a3134080f84f8fedbac150e70d023060d0 Mon Sep 17 00:00:00 2001 From: mattijs Date: Thu, 7 Jun 2012 17:27:02 +0100 Subject: [PATCH] ENH: mpi/Pstream: encapsulate mpi_reduce calls --- .../db/IOstreams/Pstreams/PstreamReduceOps.H | 9 +- src/Pstream/dummy/UPstream.C | 4 + src/Pstream/mpi/UPstream.C | 277 +----------------- src/Pstream/mpi/allReduce.H | 72 +++++ src/Pstream/mpi/allReduceTemplates.C | 202 +++++++++++++ 5 files changed, 293 insertions(+), 271 deletions(-) create mode 100644 src/Pstream/mpi/allReduce.H create mode 100644 src/Pstream/mpi/allReduceTemplates.C diff --git a/src/OpenFOAM/db/IOstreams/Pstreams/PstreamReduceOps.H b/src/OpenFOAM/db/IOstreams/Pstreams/PstreamReduceOps.H index f6f69bba9e..47c7c51acd 100644 --- a/src/OpenFOAM/db/IOstreams/Pstreams/PstreamReduceOps.H +++ b/src/OpenFOAM/db/IOstreams/Pstreams/PstreamReduceOps.H @@ -124,7 +124,7 @@ void reduce } -// Insist there is a specialisation for the sum reduction of scalar(s) +// Insist there are specialisations for the common reductions of scalar(s) void reduce ( scalar& Value, @@ -132,6 +132,13 @@ void reduce const int tag = Pstream::msgType() ); +void reduce +( + scalar& Value, + const minOp& bop, + const int tag = Pstream::msgType() +); + void reduce ( vector2D& Value, diff --git a/src/Pstream/dummy/UPstream.C b/src/Pstream/dummy/UPstream.C index d787291f1d..183a86cb9e 100644 --- a/src/Pstream/dummy/UPstream.C +++ b/src/Pstream/dummy/UPstream.C @@ -59,6 +59,10 @@ void Foam::reduce(scalar&, const sumOp&, const int) {} +void Foam::reduce(scalar&, const minOp&, const int) +{} + + void Foam::reduce(vector2D&, const sumOp&, const int) {} diff --git a/src/Pstream/mpi/UPstream.C b/src/Pstream/mpi/UPstream.C index 9438b1ca9a..bf7af5dd12 100644 --- a/src/Pstream/mpi/UPstream.C +++ b/src/Pstream/mpi/UPstream.C @@ -30,6 +30,7 @@ License #include "OSspecific.H" #include "PstreamGlobals.H" #include "SubList.H" +#include "allReduce.H" #include #include @@ -172,283 +173,19 @@ void Foam::UPstream::abort() void Foam::reduce(scalar& Value, const sumOp& bop, const int tag) { - if (Pstream::debug) - { - Pout<< "Foam::reduce : value:" << Value << endl; - } - - if (!UPstream::parRun()) - { - return; - } - - if (UPstream::nProcs() <= UPstream::nProcsSimpleSum) - { - if (UPstream::master()) - { - for - ( - int slave=UPstream::firstSlave(); - slave<=UPstream::lastSlave(); - slave++ - ) - { - scalar value; - - if - ( - MPI_Recv - ( - &value, - 1, - MPI_SCALAR, - UPstream::procID(slave), - tag, - MPI_COMM_WORLD, - MPI_STATUS_IGNORE - ) - ) - { - FatalErrorIn - ( - "reduce(scalar& Value, const sumOp& sumOp)" - ) << "MPI_Recv failed" - << Foam::abort(FatalError); - } - - Value = bop(Value, value); - } - } - else - { - if - ( - MPI_Send - ( - &Value, - 1, - MPI_SCALAR, - UPstream::procID(UPstream::masterNo()), - tag, - MPI_COMM_WORLD - ) - ) - { - FatalErrorIn - ( - "reduce(scalar& Value, const sumOp& sumOp)" - ) << "MPI_Send failed" - << Foam::abort(FatalError); - } - } + allReduce(Value, 1, MPI_SCALAR, MPI_SUM, bop, tag); +} - if (UPstream::master()) - { - for - ( - int slave=UPstream::firstSlave(); - slave<=UPstream::lastSlave(); - slave++ - ) - { - if - ( - MPI_Send - ( - &Value, - 1, - MPI_SCALAR, - UPstream::procID(slave), - tag, - MPI_COMM_WORLD - ) - ) - { - FatalErrorIn - ( - "reduce(scalar& Value, const sumOp& sumOp)" - ) << "MPI_Send failed" - << Foam::abort(FatalError); - } - } - } - else - { - if - ( - MPI_Recv - ( - &Value, - 1, - MPI_SCALAR, - UPstream::procID(UPstream::masterNo()), - tag, - MPI_COMM_WORLD, - MPI_STATUS_IGNORE - ) - ) - { - FatalErrorIn - ( - "reduce(scalar& Value, const sumOp& sumOp)" - ) << "MPI_Recv failed" - << Foam::abort(FatalError); - } - } - } - else - { - scalar sum; - MPI_Allreduce(&Value, &sum, 1, MPI_SCALAR, MPI_SUM, MPI_COMM_WORLD); - Value = sum; - } - - if (Pstream::debug) - { - Pout<< "Foam::reduce : reduced value:" << Value << endl; - } +void Foam::reduce(scalar& Value, const minOp& bop, const int tag) +{ + allReduce(Value, 1, MPI_SCALAR, MPI_MIN, bop, tag); } void Foam::reduce(vector2D& Value, const sumOp& bop, const int tag) { - if (Pstream::debug) - { - Pout<< "Foam::reduce : value:" << Value << endl; - } - - if (!UPstream::parRun()) - { - return; - } - - if (UPstream::nProcs() <= UPstream::nProcsSimpleSum) - { - if (UPstream::master()) - { - for - ( - int slave=UPstream::firstSlave(); - slave<=UPstream::lastSlave(); - slave++ - ) - { - vector2D value; - - if - ( - MPI_Recv - ( - &value, - 2, - MPI_SCALAR, - UPstream::procID(slave), - tag, - MPI_COMM_WORLD, - MPI_STATUS_IGNORE - ) - ) - { - FatalErrorIn - ( - "reduce(vector2D& Value, const sumOp& sumOp)" - ) << "MPI_Recv failed" - << Foam::abort(FatalError); - } - - Value = bop(Value, value); - } - } - else - { - if - ( - MPI_Send - ( - &Value, - 2, - MPI_SCALAR, - UPstream::procID(UPstream::masterNo()), - tag, - MPI_COMM_WORLD - ) - ) - { - FatalErrorIn - ( - "reduce(vector2D& Value, const sumOp& sumOp)" - ) << "MPI_Send failed" - << Foam::abort(FatalError); - } - } - - - if (UPstream::master()) - { - for - ( - int slave=UPstream::firstSlave(); - slave<=UPstream::lastSlave(); - slave++ - ) - { - if - ( - MPI_Send - ( - &Value, - 2, - MPI_SCALAR, - UPstream::procID(slave), - tag, - MPI_COMM_WORLD - ) - ) - { - FatalErrorIn - ( - "reduce(vector2D& Value, const sumOp& sumOp)" - ) << "MPI_Send failed" - << Foam::abort(FatalError); - } - } - } - else - { - if - ( - MPI_Recv - ( - &Value, - 2, - MPI_SCALAR, - UPstream::procID(UPstream::masterNo()), - tag, - MPI_COMM_WORLD, - MPI_STATUS_IGNORE - ) - ) - { - FatalErrorIn - ( - "reduce(vector2D& Value, const sumOp& sumOp)" - ) << "MPI_Recv failed" - << Foam::abort(FatalError); - } - } - } - else - { - vector2D sum; - MPI_Allreduce(&Value, &sum, 2, MPI_SCALAR, MPI_SUM, MPI_COMM_WORLD); - Value = sum; - } - - if (Pstream::debug) - { - Pout<< "Foam::reduce : reduced value:" << Value << endl; - } + allReduce(Value, 2, MPI_SCALAR, MPI_SUM, bop, tag); } diff --git a/src/Pstream/mpi/allReduce.H b/src/Pstream/mpi/allReduce.H new file mode 100644 index 0000000000..bb4c693113 --- /dev/null +++ b/src/Pstream/mpi/allReduce.H @@ -0,0 +1,72 @@ +/*---------------------------------------------------------------------------*\ + ========= | + \\ / F ield | OpenFOAM: The Open Source CFD Toolbox + \\ / O peration | + \\ / A nd | Copyright (C) 2012 OpenFOAM Foundation + \\/ M anipulation | +------------------------------------------------------------------------------- +License + This file is part of OpenFOAM. + + OpenFOAM is free software: you can redistribute it and/or modify it + under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + OpenFOAM is distributed in the hope that it will be useful, but WITHOUT + ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + for more details. + + You should have received a copy of the GNU General Public License + along with OpenFOAM. If not, see . + +InNamspace + Foam + +Description + Various functions to wrap MPI_Allreduce + +SourceFiles + allReduceTemplates.C + +\*---------------------------------------------------------------------------*/ + +#ifndef allReduce_H +#define allReduce_H + +#include "mpi.h" + +#include "UPstream.H" + +// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * // + +namespace Foam +{ + +template +void allReduce +( + Type& Value, + int count, + MPI_Datatype MPIType, + MPI_Op op, + const BinaryOp& bop, + const int tag +); + +// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * // + +} // End namespace Foam + +// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * // + +#ifdef NoRepository +# include "allReduceTemplates.C" +#endif + +// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * // + +#endif + +// ************************************************************************* // diff --git a/src/Pstream/mpi/allReduceTemplates.C b/src/Pstream/mpi/allReduceTemplates.C new file mode 100644 index 0000000000..98b8d97b0a --- /dev/null +++ b/src/Pstream/mpi/allReduceTemplates.C @@ -0,0 +1,202 @@ +/*---------------------------------------------------------------------------*\ + ========= | + \\ / F ield | OpenFOAM: The Open Source CFD Toolbox + \\ / O peration | + \\ / A nd | Copyright (C) 2012 OpenFOAM Foundation + \\/ M anipulation | +------------------------------------------------------------------------------- +License + This file is part of OpenFOAM. + + OpenFOAM is free software: you can redistribute it and/or modify it + under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + OpenFOAM is distributed in the hope that it will be useful, but WITHOUT + ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + for more details. + + You should have received a copy of the GNU General Public License + along with OpenFOAM. If not, see . + +\*---------------------------------------------------------------------------*/ + +#include "allReduce.H" + +// * * * * * * * * * * * * * * * Global Functions * * * * * * * * * * * * * // + +template +void Foam::allReduce +( + Type& Value, + int MPICount, + MPI_Datatype MPIType, + MPI_Op MPIOp, + const BinaryOp& bop, + const int tag +) +{ + if (!UPstream::parRun()) + { + return; + } + + if (UPstream::nProcs() <= UPstream::nProcsSimpleSum) + { + if (UPstream::master()) + { + for + ( + int slave=UPstream::firstSlave(); + slave<=UPstream::lastSlave(); + slave++ + ) + { + Type value; + + if + ( + MPI_Recv + ( + &value, + MPICount, + MPIType, + UPstream::procID(slave), + tag, + MPI_COMM_WORLD, + MPI_STATUS_IGNORE + ) + ) + { + FatalErrorIn + ( + "void Foam::allReduce\n" + "(\n" + " Type&,\n" + " int,\n" + " MPI_Datatype,\n" + " MPI_Op,\n" + " const BinaryOp&,\n" + " const int\n" + ")\n" + ) << "MPI_Recv failed" + << Foam::abort(FatalError); + } + + Value = bop(Value, value); + } + } + else + { + if + ( + MPI_Send + ( + &Value, + MPICount, + MPIType, + UPstream::procID(UPstream::masterNo()), + tag, + MPI_COMM_WORLD + ) + ) + { + FatalErrorIn + ( + "void Foam::allReduce\n" + "(\n" + " Type&,\n" + " int,\n" + " MPI_Datatype,\n" + " MPI_Op,\n" + " const BinaryOp&,\n" + " const int\n" + ")\n" + ) << "MPI_Send failed" + << Foam::abort(FatalError); + } + } + + + if (UPstream::master()) + { + for + ( + int slave=UPstream::firstSlave(); + slave<=UPstream::lastSlave(); + slave++ + ) + { + if + ( + MPI_Send + ( + &Value, + MPICount, + MPIType, + UPstream::procID(slave), + tag, + MPI_COMM_WORLD + ) + ) + { + FatalErrorIn + ( + "void Foam::allReduce\n" + "(\n" + " Type&,\n" + " int,\n" + " MPI_Datatype,\n" + " MPI_Op,\n" + " const BinaryOp&,\n" + " const int\n" + ")\n" + ) << "MPI_Send failed" + << Foam::abort(FatalError); + } + } + } + else + { + if + ( + MPI_Recv + ( + &Value, + MPICount, + MPIType, + UPstream::procID(UPstream::masterNo()), + tag, + MPI_COMM_WORLD, + MPI_STATUS_IGNORE + ) + ) + { + FatalErrorIn + ( + "void Foam::allReduce\n" + "(\n" + " Type&,\n" + " int,\n" + " MPI_Datatype,\n" + " MPI_Op,\n" + " const BinaryOp&,\n" + " const int\n" + ")\n" + ) << "MPI_Recv failed" + << Foam::abort(FatalError); + } + } + } + else + { + Type sum; + MPI_Allreduce(&Value, &sum, MPICount, MPIType, MPIOp, MPI_COMM_WORLD); + Value = sum; + } +} + + +// ************************************************************************* //