ENH: add simple profiling of MPI communications

This commit is contained in:
Mark Olesen
2019-04-25 15:33:51 +02:00
committed by Andrew Heather
parent 53be19989b
commit 828f8e8504
12 changed files with 715 additions and 11 deletions

View File

@ -11,6 +11,7 @@ global/profiling/profiling.C
global/profiling/profilingInformation.C
global/profiling/profilingSysInfo.C
global/profiling/profilingTrigger.C
global/profiling/profilingPstream.C
global/etcFiles/etcFiles.C
global/version/foamVersion.C

View File

@ -0,0 +1,100 @@
/*---------------------------------------------------------------------------*\
========= |
\\ / F ield | OpenFOAM: The Open Source CFD Toolbox
\\ / O peration |
\\ / A nd | Copyright (C) 2019 OpenCFD Ltd.
\\/ M anipulation |
-------------------------------------------------------------------------------
License
This file is part of OpenFOAM.
OpenFOAM is free software: you can redistribute it and/or modify it
under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
OpenFOAM is distributed in the hope that it will be useful, but WITHOUT
ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
for more details.
You should have received a copy of the GNU General Public License
along with OpenFOAM. If not, see <http://www.gnu.org/licenses/>.
\*---------------------------------------------------------------------------*/
#include "profilingPstream.H"
// * * * * * * * * * * * * * * Static Data Members * * * * * * * * * * * * * //
Foam::autoPtr<Foam::cpuTime> Foam::profilingPstream::timer_(nullptr);
Foam::autoPtr<Foam::cpuTime> Foam::profilingPstream::suspend_(nullptr);
Foam::FixedList<Foam::scalar, 5> Foam::profilingPstream::times_(Zero);
// * * * * * * * * * * * * * * * * Constructors * * * * * * * * * * * * * * //
Foam::profilingPstream::profilingPstream()
{
enable();
}
// * * * * * * * * * * * * * * * * Destructor * * * * * * * * * * * * * * * //
Foam::profilingPstream::~profilingPstream()
{
disable();
}
// * * * * * * * * * * * * * * * Member Functions * * * * * * * * * * * * * //
void Foam::profilingPstream::enable()
{
if (timer_.valid())
{
timer_->resetCpuTime(); // Not really needed ...
}
else if (suspend_.valid())
{
suspend_.swap(timer_);
timer_->resetCpuTime(); // Not really needed ...
}
else
{
timer_.reset(new cpuTime);
times_ = Zero;
}
suspend_.clear();
}
void Foam::profilingPstream::disable()
{
timer_.clear();
suspend_.clear();
}
void Foam::profilingPstream::suspend()
{
suspend_.clear();
suspend_.swap(timer_);
}
void Foam::profilingPstream::resume()
{
if (suspend_.valid())
{
timer_.clear();
timer_.swap(suspend_);
}
}
// ************************************************************************* //

View File

@ -0,0 +1,179 @@
/*---------------------------------------------------------------------------*\
========= |
\\ / F ield | OpenFOAM: The Open Source CFD Toolbox
\\ / O peration |
\\ / A nd | Copyright (C) 2019 OpenCFD Ltd.
\\/ M anipulation |
-------------------------------------------------------------------------------
License
This file is part of OpenFOAM.
OpenFOAM is free software: you can redistribute it and/or modify it
under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
OpenFOAM is distributed in the hope that it will be useful, but WITHOUT
ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
for more details.
You should have received a copy of the GNU General Public License
along with OpenFOAM. If not, see <http://www.gnu.org/licenses/>.
Class
Foam::profilingPstream
Description
Timers and values for simple (simplistic) mpi-profiling. The entire
class behaves as a singleton.
SourceFiles
profilingPstream.C
\*---------------------------------------------------------------------------*/
#ifndef profilingPstream_H
#define profilingPstream_H
#include "autoPtr.H"
#include "cpuTime.H"
#include "scalar.H"
#include "FixedList.H"
// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
namespace Foam
{
/*---------------------------------------------------------------------------*\
Class profilingPstream Declaration
\*---------------------------------------------------------------------------*/
class profilingPstream
{
//- Timer to use
static autoPtr<cpuTime> timer_;
//- Stash for timer during suspend
static autoPtr<cpuTime> suspend_;
//- The timing values
static FixedList<scalar, 5> times_;
public:
//- Enumeration within times array
enum timingType
{
GATHER = 0,
SCATTER,
REDUCE,
WAIT,
ALL_TO_ALL
};
public:
// Constructors
//- Construct and enable global timer
profilingPstream();
//- Destructor - remove global timer
~profilingPstream();
// Member Functions
//- Create timer for measuring communication, or reset existing
static void enable();
//- Remove timer for measuring communication activity
static void disable();
//- Suspend use of timer (if active)
static void suspend();
//- Resume use of timer (if previously active)
static void resume();
//- Timer is active
inline static bool active()
{
return timer_.valid();
}
//- Access to the timing information
inline static FixedList<scalar, 5>& times()
{
return times_;
}
//- Access to the timing information
inline static scalar times(const enum timingType idx)
{
return times_[idx];
}
//- Update timer prior to measurement
inline static void beginTiming()
{
if (timer_.valid())
{
(void) timer_->cpuTimeIncrement();
}
}
//- Add time increment
inline static void addTime(const enum timingType idx)
{
if (timer_.valid())
{
times_[idx] += timer_->cpuTimeIncrement();
}
}
//- Add time increment to gatherTime
inline static void addGatherTime()
{
addTime(GATHER);
}
//- Add time increment to scatterTime
inline static void addScatterTime()
{
addTime(SCATTER);
}
//- Add time increment to reduceTime
inline static void addReduceTime()
{
addTime(REDUCE);
}
//- Add time increment to waitTime
inline static void addWaitTime()
{
addTime(WAIT);
}
//- Add time increment to allToAllTime
inline static void addAllToAllTime()
{
addTime(ALL_TO_ALL);
}
};
// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
} // End namespace Foam
// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
#endif
// ************************************************************************* //

View File

@ -2,7 +2,7 @@
========= |
\\ / F ield | OpenFOAM: The Open Source CFD Toolbox
\\ / O peration |
\\ / A nd |
\\ / A nd | Copyright (C) 2019 OpenCFD Ltd.
\\/ M anipulation |
-------------------------------------------------------------------------------
| Copyright (C) 2011-2017 OpenFOAM Foundation
@ -30,6 +30,7 @@ Description
#include "UIPstream.H"
#include "PstreamGlobals.H"
#include "profilingPstream.H"
#include "IOstreams.H"
#include <mpi.h>
@ -85,6 +86,8 @@ Foam::UIPstream::UIPstream
// and set it
if (!wantedSize)
{
profilingPstream::beginTiming();
MPI_Probe
(
fromProcNo_,
@ -94,6 +97,8 @@ Foam::UIPstream::UIPstream
);
MPI_Get_count(&status, MPI_BYTE, &messageSize_);
profilingPstream::addWaitTime();
externalBuf_.setCapacity(messageSize_);
wantedSize = messageSize_;
@ -186,6 +191,8 @@ Foam::UIPstream::UIPstream(const int fromProcNo, PstreamBuffers& buffers)
// and set it
if (!wantedSize)
{
profilingPstream::beginTiming();
MPI_Probe
(
fromProcNo_,
@ -195,6 +202,8 @@ Foam::UIPstream::UIPstream(const int fromProcNo, PstreamBuffers& buffers)
);
MPI_Get_count(&status, MPI_BYTE, &messageSize_);
profilingPstream::addWaitTime();
externalBuf_.setCapacity(messageSize_);
wantedSize = messageSize_;
@ -257,6 +266,8 @@ Foam::label Foam::UIPstream::read
error::printStack(Pout);
}
profilingPstream::beginTiming();
if (commsType == commsTypes::blocking || commsType == commsTypes::scheduled)
{
MPI_Status status;
@ -282,6 +293,7 @@ Foam::label Foam::UIPstream::read
return 0;
}
profilingPstream::addScatterTime();
// Check size of message read
@ -332,6 +344,8 @@ Foam::label Foam::UIPstream::read
return 0;
}
profilingPstream::addScatterTime();
if (debug)
{
Pout<< "UIPstream::read : started read from:" << fromProcNo
@ -346,16 +360,13 @@ Foam::label Foam::UIPstream::read
// Assume the message is completely received.
return bufSize;
}
else
{
FatalErrorInFunction
<< "Unsupported communications type "
<< int(commsType)
<< "Unsupported communications type " << int(commsType)
<< Foam::abort(FatalError);
return 0;
}
}
// ************************************************************************* //

View File

@ -2,7 +2,7 @@
========= |
\\ / F ield | OpenFOAM: The Open Source CFD Toolbox
\\ / O peration |
\\ / A nd |
\\ / A nd | Copyright (C) 2019 OpenCFD Ltd.
\\/ M anipulation |
-------------------------------------------------------------------------------
| Copyright (C) 2011-2017 OpenFOAM Foundation
@ -30,6 +30,7 @@ Description
#include "UOPstream.H"
#include "PstreamGlobals.H"
#include "profilingPstream.H"
#include <mpi.h>
@ -70,6 +71,8 @@ bool Foam::UOPstream::write
bool transferFailed = true;
profilingPstream::beginTiming();
if (commsType == commsTypes::blocking)
{
transferFailed = MPI_Bsend
@ -82,6 +85,8 @@ bool Foam::UOPstream::write
PstreamGlobals::MPICommunicators_[communicator] //MPI_COMM_WORLD
);
profilingPstream::addGatherTime();
if (debug)
{
Pout<< "UOPstream::write : finished write to:" << toProcNo
@ -102,6 +107,8 @@ bool Foam::UOPstream::write
PstreamGlobals::MPICommunicators_[communicator] //MPI_COMM_WORLD
);
profilingPstream::addWaitTime();
if (debug)
{
Pout<< "UOPstream::write : finished write to:" << toProcNo
@ -125,6 +132,8 @@ bool Foam::UOPstream::write
&request
);
profilingPstream::addWaitTime();
if (debug)
{
Pout<< "UOPstream::write : started write to:" << toProcNo

View File

@ -28,6 +28,7 @@ License
#include "Pstream.H"
#include "PstreamReduceOps.H"
#include "PstreamGlobals.H"
#include "profilingPstream.H"
#include "SubList.H"
#include "allReduce.H"
#include "int.H"
@ -501,6 +502,8 @@ void Foam::UPstream::allToAll
}
else
{
profilingPstream::beginTiming();
if
(
MPI_Alltoall
@ -522,6 +525,8 @@ void Foam::UPstream::allToAll
<< " on communicator " << communicator
<< Foam::abort(FatalError);
}
profilingPstream::addAllToAllTime();
}
}
@ -572,6 +577,8 @@ void Foam::UPstream::allToAll
}
else
{
profilingPstream::beginTiming();
if
(
MPI_Alltoallv
@ -594,6 +601,8 @@ void Foam::UPstream::allToAll
<< " communicator " << communicator
<< Foam::abort(FatalError);
}
profilingPstream::addAllToAllTime();
}
}
@ -634,6 +643,8 @@ void Foam::UPstream::gather
}
else
{
profilingPstream::beginTiming();
if
(
MPI_Gatherv
@ -656,6 +667,8 @@ void Foam::UPstream::gather
<< " communicator " << communicator
<< Foam::abort(FatalError);
}
profilingPstream::addGatherTime();
}
}
@ -693,6 +706,8 @@ void Foam::UPstream::scatter
}
else
{
profilingPstream::beginTiming();
if
(
MPI_Scatterv
@ -715,6 +730,8 @@ void Foam::UPstream::scatter
<< " communicator " << communicator
<< Foam::abort(FatalError);
}
profilingPstream::addScatterTime();
}
}
@ -869,6 +886,8 @@ void Foam::UPstream::waitRequests(const label start)
start
);
profilingPstream::beginTiming();
if
(
MPI_Waitall
@ -883,6 +902,8 @@ void Foam::UPstream::waitRequests(const label start)
<< "MPI_Waitall returned with error" << Foam::endl;
}
profilingPstream::addWaitTime();
resetRequests(start);
}
@ -911,6 +932,8 @@ void Foam::UPstream::waitRequest(const label i)
<< Foam::abort(FatalError);
}
profilingPstream::beginTiming();
if
(
MPI_Wait
@ -924,6 +947,8 @@ void Foam::UPstream::waitRequest(const label i)
<< "MPI_Wait returned with error" << Foam::endl;
}
profilingPstream::addWaitTime();
if (debug)
{
Pout<< "UPstream::waitRequest : finished wait for request:" << i

View File

@ -2,7 +2,7 @@
========= |
\\ / F ield | OpenFOAM: The Open Source CFD Toolbox
\\ / O peration |
\\ / A nd |
\\ / A nd | Copyright (C) 2019 OpenCFD Ltd.
\\/ M anipulation |
-------------------------------------------------------------------------------
| Copyright (C) 2012-2015 OpenFOAM Foundation
@ -26,6 +26,7 @@ License
\*---------------------------------------------------------------------------*/
#include "allReduce.H"
#include "profilingPstream.H"
// * * * * * * * * * * * * * * * Global Functions * * * * * * * * * * * * * //
@ -46,6 +47,8 @@ void Foam::allReduce
return;
}
profilingPstream::beginTiming();
if (UPstream::nProcs(communicator) <= UPstream::nProcsSimpleSum)
{
if (UPstream::master(communicator))
@ -167,6 +170,8 @@ void Foam::allReduce
);
Value = sum;
}
profilingPstream::addReduceTime();
}

View File

@ -12,6 +12,8 @@ vtkWrite/vtkWriteUpdate.C
removeRegisteredObject/removeRegisteredObject.C
parProfiling/parProfiling.C
solverInfo/solverInfo.C
runTimeControl/runTimeControl.C

View File

@ -0,0 +1,233 @@
/*---------------------------------------------------------------------------*\
========= |
\\ / F ield | OpenFOAM: The Open Source CFD Toolbox
\\ / O peration |
\\ / A nd | Copyright (C) 2019 OpenCFD Ltd.
\\/ M anipulation |
-------------------------------------------------------------------------------
License
This file is part of OpenFOAM.
OpenFOAM is free software: you can redistribute it and/or modify it
under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
OpenFOAM is distributed in the hope that it will be useful, but WITHOUT
ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
for more details.
You should have received a copy of the GNU General Public License
along with OpenFOAM. If not, see <http://www.gnu.org/licenses/>.
\*---------------------------------------------------------------------------*/
#include "parProfiling.H"
#include "addToRunTimeSelectionTable.H"
#include "UPstream.H"
#include "Pstream.H"
#include "PstreamReduceOps.H"
#include "profilingPstream.H"
#include "Tuple2.H"
#include "FixedList.H"
// * * * * * * * * * * * * * * Static Data Members * * * * * * * * * * * * * //
namespace Foam
{
namespace functionObjects
{
defineTypeNameAndDebug(parProfiling, 0);
addToRunTimeSelectionTable
(
functionObject,
parProfiling,
dictionary
);
} // End namespace functionObject
// Processor and time for each of: -min -max -sum
typedef FixedList<Tuple2<label, scalar>, 3> statData;
//- Reduction class. If x and y are not equal assign value.
struct statsEqOp
{
void operator()
(
FixedList<statData, 2>& xStats,
const FixedList<statData, 2>& yStats
) const
{
forAll(xStats, i)
{
statData& x = xStats[i];
const statData& y = yStats[i];
// 0 : min
// 1 : max
// 2 : sum
if (x[0].second() > y[0].second())
{
x[0].second() = y[0].second();
x[0].first() = y[0].first();
}
if (x[1].second() < y[1].second())
{
x[1].second() = y[1].second();
x[1].first() = y[1].first();
}
x[2].second() += y[2].second();
x[2].first()++;
}
}
};
} // End namespace Foam
// * * * * * * * * * * * * * * * * Constructors * * * * * * * * * * * * * * //
Foam::functionObjects::parProfiling::parProfiling
(
const word& name,
const Time& runTime,
const dictionary& dict
)
:
functionObject(name)
{
profilingPstream::enable();
}
// * * * * * * * * * * * * * * * * Destructor * * * * * * * * * * * * * * * //
Foam::functionObjects::parProfiling::~parProfiling()
{
profilingPstream::disable();
}
// * * * * * * * * * * * * * * * Member Functions * * * * * * * * * * * * * //
void Foam::functionObjects::parProfiling::report()
{
if (!profilingPstream::active())
{
return;
}
typedef FixedList<Tuple2<label, scalar>, 3> statData;
FixedList<statData, 2> times;
{
const scalar masterTime =
(
profilingPstream::times(profilingPstream::REDUCE)
+ profilingPstream::times(profilingPstream::GATHER)
+ profilingPstream::times(profilingPstream::SCATTER)
);
statData& reduceStats = times[0];
Tuple2<label, scalar>& minTime = reduceStats[0];
minTime.first() = Pstream::myProcNo();
minTime.second() = masterTime;
Tuple2<label, scalar>& maxTime = reduceStats[1];
maxTime.first() = Pstream::myProcNo();
maxTime.second() = masterTime;
Tuple2<label, scalar>& sumTime = reduceStats[2];
sumTime.first() = 1;
sumTime.second() = masterTime;
}
{
const scalar allTime =
(
profilingPstream::times(profilingPstream::WAIT)
+ profilingPstream::times(profilingPstream::ALL_TO_ALL)
);
statData& allToAllStats = times[1];
Tuple2<label, scalar>& minTime = allToAllStats[0];
minTime.first() = Pstream::myProcNo();
minTime.second() = allTime;
Tuple2<label, scalar>& maxTime = allToAllStats[1];
maxTime.first() = Pstream::myProcNo();
maxTime.second() = allTime;
Tuple2<label, scalar>& sumTime = allToAllStats[2];
sumTime.first() = 1;
sumTime.second() = allTime;
}
profilingPstream::suspend();
Pstream::combineGather(times, statsEqOp());
profilingPstream::resume();
if (Pstream::master())
{
const statData& reduceStats = times[0];
const statData& allToAllStats = times[1];
scalar reduceAvg = reduceStats[2].second()/Pstream::nProcs();
scalar reduceRelMin =
(reduceStats[0].second()-reduceAvg)/(reduceAvg + VSMALL);
scalar reduceRelMax =
(reduceStats[1].second()-reduceAvg)/(reduceAvg + VSMALL);
scalar allToAllAvg = allToAllStats[2].second()/Pstream::nProcs();
scalar allToAllRelMin =
(allToAllStats[0].second()-allToAllAvg)/(allToAllAvg + VSMALL);
scalar allToAllRelMax =
(allToAllStats[1].second()-allToAllAvg)/(allToAllAvg + VSMALL);
Info<< type() << ':' << nl
<< "\treduce : avg = " << reduceAvg << 's' << nl
<< "\t min = " << reduceRelMin*100
<< "% (processor " << reduceStats[0].first() << ')' << nl
<< "\t max = +" << reduceRelMax*100
<< "% (processor " << reduceStats[1].first() << ')' << nl
<< "\tall-all : avg = " << allToAllAvg << 's' << nl
<< "\t min = " << allToAllRelMin*100
<< "% (processor " << allToAllStats[0].first() << ')' << nl
<< "\t max = +" << allToAllRelMax*100
<< "% (processor " << allToAllStats[1].first() << ')' << endl;
}
}
bool Foam::functionObjects::parProfiling::execute()
{
report();
return true;
}
bool Foam::functionObjects::parProfiling::write()
{
return true;
}
bool Foam::functionObjects::parProfiling::end()
{
report();
profilingPstream::disable();
return true;
}
// ************************************************************************* //

View File

@ -0,0 +1,117 @@
/*---------------------------------------------------------------------------*\
========= |
\\ / F ield | OpenFOAM: The Open Source CFD Toolbox
\\ / O peration |
\\ / A nd | Copyright (C) 2019 OpenCFD Ltd.
\\/ M anipulation |
-------------------------------------------------------------------------------
License
This file is part of OpenFOAM.
OpenFOAM is free software: you can redistribute it and/or modify it
under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
OpenFOAM is distributed in the hope that it will be useful, but WITHOUT
ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
for more details.
You should have received a copy of the GNU General Public License
along with OpenFOAM. If not, see <http://www.gnu.org/licenses/>.
Class
Foam::functionObjects::parProfiling
Group
grpUtilitiesFunctionObjects
Description
Simple (simplistic) mpi-profiling.
SourceFiles
parProfiling.C
\*---------------------------------------------------------------------------*/
#ifndef functionObjects_parProfiling_H
#define functionObjects_parProfiling_H
#include "functionObject.H"
// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
namespace Foam
{
// Forward declaration of classes
class Time;
namespace functionObjects
{
/*---------------------------------------------------------------------------*\
Class parProfiling Declaration
\*---------------------------------------------------------------------------*/
class parProfiling
:
public functionObject
{
// Private Member Functions
//- Disallow default bitwise copy construct
parProfiling(const parProfiling&) = delete;
//- Disallow default bitwise assignment
void operator=(const parProfiling&) = delete;
public:
//- Runtime type information
TypeName("parProfiling");
// Constructors
//- Construct from Time and dictionary
parProfiling
(
const word& name,
const Time& runTime,
const dictionary& dict
);
//- Destructor
virtual ~parProfiling();
// Member Functions
//- Report the current profiling information
void report();
//- Report
virtual bool execute();
//- Do nothing
virtual bool write();
//- Report
virtual bool end();
};
// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
} // End namespace functionObjects
} // End namespace Foam
// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
#endif
// ************************************************************************* //

View File

@ -55,6 +55,8 @@ functions
// Demonstrate runTimePostProcessing
#include "visualization"
#include "profiling"
}

View File

@ -0,0 +1,20 @@
/*--------------------------------*- C++ -*----------------------------------*\
| ========= | |
| \\ / F ield | OpenFOAM: The Open Source CFD Toolbox |
| \\ / O peration | Version: v1812 |
| \\ / A nd | Web: www.OpenFOAM.com |
| \\/ M anipulation | |
\*---------------------------------------------------------------------------*/
profiling
{
type parProfiling;
libs ("libutilityFunctionObjects.so");
executeControl none;
writeControl writeTime;
}
// ************************************************************************* //