diff --git a/src/OpenFOAM/Make/files b/src/OpenFOAM/Make/files
index 8d53c6653a..bc6680fc53 100644
--- a/src/OpenFOAM/Make/files
+++ b/src/OpenFOAM/Make/files
@@ -11,6 +11,7 @@ global/profiling/profiling.C
global/profiling/profilingInformation.C
global/profiling/profilingSysInfo.C
global/profiling/profilingTrigger.C
+global/profiling/profilingPstream.C
global/etcFiles/etcFiles.C
global/version/foamVersion.C
diff --git a/src/OpenFOAM/global/profiling/profilingPstream.C b/src/OpenFOAM/global/profiling/profilingPstream.C
new file mode 100644
index 0000000000..871b54cac9
--- /dev/null
+++ b/src/OpenFOAM/global/profiling/profilingPstream.C
@@ -0,0 +1,100 @@
+/*---------------------------------------------------------------------------*\
+ ========= |
+ \\ / F ield | OpenFOAM: The Open Source CFD Toolbox
+ \\ / O peration |
+ \\ / A nd | Copyright (C) 2019 OpenCFD Ltd.
+ \\/ M anipulation |
+-------------------------------------------------------------------------------
+License
+ This file is part of OpenFOAM.
+
+ OpenFOAM is free software: you can redistribute it and/or modify it
+ under the terms of the GNU General Public License as published by
+ the Free Software Foundation, either version 3 of the License, or
+ (at your option) any later version.
+
+ OpenFOAM is distributed in the hope that it will be useful, but WITHOUT
+ ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+ for more details.
+
+ You should have received a copy of the GNU General Public License
+ along with OpenFOAM. If not, see .
+
+\*---------------------------------------------------------------------------*/
+
+#include "profilingPstream.H"
+
+// * * * * * * * * * * * * * * Static Data Members * * * * * * * * * * * * * //
+
+Foam::autoPtr Foam::profilingPstream::timer_(nullptr);
+
+Foam::autoPtr Foam::profilingPstream::suspend_(nullptr);
+
+Foam::FixedList Foam::profilingPstream::times_(Zero);
+
+
+// * * * * * * * * * * * * * * * * Constructors * * * * * * * * * * * * * * //
+
+Foam::profilingPstream::profilingPstream()
+{
+ enable();
+}
+
+
+// * * * * * * * * * * * * * * * * Destructor * * * * * * * * * * * * * * * //
+
+Foam::profilingPstream::~profilingPstream()
+{
+ disable();
+}
+
+
+// * * * * * * * * * * * * * * * Member Functions * * * * * * * * * * * * * //
+
+void Foam::profilingPstream::enable()
+{
+ if (timer_.valid())
+ {
+ timer_->resetCpuTime(); // Not really needed ...
+ }
+ else if (suspend_.valid())
+ {
+ suspend_.swap(timer_);
+ timer_->resetCpuTime(); // Not really needed ...
+ }
+ else
+ {
+ timer_.reset(new cpuTime);
+ times_ = Zero;
+ }
+
+ suspend_.clear();
+}
+
+
+void Foam::profilingPstream::disable()
+{
+ timer_.clear();
+ suspend_.clear();
+}
+
+
+void Foam::profilingPstream::suspend()
+{
+ suspend_.clear();
+ suspend_.swap(timer_);
+}
+
+
+void Foam::profilingPstream::resume()
+{
+ if (suspend_.valid())
+ {
+ timer_.clear();
+ timer_.swap(suspend_);
+ }
+}
+
+
+// ************************************************************************* //
diff --git a/src/OpenFOAM/global/profiling/profilingPstream.H b/src/OpenFOAM/global/profiling/profilingPstream.H
new file mode 100644
index 0000000000..213a754414
--- /dev/null
+++ b/src/OpenFOAM/global/profiling/profilingPstream.H
@@ -0,0 +1,179 @@
+/*---------------------------------------------------------------------------*\
+ ========= |
+ \\ / F ield | OpenFOAM: The Open Source CFD Toolbox
+ \\ / O peration |
+ \\ / A nd | Copyright (C) 2019 OpenCFD Ltd.
+ \\/ M anipulation |
+-------------------------------------------------------------------------------
+License
+ This file is part of OpenFOAM.
+
+ OpenFOAM is free software: you can redistribute it and/or modify it
+ under the terms of the GNU General Public License as published by
+ the Free Software Foundation, either version 3 of the License, or
+ (at your option) any later version.
+
+ OpenFOAM is distributed in the hope that it will be useful, but WITHOUT
+ ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+ for more details.
+
+ You should have received a copy of the GNU General Public License
+ along with OpenFOAM. If not, see .
+
+Class
+ Foam::profilingPstream
+
+Description
+ Timers and values for simple (simplistic) mpi-profiling. The entire
+ class behaves as a singleton.
+
+SourceFiles
+ profilingPstream.C
+
+\*---------------------------------------------------------------------------*/
+
+#ifndef profilingPstream_H
+#define profilingPstream_H
+
+#include "autoPtr.H"
+#include "cpuTime.H"
+#include "scalar.H"
+#include "FixedList.H"
+
+// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
+
+namespace Foam
+{
+
+/*---------------------------------------------------------------------------*\
+ Class profilingPstream Declaration
+\*---------------------------------------------------------------------------*/
+
+class profilingPstream
+{
+ //- Timer to use
+ static autoPtr timer_;
+
+ //- Stash for timer during suspend
+ static autoPtr suspend_;
+
+ //- The timing values
+ static FixedList times_;
+
+
+public:
+
+ //- Enumeration within times array
+ enum timingType
+ {
+ GATHER = 0,
+ SCATTER,
+ REDUCE,
+ WAIT,
+ ALL_TO_ALL
+ };
+
+public:
+
+ // Constructors
+
+ //- Construct and enable global timer
+ profilingPstream();
+
+
+ //- Destructor - remove global timer
+ ~profilingPstream();
+
+
+ // Member Functions
+
+ //- Create timer for measuring communication, or reset existing
+ static void enable();
+
+ //- Remove timer for measuring communication activity
+ static void disable();
+
+ //- Suspend use of timer (if active)
+ static void suspend();
+
+ //- Resume use of timer (if previously active)
+ static void resume();
+
+ //- Timer is active
+ inline static bool active()
+ {
+ return timer_.valid();
+ }
+
+ //- Access to the timing information
+ inline static FixedList& times()
+ {
+ return times_;
+ }
+
+ //- Access to the timing information
+ inline static scalar times(const enum timingType idx)
+ {
+ return times_[idx];
+ }
+
+ //- Update timer prior to measurement
+ inline static void beginTiming()
+ {
+ if (timer_.valid())
+ {
+ (void) timer_->cpuTimeIncrement();
+ }
+ }
+
+ //- Add time increment
+ inline static void addTime(const enum timingType idx)
+ {
+ if (timer_.valid())
+ {
+ times_[idx] += timer_->cpuTimeIncrement();
+ }
+ }
+
+ //- Add time increment to gatherTime
+ inline static void addGatherTime()
+ {
+ addTime(GATHER);
+ }
+
+ //- Add time increment to scatterTime
+ inline static void addScatterTime()
+ {
+ addTime(SCATTER);
+ }
+
+ //- Add time increment to reduceTime
+ inline static void addReduceTime()
+ {
+ addTime(REDUCE);
+ }
+
+ //- Add time increment to waitTime
+ inline static void addWaitTime()
+ {
+ addTime(WAIT);
+ }
+
+ //- Add time increment to allToAllTime
+ inline static void addAllToAllTime()
+ {
+ addTime(ALL_TO_ALL);
+ }
+};
+
+
+// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
+
+} // End namespace Foam
+
+// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
+
+#endif
+
+// ************************************************************************* //
diff --git a/src/Pstream/mpi/UIPread.C b/src/Pstream/mpi/UIPread.C
index bd42449bd0..3a380fe5a8 100644
--- a/src/Pstream/mpi/UIPread.C
+++ b/src/Pstream/mpi/UIPread.C
@@ -2,7 +2,7 @@
========= |
\\ / F ield | OpenFOAM: The Open Source CFD Toolbox
\\ / O peration |
- \\ / A nd |
+ \\ / A nd | Copyright (C) 2019 OpenCFD Ltd.
\\/ M anipulation |
-------------------------------------------------------------------------------
| Copyright (C) 2011-2017 OpenFOAM Foundation
@@ -30,6 +30,7 @@ Description
#include "UIPstream.H"
#include "PstreamGlobals.H"
+#include "profilingPstream.H"
#include "IOstreams.H"
#include
@@ -85,6 +86,8 @@ Foam::UIPstream::UIPstream
// and set it
if (!wantedSize)
{
+ profilingPstream::beginTiming();
+
MPI_Probe
(
fromProcNo_,
@@ -94,6 +97,8 @@ Foam::UIPstream::UIPstream
);
MPI_Get_count(&status, MPI_BYTE, &messageSize_);
+ profilingPstream::addWaitTime();
+
externalBuf_.setCapacity(messageSize_);
wantedSize = messageSize_;
@@ -186,6 +191,8 @@ Foam::UIPstream::UIPstream(const int fromProcNo, PstreamBuffers& buffers)
// and set it
if (!wantedSize)
{
+ profilingPstream::beginTiming();
+
MPI_Probe
(
fromProcNo_,
@@ -195,6 +202,8 @@ Foam::UIPstream::UIPstream(const int fromProcNo, PstreamBuffers& buffers)
);
MPI_Get_count(&status, MPI_BYTE, &messageSize_);
+ profilingPstream::addWaitTime();
+
externalBuf_.setCapacity(messageSize_);
wantedSize = messageSize_;
@@ -257,6 +266,8 @@ Foam::label Foam::UIPstream::read
error::printStack(Pout);
}
+ profilingPstream::beginTiming();
+
if (commsType == commsTypes::blocking || commsType == commsTypes::scheduled)
{
MPI_Status status;
@@ -282,6 +293,7 @@ Foam::label Foam::UIPstream::read
return 0;
}
+ profilingPstream::addScatterTime();
// Check size of message read
@@ -332,6 +344,8 @@ Foam::label Foam::UIPstream::read
return 0;
}
+ profilingPstream::addScatterTime();
+
if (debug)
{
Pout<< "UIPstream::read : started read from:" << fromProcNo
@@ -346,15 +360,12 @@ Foam::label Foam::UIPstream::read
// Assume the message is completely received.
return bufSize;
}
- else
- {
- FatalErrorInFunction
- << "Unsupported communications type "
- << int(commsType)
- << Foam::abort(FatalError);
- return 0;
- }
+ FatalErrorInFunction
+ << "Unsupported communications type " << int(commsType)
+ << Foam::abort(FatalError);
+
+ return 0;
}
diff --git a/src/Pstream/mpi/UOPwrite.C b/src/Pstream/mpi/UOPwrite.C
index 7441c429f3..968a4753f9 100644
--- a/src/Pstream/mpi/UOPwrite.C
+++ b/src/Pstream/mpi/UOPwrite.C
@@ -2,7 +2,7 @@
========= |
\\ / F ield | OpenFOAM: The Open Source CFD Toolbox
\\ / O peration |
- \\ / A nd |
+ \\ / A nd | Copyright (C) 2019 OpenCFD Ltd.
\\/ M anipulation |
-------------------------------------------------------------------------------
| Copyright (C) 2011-2017 OpenFOAM Foundation
@@ -30,6 +30,7 @@ Description
#include "UOPstream.H"
#include "PstreamGlobals.H"
+#include "profilingPstream.H"
#include
@@ -70,6 +71,8 @@ bool Foam::UOPstream::write
bool transferFailed = true;
+ profilingPstream::beginTiming();
+
if (commsType == commsTypes::blocking)
{
transferFailed = MPI_Bsend
@@ -82,6 +85,8 @@ bool Foam::UOPstream::write
PstreamGlobals::MPICommunicators_[communicator] //MPI_COMM_WORLD
);
+ profilingPstream::addGatherTime();
+
if (debug)
{
Pout<< "UOPstream::write : finished write to:" << toProcNo
@@ -102,6 +107,8 @@ bool Foam::UOPstream::write
PstreamGlobals::MPICommunicators_[communicator] //MPI_COMM_WORLD
);
+ profilingPstream::addWaitTime();
+
if (debug)
{
Pout<< "UOPstream::write : finished write to:" << toProcNo
@@ -125,6 +132,8 @@ bool Foam::UOPstream::write
&request
);
+ profilingPstream::addWaitTime();
+
if (debug)
{
Pout<< "UOPstream::write : started write to:" << toProcNo
diff --git a/src/Pstream/mpi/UPstream.C b/src/Pstream/mpi/UPstream.C
index 428e21b00c..1ca6e5903a 100644
--- a/src/Pstream/mpi/UPstream.C
+++ b/src/Pstream/mpi/UPstream.C
@@ -28,6 +28,7 @@ License
#include "Pstream.H"
#include "PstreamReduceOps.H"
#include "PstreamGlobals.H"
+#include "profilingPstream.H"
#include "SubList.H"
#include "allReduce.H"
#include "int.H"
@@ -501,6 +502,8 @@ void Foam::UPstream::allToAll
}
else
{
+ profilingPstream::beginTiming();
+
if
(
MPI_Alltoall
@@ -522,6 +525,8 @@ void Foam::UPstream::allToAll
<< " on communicator " << communicator
<< Foam::abort(FatalError);
}
+
+ profilingPstream::addAllToAllTime();
}
}
@@ -572,6 +577,8 @@ void Foam::UPstream::allToAll
}
else
{
+ profilingPstream::beginTiming();
+
if
(
MPI_Alltoallv
@@ -594,6 +601,8 @@ void Foam::UPstream::allToAll
<< " communicator " << communicator
<< Foam::abort(FatalError);
}
+
+ profilingPstream::addAllToAllTime();
}
}
@@ -634,6 +643,8 @@ void Foam::UPstream::gather
}
else
{
+ profilingPstream::beginTiming();
+
if
(
MPI_Gatherv
@@ -656,6 +667,8 @@ void Foam::UPstream::gather
<< " communicator " << communicator
<< Foam::abort(FatalError);
}
+
+ profilingPstream::addGatherTime();
}
}
@@ -693,6 +706,8 @@ void Foam::UPstream::scatter
}
else
{
+ profilingPstream::beginTiming();
+
if
(
MPI_Scatterv
@@ -715,6 +730,8 @@ void Foam::UPstream::scatter
<< " communicator " << communicator
<< Foam::abort(FatalError);
}
+
+ profilingPstream::addScatterTime();
}
}
@@ -869,6 +886,8 @@ void Foam::UPstream::waitRequests(const label start)
start
);
+ profilingPstream::beginTiming();
+
if
(
MPI_Waitall
@@ -883,6 +902,8 @@ void Foam::UPstream::waitRequests(const label start)
<< "MPI_Waitall returned with error" << Foam::endl;
}
+ profilingPstream::addWaitTime();
+
resetRequests(start);
}
@@ -911,6 +932,8 @@ void Foam::UPstream::waitRequest(const label i)
<< Foam::abort(FatalError);
}
+ profilingPstream::beginTiming();
+
if
(
MPI_Wait
@@ -924,6 +947,8 @@ void Foam::UPstream::waitRequest(const label i)
<< "MPI_Wait returned with error" << Foam::endl;
}
+ profilingPstream::addWaitTime();
+
if (debug)
{
Pout<< "UPstream::waitRequest : finished wait for request:" << i
diff --git a/src/Pstream/mpi/allReduceTemplates.C b/src/Pstream/mpi/allReduceTemplates.C
index 35407b8f5a..9c4bdd4176 100644
--- a/src/Pstream/mpi/allReduceTemplates.C
+++ b/src/Pstream/mpi/allReduceTemplates.C
@@ -2,7 +2,7 @@
========= |
\\ / F ield | OpenFOAM: The Open Source CFD Toolbox
\\ / O peration |
- \\ / A nd |
+ \\ / A nd | Copyright (C) 2019 OpenCFD Ltd.
\\/ M anipulation |
-------------------------------------------------------------------------------
| Copyright (C) 2012-2015 OpenFOAM Foundation
@@ -26,6 +26,7 @@ License
\*---------------------------------------------------------------------------*/
#include "allReduce.H"
+#include "profilingPstream.H"
// * * * * * * * * * * * * * * * Global Functions * * * * * * * * * * * * * //
@@ -46,6 +47,8 @@ void Foam::allReduce
return;
}
+ profilingPstream::beginTiming();
+
if (UPstream::nProcs(communicator) <= UPstream::nProcsSimpleSum)
{
if (UPstream::master(communicator))
@@ -167,6 +170,8 @@ void Foam::allReduce
);
Value = sum;
}
+
+ profilingPstream::addReduceTime();
}
diff --git a/src/functionObjects/utilities/Make/files b/src/functionObjects/utilities/Make/files
index c64ef4e531..79c11e83e2 100644
--- a/src/functionObjects/utilities/Make/files
+++ b/src/functionObjects/utilities/Make/files
@@ -12,6 +12,8 @@ vtkWrite/vtkWriteUpdate.C
removeRegisteredObject/removeRegisteredObject.C
+parProfiling/parProfiling.C
+
solverInfo/solverInfo.C
runTimeControl/runTimeControl.C
diff --git a/src/functionObjects/utilities/parProfiling/parProfiling.C b/src/functionObjects/utilities/parProfiling/parProfiling.C
new file mode 100644
index 0000000000..ade16fb051
--- /dev/null
+++ b/src/functionObjects/utilities/parProfiling/parProfiling.C
@@ -0,0 +1,233 @@
+/*---------------------------------------------------------------------------*\
+ ========= |
+ \\ / F ield | OpenFOAM: The Open Source CFD Toolbox
+ \\ / O peration |
+ \\ / A nd | Copyright (C) 2019 OpenCFD Ltd.
+ \\/ M anipulation |
+-------------------------------------------------------------------------------
+License
+ This file is part of OpenFOAM.
+
+ OpenFOAM is free software: you can redistribute it and/or modify it
+ under the terms of the GNU General Public License as published by
+ the Free Software Foundation, either version 3 of the License, or
+ (at your option) any later version.
+
+ OpenFOAM is distributed in the hope that it will be useful, but WITHOUT
+ ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+ for more details.
+
+ You should have received a copy of the GNU General Public License
+ along with OpenFOAM. If not, see .
+
+\*---------------------------------------------------------------------------*/
+
+#include "parProfiling.H"
+#include "addToRunTimeSelectionTable.H"
+#include "UPstream.H"
+#include "Pstream.H"
+#include "PstreamReduceOps.H"
+#include "profilingPstream.H"
+#include "Tuple2.H"
+#include "FixedList.H"
+
+// * * * * * * * * * * * * * * Static Data Members * * * * * * * * * * * * * //
+
+namespace Foam
+{
+namespace functionObjects
+{
+ defineTypeNameAndDebug(parProfiling, 0);
+
+ addToRunTimeSelectionTable
+ (
+ functionObject,
+ parProfiling,
+ dictionary
+ );
+
+} // End namespace functionObject
+
+
+ // Processor and time for each of: -min -max -sum
+ typedef FixedList, 3> statData;
+
+
+ //- Reduction class. If x and y are not equal assign value.
+ struct statsEqOp
+ {
+ void operator()
+ (
+ FixedList& xStats,
+ const FixedList& yStats
+ ) const
+ {
+ forAll(xStats, i)
+ {
+ statData& x = xStats[i];
+ const statData& y = yStats[i];
+
+ // 0 : min
+ // 1 : max
+ // 2 : sum
+ if (x[0].second() > y[0].second())
+ {
+ x[0].second() = y[0].second();
+ x[0].first() = y[0].first();
+ }
+ if (x[1].second() < y[1].second())
+ {
+ x[1].second() = y[1].second();
+ x[1].first() = y[1].first();
+ }
+ x[2].second() += y[2].second();
+ x[2].first()++;
+ }
+ }
+ };
+
+} // End namespace Foam
+
+
+// * * * * * * * * * * * * * * * * Constructors * * * * * * * * * * * * * * //
+
+Foam::functionObjects::parProfiling::parProfiling
+(
+ const word& name,
+ const Time& runTime,
+ const dictionary& dict
+)
+:
+ functionObject(name)
+{
+ profilingPstream::enable();
+}
+
+
+// * * * * * * * * * * * * * * * * Destructor * * * * * * * * * * * * * * * //
+
+Foam::functionObjects::parProfiling::~parProfiling()
+{
+ profilingPstream::disable();
+}
+
+
+// * * * * * * * * * * * * * * * Member Functions * * * * * * * * * * * * * //
+
+void Foam::functionObjects::parProfiling::report()
+{
+ if (!profilingPstream::active())
+ {
+ return;
+ }
+
+ typedef FixedList, 3> statData;
+ FixedList times;
+
+ {
+ const scalar masterTime =
+ (
+ profilingPstream::times(profilingPstream::REDUCE)
+ + profilingPstream::times(profilingPstream::GATHER)
+ + profilingPstream::times(profilingPstream::SCATTER)
+ );
+
+ statData& reduceStats = times[0];
+
+ Tuple2