diff --git a/applications/test/00-dummy/library/mpi/Make/options b/applications/test/00-dummy/library/mpi/Make/options
index 2c788d6c92..2e6c4826b3 100644
--- a/applications/test/00-dummy/library/mpi/Make/options
+++ b/applications/test/00-dummy/library/mpi/Make/options
@@ -2,13 +2,7 @@
PROJECT_INC =
PROJECT_LIBS =
-sinclude $(GENERAL_RULES)/mplib$(WM_MPLIB)
-sinclude $(DEFAULT_RULES)/mplib$(WM_MPLIB)
-
-/* Default is PROJECT LIB/mpi target */
-ifeq (,$(FOAM_MPI_LIBBIN))
- FOAM_MPI_LIBBIN := $(FOAM_LIBBIN)/$(FOAM_MPI)
-endif
+include $(GENERAL_RULES)/mpi-rules
EXE_INC = \
-I.. \
diff --git a/applications/test/parallel-external-init/Make/options b/applications/test/parallel-external-init/Make/options
index e93d17dd54..b3a33caee1 100644
--- a/applications/test/parallel-external-init/Make/options
+++ b/applications/test/parallel-external-init/Make/options
@@ -1,5 +1,4 @@
-sinclude $(GENERAL_RULES)/mplib$(WM_MPLIB)
-sinclude $(DEFAULT_RULES)/mplib$(WM_MPLIB)
+include $(GENERAL_RULES)/mpi-rules
EXE_INC = $(PFLAGS) $(PINC) $(c++LESSWARN)
EXE_LIBS = $(PLIBS)
diff --git a/bin/tools/create-mpi-config b/bin/tools/create-mpi-config
index 561923f59d..2b65135ddb 100755
--- a/bin/tools/create-mpi-config
+++ b/bin/tools/create-mpi-config
@@ -41,7 +41,7 @@ printHelp() {
usage: ${0##*/} options
options:
- -dry-run Report but do not write config files
+ -dry-run, -n Report but do not write config files
-no-mpicc Bypass any use of mpicc
-query-openmpi Report installation directory for system openmpi
-write-openmpi Query system openmpi and write config files
@@ -75,7 +75,7 @@ die()
#------------------------------------------------------------------------------
# Options
unset optDryRun
-useMpicc=true
+withMpicc="mpicc"
# Get installation directory for system openmpi
# - from "mpicc --showme:link"
@@ -83,17 +83,19 @@ useMpicc=true
#
# The mpicc content looks like this:
# ----
-# -pthread -L/usr/lib64/mpi/gcc/openmpi/lib64 -lmpi
+# ... -L/usr/lib64/mpi/gcc/openmpi/lib64 -lmpi
+# ... -L/usr/lib64/mpi/gcc/openmpi2/lib64 -lmpi
+# ... -L/usr/lib64/openmpi/lib -lmpi
# ----
query_system_openmpi()
{
unset arch_path
- if [ "$useMpicc" = true ]
+ if [ -n "$withMpicc" ]
then
- arch_path=$(mpicc --showme:link 2>/dev/null | sed -e 's#^.*-L\([^ ]*\).*#\1#')
- arch_path="${arch_path%/*}"
+ arch_path=$(mpicc --showme:link 2>/dev/null | sed -e 's/^.*-L\([^ ]*\).*/\1/')
+ arch_path="${arch_path%/*}" # prefix from libdir
if [ -n "$arch_path" ]
then
@@ -112,14 +114,16 @@ query_system_openmpi()
fi
# Include is under /usr/lib... (eg, debian, openSUSE)
+ # Note this cannot handle (openmpi | openmpi1 | openmpi2 | ...) include directories
+ # unless we also try to grab information out of PATH or LD_LIBRARY_PATH
for testdir in \
- /usr/lib/"${DEB_TARGET_MULTIARCH:+${DEB_TARGET_MULTIARCH}/}"openmpi/include \
- /usr/lib64/mpi/gcc/openmpi/include \
+ /usr/lib/"${DEB_TARGET_MULTIARCH:+${DEB_TARGET_MULTIARCH}/}"openmpi \
+ /usr/lib64/mpi/gcc/openmpi \
;
do
- if [ -e "$testdir/mpi.h" ]
+ if [ -e "$testdir/include/mpi.h" ]
then
- echo "${testdir%/*}"
+ echo "$testdir"
return 0 # Clean exit
fi
done
@@ -137,6 +141,16 @@ query_system_openmpi()
fi
done
+ # Partial env from RedHat "module load mpi/openmpi-x86_64"
+ #
+ ## MPI_COMPILER=openmpi-x86_64
+ ## MPI_HOME=/usr/lib64/openmpi
+ ## MPI_BIN=/usr/lib64/openmpi/bin
+ ## MPI_LIB=/usr/lib64/openmpi/lib
+ ## MPI_INCLUDE=/usr/include/openmpi-x86_64
+ ## MPI_SUFFIX=_openmpi
+
+
# Failed (should not happen)
# - report '/usr', but with error code 2
echo "/usr"
@@ -151,60 +165,68 @@ create_files()
{
[ -n "$FOAM_MPI" ] || die "FOAM_MPI not set"
+ # MPI-name without trailing major version
+ mpiName="${FOAM_MPI%[0-9]}"
+
+ # The prefs name
+ prefsName="prefs.$mpiName"
+
if [ -d "$MPI_ARCH_PATH" ]
then
echo "Define $FOAM_MPI with $MPI_ARCH_PATH" 1>&2
- case "$FOAM_MPI" in
- (openmpi-system)
- configDir="etc/config.sh"
- if [ "$optDryRun" = true ]
+ case "$mpiName" in
+ (sys-openmpi | openmpi-system)
+
+ # POSIX shell
+ prefsFile="etc/config.sh/$prefsName"
+ if [ -d "${prefsFile%/*}" ] || [ -n "$optDryRun" ]
then
- cat << CONTENT 1>&2
-dry-run: $configDir/$FOAM_MPI
+ (
+ if [ -n "$optDryRun" ]
+ then
+ exec 1>&2
+ else
+ exec 1> "$prefsFile"
+ fi
+
+ echo "${optDryRun}Write $prefsFile" 1>&2
+ cat << CONTENT
+# $prefsFile
#
# Packaging configured value for $FOAM_MPI
-export MPI_ARCH_PATH="$MPI_ARCH_PATH"
-
-CONTENT
- elif [ -d "$configDir" ]
- then
- echo "Write $configDir/$FOAM_MPI" 1>&2
- cat << CONTENT > "$configDir/$FOAM_MPI"
-# $configDir/$FOAM_MPI
-#
-# Packaging configured value for $FOAM_MPI
-
export MPI_ARCH_PATH="$MPI_ARCH_PATH"
#----
CONTENT
+ )
else
- echo "Cannot write $configDir/$FOAM_MPI - no directory" 1>&2
+ echo "Cannot write $prefsFile - no directory" 1>&2
fi
- configDir="etc/config.csh"
- if [ "$optDryRun" = true ]
+
+ # C-shell
+ prefsFile="etc/config.csh/$prefsName"
+ if [ -d "${prefsFile%/*}" ] || [ -n "$optDryRun" ]
then
- cat << CONTENT 1>&2
-dry-run: $configDir/$FOAM_MPI
+ (
+ if [ -n "$optDryRun" ]
+ then
+ exec 1>&2
+ else
+ exec 1> "$prefsFile"
+ fi
+
+ echo "${optDryRun}Write $prefsFile" 1>&2
+ cat << CONTENT
+# $prefsFile
#
# Packaging configured value for $FOAM_MPI
-setenv MPI_ARCH_PATH "$MPI_ARCH_PATH"
-
-CONTENT
- elif [ -d "$configDir" ]
- then
- echo "Write $configDir/$FOAM_MPI" 1>&2
- cat << CONTENT > "$configDir/$FOAM_MPI"
-# $configDir/$FOAM_MPI
-#
-# Packaging configured value for $FOAM_MPI
-
setenv MPI_ARCH_PATH "$MPI_ARCH_PATH"
#----
CONTENT
+ )
else
- echo "Cannot write $configDir/$FOAM_MPI - no directory" 1>&2
+ echo "Cannot write $prefsFile - no directory" 1>&2
fi
;;
esac
@@ -212,10 +234,7 @@ CONTENT
echo "Warning: $FOAM_MPI with bad MPI_ARCH_PATH: $MPI_ARCH_PATH" 1>&2
# TBD - remove old/bad entries?
#
- # for file in "etc/config.sh/$FOAM_MPI" "etc/config.csh/$FOAM_MPI"
- # do
- # [ -f "$file" ] && rm -f "$file"
- # done
+ # rm -f "etc/config.sh/$prefsName" "etc/config.csh/$prefsName"
fi
}
@@ -226,19 +245,12 @@ CONTENT
while [ "$#" -gt 0 ]
do
case "$1" in
- -h | -help* | --help*)
- printHelp
- ;;
- '')
- # Discard empty arguments
- ;;
-
- -dry-run)
- optDryRun=true
- ;;
+ '') true ;; # Discard empty arguments
+ -h | -help* | --help*) printHelp ;;
+ -n | -dry-run) optDryRun="(dry-run) " ;;
-no-mpicc)
- unset useMpicc
+ unset withMpicc
;;
-query-openmpi | -query-system-openmpi)
@@ -249,8 +261,7 @@ do
-write-openmpi | -write-system-openmpi)
if MPI_ARCH_PATH=$(query_system_openmpi)
then
- FOAM_MPI="openmpi-system"
- create_files
+ FOAM_MPI="sys-openmpi" create_files
else
die "Failed query for system openmpi"
fi
diff --git a/etc/bashrc b/etc/bashrc
index d84355827d..ab79b9ad74 100644
--- a/etc/bashrc
+++ b/etc/bashrc
@@ -91,7 +91,8 @@ export WM_COMPILE_OPTION=Opt
# [WM_MPLIB] - MPI implementation:
# = SYSTEMOPENMPI | OPENMPI | SYSTEMMPI | MPI | MPICH | MPICH-GM |
# HPMPI | CRAY-MPICH | FJMPI | QSMPI | SGIMPI | INTELMPI | USERMPI
-# Also possible to use INTELMPI-xyz etc and define your own wmake rule
+# Specify SYSTEMOPENMPI1, SYSTEMOPENMPI2 for internal tracking (if desired)
+# Can also use INTELMPI-xyz etc and define your own wmake rule
export WM_MPLIB=SYSTEMOPENMPI
diff --git a/etc/config.csh/example/openmpi b/etc/config.csh/example/prefs.openmpi
similarity index 66%
rename from etc/config.csh/example/openmpi
rename to etc/config.csh/example/prefs.openmpi
index 1e9c1c040a..79cb43ccad 100644
--- a/etc/config.csh/example/openmpi
+++ b/etc/config.csh/example/prefs.openmpi
@@ -6,20 +6,22 @@
# \\/ M anipulation |
#------------------------------------------------------------------------------
# Copyright (C) 2011-2016 OpenFOAM Foundation
+# Copyright (C) 2020 OpenCFD Ltd.
#------------------------------------------------------------------------------
# License
-# This file is part of OpenFOAM, licensed under GNU General Public License
-# .
+# This file is part of OpenFOAM, distributed under GPL-3.0-or-later.
#
# File
-# config.csh/example/openmpi
-# - sourced by OpenFOAM-*/etc/config.csh/mpi
+# config.csh/example/prefs.openmpi
#
# Description
-# Example of fine tuning openmpi settings for OpenFOAM
+# Tuning example for openmpi (C-shell)
+#
+# Copy to etc/config.csh (or ~/.OpenFOAM/config.csh) and it will be
+# sourced by OpenFOAM-*/etc/config.csh/mpi
#
#------------------------------------------------------------------------------
-setenv FOAM_MPI openmpi-3.0.0
+setenv FOAM_MPI openmpi-4.0.5
#------------------------------------------------------------------------------
diff --git a/etc/config.csh/mpi b/etc/config.csh/mpi
index 76800e0868..b937beb4b7 100644
--- a/etc/config.csh/mpi
+++ b/etc/config.csh/mpi
@@ -18,314 +18,364 @@
# Description
# Setup for MPI communications library for OpenFOAM
#
-# User adjustments are possible in these files:
-# - config.csh/openmpi-system
-# - config.csh/openmpi
-# - config.csh/mpi-user
-# - config.csh/mpi-system
-#
# For USERMPI, the user is responsible for supplying an appropriate
# wmake/rules/General/mplibUSERMPI file and managing all settings
#
# For INTELMPI, note that the (I_MPI_CC, I_MPI_CCX) environment variables
# define the underlying compiler to be used
#
+# User adjustments are controlled by these types of files:
+# - config.csh/prefs.intelmpi
+# - config.csh/prefs.mpich
+# - config.csh/prefs.mpich-gm
+# - config.csh/prefs.mvapich2
+# - config.csh/prefs.openmpi
+# - config.csh/prefs.sys-mpi
+# - config.csh/prefs.sys-openmpi
+# - config.csh/prefs.user-mpi
+#
+# Note
+# All pre-v2012 names (without the 'prefs.' prefix) are ignored.
+#
+# User adjustments may set MPI_ARCH_PATH to an empty string,
+# but are not permitted to unset it.
+#
+# Environment
+# MPI_BUFFER_SIZE overrides 'mpiBufferSize' (controlDict entry).
+# Eg, setenv MPI_BUFFER_SIZE 20000000
+#
#------------------------------------------------------------------------------
-unsetenv MPI_ARCH_PATH MPI_HOME
+unsetenv MPI_HOME
setenv FOAM_MPI dummy # Fallback value
+setenv MPI_ARCH_PATH # Empty, but defined
+set _foamMpiVersion # Track mpi (major) version as required
+
+# Location for ThirdParty installations
+set _foamMpiPrefixDir="$WM_THIRD_PARTY_DIR/platforms/$WM_ARCH$WM_COMPILER"
+set _foamMpiWarning="openfoam (etc/config.csh/mpi) => " # Prefix for warnings
+
+#------------------------------------------------------------------------------
switch ("$WM_MPLIB")
-case SYSTEMOPENMPI*:
- # The system installed openmpi, locations discovery via mpicc.
- setenv FOAM_MPI openmpi-system
- _foamEtc -config openmpi-system # <- Adjustments (optional)
- # Respect MPI_ARCH_PATH if set to valid directory (eg, user adjustments)
- if (! $?MPI_ARCH_PATH ) setenv MPI_ARCH_PATH
+# The system openmpi, discover locations via
+case SYSTEMOPENMPI[1-9]:
+ # Preserve major version for the naming
+ set _foamMpiVersion=`echo "$WM_MPLIB" | sed -e 's/^.*MPI//'`
+ #[fallthrough]
+
+case SYSTEMOPENMPI:
+ setenv FOAM_MPI "sys-openmpi${_foamMpiVersion}"
+ _foamEtc -config prefs.sys-openmpi ## Optional adjustments
+
+ # MPI_ARCH_PATH (prefs) if a valid dir, or discover via
if ( -d "$MPI_ARCH_PATH" ) then
- _foamAddLibAuto $MPI_ARCH_PATH
+ _foamAddLibAuto "$MPI_ARCH_PATH"
else
- # Slight hack: strip off 'lib' to get presumed prefix for include and libs
+ # Slight hack: strip off 'lib' to get prefix directory
set libDir=`mpicc --showme:link | sed -e 's/.*-L\([^ ]*\).*/\1/'`
- setenv MPI_ARCH_PATH "${libDir:h}"
- _foamAddLib $libDir
+ setenv MPI_ARCH_PATH "${libDir:h}" # prefix from libdir
+ _foamAddLib "$libDir"
unset libDir
endif
+
+ # TDB: extra openmpi major qualifier on name?
+ ## if ( "$FOAM_MPI" == "sys-openmpi" ) then
+ ## set _foamMpiVersion=`echo "$MPI_ARCH_PATH" | sed -e 's/^.*mpi//'`
+ ## switch ("$_foamMpiVersion")
+ ## case [1-9]:
+ ## setenv FOAM_MPI "${FOAM_MPI}${_foamMpiVersion}"
+ ## breaksw
+ ## endsw
+ ## endif
breaksw
+
case OPENMPI:
setenv FOAM_MPI openmpi-4.0.3
- _foamEtc -config openmpi # <- Adjustments (optional)
+ _foamEtc -config prefs.openmpi ## Optional adjustments
- # Respect MPI_ARCH_PATH if set to valid directory (eg, user adjustments)
- if (! $?MPI_ARCH_PATH ) setenv MPI_ARCH_PATH
- if (! -d "$MPI_ARCH_PATH" ) then
- setenv MPI_ARCH_PATH $WM_THIRD_PARTY_DIR/platforms/$WM_ARCH$WM_COMPILER/$FOAM_MPI
+ # MPI_ARCH_PATH (prefs) if a valid dir, or ThirdParty location
+ if ( ! -d "$MPI_ARCH_PATH" ) then
+ setenv MPI_ARCH_PATH "$_foamMpiPrefixDir/$FOAM_MPI"
endif
# Inform openmpi where to find its install directory
- setenv OPAL_PREFIX $MPI_ARCH_PATH
+ setenv OPAL_PREFIX "$MPI_ARCH_PATH"
- if ($?FOAM_VERBOSE && $?prompt) then
- echo "Using $WM_MPLIB"
- echo " FOAM_MPI : $FOAM_MPI"
- echo " OPAL_PREFIX : $MPI_ARCH_PATH"
- endif
-
- _foamAddPath $MPI_ARCH_PATH/bin
- _foamAddMan $MPI_ARCH_PATH/share/man
- _foamAddLibAuto $MPI_ARCH_PATH lib$WM_COMPILER_LIB_ARCH
+ _foamAddPath "$MPI_ARCH_PATH"/bin
+ _foamAddMan "$MPI_ARCH_PATH"/share/man
+ _foamAddLibAuto "$MPI_ARCH_PATH" "lib$WM_COMPILER_LIB_ARCH"
breaksw
+
+# Arbitrary, user-specified mpi implementation
+# - used by spack, which populates appropriate wmake rules
case USERMPI:
- # Arbitrary, user-specified mpi implementation
- setenv FOAM_MPI mpi-user
- _foamEtc -config mpi-user # <- Adjustments (optional)
+ setenv FOAM_MPI user-mpi
+ _foamEtc -config prefs.user-mpi ## Optional adjustments
breaksw
+
+# EasyBuild handles everything via
+case EASYBUILDMPI:
+ setenv FOAM_MPI eb-mpi
+ breaksw
+
+
case SYSTEMMPI:
- setenv FOAM_MPI mpi-system
- _foamEtc -config mpi-system # <- Adjustments (optional)
+ setenv FOAM_MPI sys-mpi
+ _foamEtc -config prefs.sys-mpi ## Optional adjustments
- if ( ! $?MPI_ROOT ) then
- echo
- echo "Warning in $WM_PROJECT_DIR/etc/config.csh/mpi:"
- echo " Please set the environment variable MPI_ROOT to point to" \
- " the base folder for the system MPI in use."
- echo " Example:"
- echo
- echo " setenv MPI_ROOT /opt/mpi"
+ # MPI_ARCH_PATH (prefs) or MPI_ROOT (inherited), in that order
+ if ( "$MPI_ARCH_PATH" != "" ) then
+ setenv MPI_ROOT "$MPI_ARCH_PATH"
+ else if ( $?MPI_ROOT ) then
+ setenv MPI_ARCH_PATH "$MPI_ROOT"
+ endif
+
+ set _foamMpiWarning="${_foamMpiWarning}system MPI: "
+ if ( ! -d "$MPI_ARCH_PATH" ) then
+ echo "${_foamMpiWarning}has invalid MPI_ROOT"
+ echo "Please set correctly. For example,"
+ echo ' setenv MPI_ROOT "/opt/mpi"'
echo
else
- setenv MPI_ARCH_PATH $MPI_ROOT
+ set _foamMpiWarning="${_foamMpiWarning}has unset "
if ( ! $?MPI_ARCH_FLAGS ) then
- echo
- echo "Warning in $WM_PROJECT_DIR/etc/config.csh/mpi:"
- echo " MPI_ARCH_FLAGS is not set. Example:"
- echo
- echo ' setenv MPI_ARCH_FLAGS "-DOMPI_SKIP_MPICXX"'
+ echo "${_foamMpiWarning}MPI_ARCH_FLAGS"
+ echo "Please set correctly. For example,"
+ echo ' setenv MPI_ARCH_FLAGS "-DOMPI_SKIP_MPICXX"'
echo
endif
-
if ( ! $?MPI_ARCH_INC ) then
- echo
- echo "Warning in $WM_PROJECT_DIR/etc/config.csh/mpi:"
- echo " MPI_ARCH_INC is not set. Example:"
- echo
- echo ' setenv MPI_ARCH_INC "-isystem $MPI_ROOT/include"'
+ echo "${_foamMpiWarning}MPI_ARCH_INC"
+ echo "Please set correctly. For example,"
+ echo ' setenv MPI_ARCH_INC "-isystem $MPI_ROOT/include"'
echo
endif
-
if ( ! $?MPI_ARCH_LIBS ) then
- echo
- echo "Warning in $WM_PROJECT_DIR/etc/config.csh/mpi:"
- echo " MPI_ARCH_LIBS is not set. Example:"
- echo
- echo ' setenv MPI_ARCH_LIBS "-L$MPI_ROOT/lib -lmpi"'
+ echo "${_foamMpiWarning}MPI_ARCH_LIBS"
+ echo "Please set correctly. For example,"
+ echo ' setenv MPI_ARCH_LIBS "-L$MPI_ROOT/lib -lmpi"'
echo
endif
endif
breaksw
+
case MPICH:
setenv FOAM_MPI mpich-3.3
- setenv MPI_ARCH_PATH $WM_THIRD_PARTY_DIR/platforms/$WM_ARCH$WM_COMPILER/$FOAM_MPI
- setenv MPI_HOME $MPI_ARCH_PATH
+ _foamEtc -config prefs.mpich ## Optional adjustments
- _foamAddPath $MPI_ARCH_PATH/bin
- _foamAddMan $MPI_ARCH_PATH/share/man
- _foamAddLibAuto $MPI_ARCH_PATH lib$WM_COMPILER_LIB_ARCH
+ # MPI_ARCH_PATH (prefs) if a valid dir, or use ThirdParty location
+ if ( ! -d "$MPI_ARCH_PATH" ) then
+ setenv MPI_ARCH_PATH "$_foamMpiPrefixDir/$FOAM_MPI"
+ endif
+
+ setenv MPI_HOME "$MPI_ARCH_PATH"
+
+ _foamAddPath "$MPI_ARCH_PATH"/bin
+ _foamAddMan "$MPI_ARCH_PATH"/share/man
+ _foamAddLibAuto "$MPI_ARCH_PATH" "lib$WM_COMPILER_LIB_ARCH"
breaksw
+
case MPICH-GM:
setenv FOAM_MPI mpich-gm
- setenv MPI_ARCH_PATH /opt/mpi
- setenv MPICH_PATH $MPI_ARCH_PATH
- setenv GM_LIB_PATH /opt/gm/lib$WM_COMPILER_LIB_ARCH
+ _foamEtc -config prefs.mpich-gm ## Optional adjustments
- _foamAddPath $MPI_ARCH_PATH/bin
- _foamAddLib $MPI_ARCH_PATH/lib$WM_COMPILER_LIB_ARCH
- _foamAddLib $GM_LIB_PATH
+ # MPI_ARCH_PATH (prefs) if a valid dir, or system location
+ if ( ! -d "$MPI_ARCH_PATH" ) then
+ setenv MPI_ARCH_PATH /opt/mpi
+ endif
+
+ setenv MPICH_PATH "$MPI_ARCH_PATH"
+ setenv GM_LIB_PATH "$MPICH_PATH/lib$WM_COMPILER_LIB_ARCH"
+
+ _foamAddPath "$MPI_ARCH_PATH"/bin
+ _foamAddLib "$MPI_ARCH_PATH/lib$WM_COMPILER_LIB_ARCH"
+ _foamAddLib "$GM_LIB_PATH"
breaksw
-case MVA2MPI: # MVAMPICH2
+
+# MVAPICH2, renamed to contain 'MPI' in its name
+case MVA2MPI:
setenv FOAM_MPI mvapich2-2.3
+ _foamEtc -config prefs.mvapich2 ## Optional adjustments
- _foamAddPath $MPI_ARCH_PATH/bin
- _foamAddMan $MPI_ARCH_PATH/share/man
- _foamAddLibAuto $MPI_ARCH_PATH lib$WM_COMPILER_LIB_ARCH
+ # MPI_ARCH_PATH (prefs) if a valid dir, or ThirdParty location
+ if ( ! -d "$MPI_ARCH_PATH" ) then
+ setenv MPI_ARCH_PATH "$_foamMpiPrefixDir/$FOAM_MPI"
+ endif
+
+ _foamAddPath "$MPI_ARCH_PATH"/bin
+ _foamAddMan "$MPI_ARCH_PATH"/share/man
+ _foamAddLibAuto "$MPI_ARCH_PATH" "lib$WM_COMPILER_LIB_ARCH"
breaksw
+
case CRAY-MPICH:
- if ( ! $?MPICH_DIR ) setenv MPICH_DIR /dev/null
setenv FOAM_MPI cray-mpich
- setenv MPI_ARCH_PATH $MPICH_DIR
-
- if ($?FOAM_VERBOSE && $?prompt) then
- echo "Using $WM_MPLIB"
- echo " FOAM_MPI : $FOAM_MPI"
- echo " MPICH_DIR : $MPI_ARCH_PATH"
- endif
+ if ( ! $?MPICH_DIR ) setenv MPICH_DIR /dummy
+ setenv MPI_ARCH_PATH "$MPICH_DIR"
if ( ! -d "$MPI_ARCH_PATH" ) then
- echo "Warning in $WM_PROJECT_DIR/etc/config.csh/mpi:"
- echo " Not a valid $WM_MPLIB installation directory."
- echo " Please set MPICH_DIR properly."
- echo " Currently using '$MPI_ARCH_PATH'"
+ echo "%{_foamMpiWarning}invalid $WM_MPLIB directory" 1>&2
+ echo " => $MPI_ARCH_PATH"
+ echo "Please set MPICH_DIR correctly"
endif
- # _foamAddPath $MPI_ARCH_PATH/bin
- _foamAddLib $MPI_ARCH_PATH/lib
+ # _foamAddPath "$MPI_ARCH_PATH"/bin
+ _foamAddLib "$MPI_ARCH_PATH"/lib
breaksw
+
case HPMPI:
setenv FOAM_MPI hpmpi
setenv MPI_HOME /opt/hpmpi
- setenv MPI_ARCH_PATH $MPI_HOME
-
- _foamAddPath $MPI_ARCH_PATH/bin
+ setenv MPI_ARCH_PATH "$MPI_HOME"
+ set libDir=""
switch (`uname -m`)
- case i686:
- _foamAddLib $MPI_ARCH_PATH/lib/linux_ia32
- breaksw
case x86_64:
- _foamAddLib $MPI_ARCH_PATH/lib/linux_amd64
+ set libDir=lib/linux_amd64
+ breaksw
+ case i686:
+ set libDir=lib/linux_ia32
breaksw
case ia64:
- _foamAddLib $MPI_ARCH_PATH/lib/linux_ia64
- breaksw
- default:
- echo Unknown processor type `uname -m` for Linux
+ set libDir=lib/linux_ia64
breaksw
endsw
+
+ if ( "${libDir}" != "" ) then
+ _foamAddPath "$MPI_ARCH_PATH"/bin
+ _foamAddLib "$MPI_ARCH_PATH/$libDir"
+ else
+ echo "openfoam: (`uname -m`) - unsupported HP-MPI processor type"
+ endif
+ unset libDir
breaksw
+
case MPI:
setenv FOAM_MPI mpi
setenv MPI_ARCH_PATH /opt/mpi
breaksw
+
case FJMPI:
setenv FOAM_MPI fjmpi
setenv MPI_ARCH_PATH /opt/FJSVmpi2
- _foamAddPath $MPI_ARCH_PATH/bin
- _foamAddLib $MPI_ARCH_PATH/lib/sparcv9
+ _foamAddPath "$MPI_ARCH_PATH"/bin
+ _foamAddLib "$MPI_ARCH_PATH"/lib/sparcv9
_foamAddLib /opt/FSUNf90/lib/sparcv9
_foamAddLib /opt/FJSVpnidt/lib
breaksw
+
case QSMPI:
setenv FOAM_MPI qsmpi
setenv MPI_ARCH_PATH /usr/lib/mpi
- _foamAddPath $MPI_ARCH_PATH/bin
- _foamAddLib $MPI_ARCH_PATH/lib
+ _foamAddPath "$MPI_ARCH_PATH"/bin
+ _foamAddLib "$MPI_ARCH_PATH"/lib
breaksw
+
case SGIMPI:
if ( ! $?MPI_ROOT ) setenv MPI_ROOT /dummy
-
- if ( ! -d "$MPI_ROOT" ) then
- echo "Warning in $WM_PROJECT_DIR/etc/config.csh/mpi:"
- echo " Not a valid $WM_MPLIB installation directory."
- echo " Please set MPI_ROOT properly (usually via the mpt module)"
- echo " Currently using '$MPI_ARCH_PATH'"
+ if ( "${MPI_ROOT:h}/" == "$MPI_ROOT" ) then
+ setenv MPI_ROOT "${MPI_ROOT:h}" # Remove trailing slash
endif
- if ( "${MPI_ROOT:h}/" == $MPI_ROOT ) then
- setenv MPI_ROOT ${MPI_ROOT:h}
- endif
-
- setenv FOAM_MPI ${MPI_ROOT:t}
- setenv MPI_ARCH_PATH $MPI_ROOT
-
- if ($?FOAM_VERBOSE && $?prompt) then
- echo "Using $WM_MPLIB"
- echo " FOAM_MPI : $FOAM_MPI"
- echo " MPI_ROOT : $MPI_ARCH_PATH"
- endif
-
- _foamAddPath $MPI_ARCH_PATH/bin
- _foamAddLib $MPI_ARCH_PATH/lib
- breaksw
-
-case INTELMPI*:
- if ( $?I_MPI_ROOT ) then
- # I_MPI_ROOT: The Intel MPI Library installation directory
-
- # Remove trailing slash
- if ( "${I_MPI_ROOT:h}/" == $I_MPI_ROOT ) then
- setenv I_MPI_ROOT ${I_MPI_ROOT:h}
- endif
- setenv MPI_ARCH_PATH $I_MPI_ROOT
- setenv FOAM_MPI ${MPI_ARCH_PATH:t}
-
- # If subdirectory is version number only, prefix with 'impi-'
- switch ("$FOAM_MPI")
- case [0-9]*:
- setenv FOAM_MPI "impi-$FOAM_MPI"
- breaksw
- endsw
-
- if ($?FOAM_VERBOSE && $?prompt) then
- echo "Using $WM_MPLIB"
- echo " FOAM_MPI : $FOAM_MPI"
- echo " I_MPI_ROOT : $MPI_ARCH_PATH"
- endif
- else if ( $?MPI_ROOT ) then
- # MPI_ROOT: General specification
-
- # Remove trailing slash
- if ( "${MPI_ROOT:h}/" == $MPI_ROOT ) then
- setenv MPI_ROOT ${MPI_ROOT:h}
- endif
-
- setenv MPI_ARCH_PATH $MPI_ROOT
- setenv FOAM_MPI ${MPI_ARCH_PATH:t}
-
- # If subdirectory is version number only, prefix with 'impi-'
- switch ("$FOAM_MPI")
- case [0-9]*:
- setenv FOAM_MPI "impi-$FOAM_MPI"
- breaksw
- endsw
-
- if ($?FOAM_VERBOSE && $?prompt) then
- echo "Using $WM_MPLIB"
- echo " FOAM_MPI : $FOAM_MPI"
- echo " MPI_ROOT : $MPI_ARCH_PATH"
- endif
- else
- setenv MPI_ARCH_PATH /dummy
- endif
+ setenv FOAM_MPI "${MPI_ROOT:t}"
+ setenv MPI_ARCH_PATH "$MPI_ROOT"
if ( ! -d "$MPI_ARCH_PATH" ) then
- echo "Warning in $WM_PROJECT_DIR/etc/config.csh/mpi:"
- echo " Not a valid $WM_MPLIB installation directory."
- echo " Please set I_MPI_ROOT or MPI_ROOT properly."
- echo " Currently using '$MPI_ARCH_PATH'"
+ echo "%{_foamMpiWarning}invalid $WM_MPLIB directory" 1>&2
+ echo " => $MPI_ARCH_PATH"
+ echo "Please set MPI_ROOT correctly"
endif
- _foamAddPath "$MPI_ARCH_PATH/intel64/bin"
- _foamAddLib "$MPI_ARCH_PATH/intel64/lib"
+ _foamAddPath "$MPI_ARCH_PATH"/bin
+ _foamAddLib "$MPI_ARCH_PATH"/lib
breaksw
+
+
+case INTELMPI*:
+ _foamEtc -config prefs.intelmpi ## Optional adjustments
+
+ if ( $?I_MPI_ROOT ) then
+ # I_MPI_ROOT: Intel MPI Library installation directory
+ setenv MPI_ARCH_PATH "${I_MPI_ROOT}"
+ else if ( $?MPI_ROOT ) then
+ # MPI_ROOT: General specification
+ setenv MPI_ARCH_PATH "$MPI_ROOT"
+ else if ( "$MPI_ARCH_PATH" != "" ) then
+ # MPI_ARCH_PATH: Set I_MPI_ROOT accordingly
+ export I_MPI_ROOT="$MPI_ARCH_PATH"
+
+ endif
+
+ if ( -d "$MPI_ARCH_PATH" ) then
+ # Remove trailing slash
+ if ( "${MPI_ARCH_PATH:h}/" == "$MPI_ARCH_PATH" ) then
+ setenv MPI_ARCH_PATH "${MPI_ARCH_PATH:h}"
+ endif
+
+ # Subdir name is FOAM_MPI name
+ setenv FOAM_MPI "${MPI_ARCH_PATH:t}"
+
+ # If subdir is version number only, prefix with 'impi-'
+ switch ("$FOAM_MPI")
+ case [0-9]*:
+ setenv FOAM_MPI "impi-$FOAM_MPI"
+ breaksw
+ endsw
+
+ else
+ echo "${_foamMpiWarning}invalid $WM_MPLIB directory" 1>&2
+ echo " => ${MPI_ARCH_PATH}" 1>&2
+ echo "Please set I_MPI_ROOT or MPI_ROOT correctly."
+ endif
+
+ _foamAddPath "$MPI_ARCH_PATH"/intel64/bin
+ _foamAddLib "$MPI_ARCH_PATH"/intel64/lib
+ breaksw
+
endsw
-# Add (non-dummy) MPI implementation
-# - dummy MPI already in lib-path and has no external libraries
+#------------------------------------------------------------------------------
+# Final
+
+# A real (non-dummy) MPI
if ( "$FOAM_MPI" != dummy ) then
- if ( $?FOAM_EXT_LIBBIN ) then # External libraries (allowed to be unset)
- _foamAddLib ${FOAM_EXT_LIBBIN}/${FOAM_MPI}
+ if ($?FOAM_VERBOSE && $?prompt) then
+ echo "Using mpi type : $WM_MPLIB"
+ echo " FOAM_MPI : $FOAM_MPI"
+ if ( "${MPI_ARCH_PATH}" == "" ) then
+ echo " prefix : unset"
+ else
+ echo " prefix : $MPI_ARCH_PATH"
+ endif
endif
- _foamAddLib ${FOAM_LIBBIN}/${FOAM_MPI}
+
+ # OpenFOAM library interface to (non-dummy) MPI
+ if ( $?FOAM_EXT_LIBBIN ) then
+ # External libraries (can be unset)
+ _foamAddLib "${FOAM_EXT_LIBBIN}/${FOAM_MPI}"
+ endif
+ _foamAddLib "${FOAM_LIBBIN}/${FOAM_MPI}"
endif
-# Alternative to 'mpiBufferSize' controlDict entry
-#setenv MPI_BUFFER_SIZE 20000000
+
+# Cleanup
+unset _foamMpiPrefixDir _foamMpiWarning _foamMpiVersion
#------------------------------------------------------------------------------
diff --git a/etc/config.sh/example/openmpi b/etc/config.sh/example/prefs.openmpi
similarity index 66%
rename from etc/config.sh/example/openmpi
rename to etc/config.sh/example/prefs.openmpi
index 3456c8cd47..d68f4d2c86 100644
--- a/etc/config.sh/example/openmpi
+++ b/etc/config.sh/example/prefs.openmpi
@@ -6,20 +6,22 @@
# \\/ M anipulation |
#------------------------------------------------------------------------------
# Copyright (C) 2011-2016 OpenFOAM Foundation
+# Copyright (C) 2020 OpenCFD Ltd.
#------------------------------------------------------------------------------
# License
-# This file is part of OpenFOAM, licensed under GNU General Public License
-# .
+# This file is part of OpenFOAM, distributed under GPL-3.0-or-later.
#
# File
-# config.sh/example/openmpi
-# - sourced by OpenFOAM-*/etc/config.sh/mpi
+# config.sh/example/prefs.openmpi
#
# Description
-# Example of fine tuning openmpi settings for OpenFOAM
+# Tuning example for openmpi (POSIX shell).
+#
+# Copy to etc/config.sh (or ~/.OpenFOAM/config.sh) and it will be
+# sourced by OpenFOAM-*/etc/config.sh/mpi
#
#------------------------------------------------------------------------------
-export FOAM_MPI=openmpi-3.0.0
+export FOAM_MPI=openmpi-4.0.5
#------------------------------------------------------------------------------
diff --git a/etc/config.sh/mpi b/etc/config.sh/mpi
index 7e515bcfbf..a015971132 100644
--- a/etc/config.sh/mpi
+++ b/etc/config.sh/mpi
@@ -18,338 +18,404 @@
# Description
# Setup for MPI communications library for OpenFOAM
#
-# User adjustments are possible in these files:
-# - config.sh/openmpi-system
-# - config.sh/openmpi
-# - config.sh/mpi-user
-# - config.sh/mpi-system
-#
# For USERMPI, the user is responsible for supplying an appropriate
# wmake/rules/General/mplibUSERMPI file and managing all settings
#
# For INTELMPI, note that the (I_MPI_CC, I_MPI_CCX) environment variables
# define the underlying compiler to be used
#
+# User adjustments are controlled by these types of files:
+# - config.sh/prefs.intelmpi
+# - config.sh/prefs.mpich
+# - config.sh/prefs.mpich-gm
+# - config.sh/prefs.msmpi
+# - config.sh/prefs.mvapich2
+# - config.sh/prefs.openmpi
+# - config.sh/prefs.sys-mpi
+# - config.sh/prefs.sys-openmpi
+# - config.sh/prefs.user-mpi
+#
+# Note
+# All pre-v2012 names (without the 'prefs.' prefix) are ignored.
+#
+# Environment
+# MPI_BUFFER_SIZE overrides 'mpiBufferSize' (controlDict entry).
+# Eg, export MPI_BUFFER_SIZE=20000000
+#
#------------------------------------------------------------------------------
unset MPI_ARCH_PATH MPI_HOME
export FOAM_MPI=dummy # Fallback value
+unset _foamMpiVersion # Track mpi (major) version as required
+
+# Location for ThirdParty installations
+_foamMpiPrefixDir="$WM_THIRD_PARTY_DIR/platforms/$WM_ARCH$WM_COMPILER"
+_foamMpiWarning="openfoam (etc/config.sh/mpi) => " # Prefix for warnings
+
+#------------------------------------------------------------------------------
case "$WM_MPLIB" in
-SYSTEMOPENMPI*)
- # The system installed openmpi, locations discovery via mpicc.
- export FOAM_MPI=openmpi-system
+
+# The system openmpi, discover locations via
+# Preserve major version for the naming
+SYSTEMOPENMPI | SYSTEMOPENMPI[1-9])
+ # Preserve major version for the naming
+ _foamMpiVersion="${WM_MPLIB##*MPI}"
+
+ export FOAM_MPI="sys-openmpi${_foamMpiVersion}"
# Undefine OPAL_PREFIX if set to one of the paths on foamOldDirs
if [ -z "$($foamClean -env=OPAL_PREFIX "$foamOldDirs")" ]
then
unset OPAL_PREFIX
fi
- _foamEtc -config openmpi-system # <- Adjustments (optional)
+ _foamEtc -config prefs.sys-openmpi ## Optional adjustments
- # Respect MPI_ARCH_PATH if set to valid directory (eg, user adjustments)
+ # MPI_ARCH_PATH (prefs) if a valid dir, or discover via
if [ -d "$MPI_ARCH_PATH" ]
then
- _foamAddLibAuto $MPI_ARCH_PATH
+ _foamAddLibAuto "$MPI_ARCH_PATH"
else
- # Slight hack: strip off 'lib' to get presumed prefix for include and libs
+ # Slight hack: strip off 'lib' to get prefix directory
libDir=$(mpicc --showme:link | sed -e 's/.*-L\([^ ]*\).*/\1/')
- export MPI_ARCH_PATH="${libDir%/*}"
- _foamAddLib $libDir
+ export MPI_ARCH_PATH="${libDir%/*}" # prefix from libdir
+ _foamAddLib "$libDir"
unset libDir
fi
+
+ # TDB: extra openmpi major qualifier on name?
+ ## if [ "$FOAM_MPI" = "sys-openmpi" ]
+ ## then
+ ## _foamMpiVersion="${MPI_ARCH_PATH##*mpi}"
+ ## case "$_foamMpiVersion" in
+ ## ([1-9])
+ ## FOAM_MPI="${FOAM_MPI}${_foamMpiVersion}"
+ ## ;;
+ ## esac
+ ## fi
;;
+
OPENMPI)
export FOAM_MPI=openmpi-4.0.3
- _foamEtc -config openmpi # <- Adjustments (optional)
+ _foamEtc -config prefs.openmpi ## Optional adjustments
- # Respect MPI_ARCH_PATH if set to valid directory (eg, user adjustments)
+ # MPI_ARCH_PATH (prefs) if a valid dir, or ThirdParty location
if [ ! -d "$MPI_ARCH_PATH" ]
then
- export MPI_ARCH_PATH=$WM_THIRD_PARTY_DIR/platforms/$WM_ARCH$WM_COMPILER/$FOAM_MPI
+ export MPI_ARCH_PATH="$_foamMpiPrefixDir/$FOAM_MPI"
fi
# Inform openmpi where to find its install directory
- export OPAL_PREFIX=$MPI_ARCH_PATH
-
- if [ -n "$FOAM_VERBOSE" ] && [ -n "$PS1" ]
- then
- echo "Using $WM_MPLIB" 1>&2
- echo " FOAM_MPI : $FOAM_MPI" 1>&2
- echo " OPAL_PREFIX : $MPI_ARCH_PATH" 1>&2
- fi
+ export OPAL_PREFIX="$MPI_ARCH_PATH"
# Could be sourced from ThirdParty with incomplete environment
if command -v _foamAddLibAuto >/dev/null # Normal sourcing
then
- _foamAddPath $MPI_ARCH_PATH/bin
- _foamAddMan $MPI_ARCH_PATH/share/man
- _foamAddLibAuto $MPI_ARCH_PATH lib$WM_COMPILER_LIB_ARCH
+ _foamAddPath "$MPI_ARCH_PATH"/bin
+ _foamAddMan "$MPI_ARCH_PATH"/share/man
+ _foamAddLibAuto "$MPI_ARCH_PATH" "lib$WM_COMPILER_LIB_ARCH"
fi
;;
+
+# Arbitrary, user-specified mpi implementation
+# - used by spack, which populates appropriate wmake rules
USERMPI)
- # Arbitrary, user-specified mpi implementation
- export FOAM_MPI=mpi-user
- _foamEtc -config mpi-user # <- Adjustments (optional)
+ export FOAM_MPI=user-mpi
+ _foamEtc -config prefs.user-mpi ## Optional adjustments
;;
+
+# EasyBuild handles everything via
+EASYBUILDMPI)
+ export FOAM_MPI=eb-mpi
+ ;;
+
+
SYSTEMMPI)
- export FOAM_MPI=mpi-system
- _foamEtc -config mpi-system # <- Adjustments (optional)
+ export FOAM_MPI=sys-mpi
+ _foamEtc -config prefs.sys-mpi ## Optional adjustments
- if [ -z "$MPI_ROOT" ]
+ # MPI_ARCH_PATH (prefs) or MPI_ROOT (inherited), in that order
+ if [ -n "$MPI_ARCH_PATH" ]
then
- echo 1>&2
- echo "Warning in $WM_PROJECT_DIR/etc/config.sh/mpi:" 1>&2
- echo " Please set the environment variable MPI_ROOT to point to" \
- " the base folder for the system MPI in use." 1>&2
- echo " Example:" 1>&2
- echo 1>&2
- echo " export MPI_ROOT=/opt/mpi" 1>&2
+ export MPI_ROOT="$MPI_ARCH_PATH"
+ elif [ -n "$MPI_ROOT" ]
+ then
+ export MPI_ARCH_PATH="$MPI_ROOT"
+ fi
+
+ _foamMpiWarning="${_foamMpiWarning}system MPI: "
+ if [ ! -d "$MPI_ARCH_PATH" ]
+ then
+ echo "${_foamMpiWarning}has invalid MPI_ROOT" 1>&2
+ echo "Please set correctly. For example," 1>&2
+ echo ' export MPI_ROOT="/opt/mpi"' 1>&2
echo 1>&2
else
- export MPI_ARCH_PATH=$MPI_ROOT
+ _foamMpiWarning="${_foamMpiWarning}has unset "
- if [ -z "$MPI_ARCH_FLAGS" ]
- then
- echo 1>&2
- echo "Warning in $WM_PROJECT_DIR/etc/config.sh/mpi:" 1>&2
- echo " MPI_ARCH_FLAGS is not set. Example:" 1>&2
- echo 1>&2
- echo " export MPI_ARCH_FLAGS=\"-DOMPI_SKIP_MPICXX\"" 1>&2
- echo 1>&2
- fi
-
- if [ -z "$MPI_ARCH_INC" ]
- then
- echo 1>&2
- echo "Warning in $WM_PROJECT_DIR/etc/config.sh/mpi:" 1>&2
- echo " MPI_ARCH_INC is not set. Example:" 1>&2
- echo 1>&2
- echo " export MPI_ARCH_INC=\"-isystem \$MPI_ROOT/include\"" 1>&2
- echo 1>&2
- fi
-
- if [ -z "$MPI_ARCH_LIBS" ]
- then
- echo 1>&2
- echo "Warning in $WM_PROJECT_DIR/etc/config.sh/mpi:" 1>&2
- echo " MPI_ARCH_LIBS is not set. Example:" 1>&2
- echo 1>&2
- echo " export MPI_ARCH_LIBS=\"-L\$MPI_ROOT/lib -lmpi\"" 1>&2
- echo 1>&2
- fi
+ [ -n "$MPI_ARCH_FLAGS" ] || {
+ exec 1>&2
+ echo "${_foamMpiWarning}MPI_ARCH_FLAGS"
+ echo "Please set correctly. For example,"
+ echo ' export MPI_ARCH_FLAGS="-DOMPI_SKIP_MPICXX"'
+ echo
+ }
+ [ -n "$MPI_ARCH_INC" ] || {
+ exec 1>&2
+ echo "${_foamMpiWarning}MPI_ARCH_INC"
+ echo "Please set correctly. For example,"
+ echo ' export MPI_ARCH_INC="-isystem $MPI_ROOT/include"'
+ echo
+ }
+ [ -n "$MPI_ARCH_LIBS" ] || {
+ exec 1>&2
+ echo "${_foamMpiWarning}MPI_ARCH_LIBS"
+ echo "Please set correctly. For example,"
+ echo ' export MPI_ARCH_LIBS="-L$MPI_ROOT/lib -lmpi'
+ echo
+ }
fi
;;
+
MPICH)
export FOAM_MPI=mpich-3.3
- export MPI_ARCH_PATH=$WM_THIRD_PARTY_DIR/platforms/$WM_ARCH$WM_COMPILER/$FOAM_MPI
- export MPI_HOME=$MPI_ARCH_PATH
+ _foamEtc -config prefs.mpich ## Optional adjustments
+
+ # MPI_ARCH_PATH (prefs) if a valid dir, or use ThirdParty location
+ if [ ! -d "$MPI_ARCH_PATH" ]
+ then
+ export MPI_ARCH_PATH="$_foamMpiPrefixDir/$FOAM_MPI"
+ fi
+
+ export MPI_HOME="$MPI_ARCH_PATH"
# Could be sourced from ThirdParty with incomplete environment
if command -v _foamAddLibAuto >/dev/null # Normal sourcing
then
- _foamAddPath $MPI_ARCH_PATH/bin
- _foamAddMan $MPI_ARCH_PATH/share/man
- _foamAddLibAuto $MPI_ARCH_PATH lib$WM_COMPILER_LIB_ARCH
+ _foamAddPath "$MPI_ARCH_PATH"/bin
+ _foamAddMan "$MPI_ARCH_PATH"/share/man
+ _foamAddLibAuto "$MPI_ARCH_PATH" "lib$WM_COMPILER_LIB_ARCH"
fi
;;
+
MPICH-GM)
export FOAM_MPI=mpich-gm
- export MPI_ARCH_PATH=/opt/mpi
- export MPICH_PATH=$MPI_ARCH_PATH
- export GM_LIB_PATH=/opt/gm/lib$WM_COMPILER_LIB_ARCH
+ _foamEtc -config prefs.mpich-gm ## Optional adjustments
- _foamAddPath $MPI_ARCH_PATH/bin
- _foamAddLib $MPI_ARCH_PATH/lib$WM_COMPILER_LIB_ARCH
- _foamAddLib $GM_LIB_PATH
+ # MPI_ARCH_PATH (prefs) if a valid dir, or system location
+ if [ ! -d "$MPI_ARCH_PATH" ]
+ then
+ export MPI_ARCH_PATH=/opt/mpi
+ fi
+
+ export MPICH_PATH="$MPI_ARCH_PATH"
+ export GM_LIB_PATH="$MPICH_PATH/lib$WM_COMPILER_LIB_ARCH"
+
+ _foamAddPath "$MPI_ARCH_PATH"/bin
+ _foamAddLib "$MPI_ARCH_PATH/lib$WM_COMPILER_LIB_ARCH"
+ _foamAddLib "$GM_LIB_PATH"
;;
-MVA2MPI) # MVAMPICH2
+
+# MVAPICH2, renamed to contain 'MPI' in its name
+MVA2MPI)
export FOAM_MPI=mvapich2-2.3
- export MPI_ARCH_PATH=$WM_THIRD_PARTY_DIR/platforms/$WM_ARCH$WM_COMPILER/$FOAM_MPI
+ _foamEtc -config prefs.mvapich2 ## Optional adjustments
+
+ # MPI_ARCH_PATH (prefs) if a valid dir, or ThirdParty location
+ if [ ! -d "$MPI_ARCH_PATH" ]
+ then
+ export MPI_ARCH_PATH="$_foamMpiPrefixDir/$FOAM_MPI"
+ fi
# Could be sourced from ThirdParty with incomplete environment
if command -v _foamAddLibAuto >/dev/null # Normal sourcing
then
- _foamAddPath $MPI_ARCH_PATH/bin
- _foamAddMan $MPI_ARCH_PATH/share/man
- _foamAddLibAuto $MPI_ARCH_PATH lib$WM_COMPILER_LIB_ARCH
+ _foamAddPath "$MPI_ARCH_PATH"/bin
+ _foamAddMan "$MPI_ARCH_PATH"/share/man
+ _foamAddLibAuto "$MPI_ARCH_PATH" "lib$WM_COMPILER_LIB_ARCH"
fi
;;
+
CRAY-MPICH)
export FOAM_MPI=cray-mpich
- export MPI_ARCH_PATH=$MPICH_DIR
-
- if [ -n "$FOAM_VERBOSE" ] && [ -n "$PS1" ]
- then
- echo "Using $WM_MPLIB" 1>&2
- echo " FOAM_MPI : $FOAM_MPI" 1>&2
- echo " MPICH_DIR : $MPI_ARCH_PATH" 1>&2
- fi
+ export MPI_ARCH_PATH="$MPICH_DIR"
[ -d "$MPI_ARCH_PATH" ] || {
- echo "Warning in $WM_PROJECT_DIR/etc/config.sh/mpi:" 1>&2
- echo " Not a valid $WM_MPLIB installation directory." 1>&2
- echo " Please set MPICH_DIR properly" 1>&2
- echo " Currently using '$MPI_ARCH_PATH'" 1>&2
+ echo "%{_foamMpiWarning}invalid $WM_MPLIB directory" 1>&2
+ echo " => ${MPI_ARCH_PATH:-unset}" 1>&2
+ echo "Please set MPICH_DIR correctly" 1>&2
}
- # _foamAddPath $MPI_ARCH_PATH/bin
- _foamAddLib $MPI_ARCH_PATH/lib
+ # _foamAddPath "$MPI_ARCH_PATH"/bin
+ _foamAddLib "$MPI_ARCH_PATH"/lib
;;
+
HPMPI)
export FOAM_MPI=hpmpi
export MPI_HOME=/opt/hpmpi
- export MPI_ARCH_PATH=$MPI_HOME
+ export MPI_ARCH_PATH="$MPI_HOME"
- _foamAddPath $MPI_ARCH_PATH/bin
-
- case $(uname -m) in
- i686)
- _foamAddLib $MPI_ARCH_PATH/lib/linux_ia32
- ;;
-
- x86_64)
- _foamAddLib $MPI_ARCH_PATH/lib/linux_amd64
- ;;
- ia64)
- _foamAddLib $MPI_ARCH_PATH/lib/linux_ia64
- ;;
- *)
- echo Unknown processor type $(uname -m) 1>&2
- ;;
+ unset libDir
+ case "$(uname -m)" in
+ x86_64) libDir=lib/linux_amd64 ;;
+ i686) libDir=lib/linux_ia32 ;;
+ ia64) libDir=lib/linux_ia64 ;;
esac
+
+ if [ -n "$libDir" ]
+ then
+ _foamAddPath "$MPI_ARCH_PATH"/bin
+ _foamAddLib "$MPI_ARCH_PATH/$libDir"
+ else
+ echo "openfoam: ($(uname -m)) - unsupported HP-MPI processor type" 1>&2
+ fi
+ unset libDir
;;
+
MPI)
export FOAM_MPI=mpi
export MPI_ARCH_PATH=/opt/mpi
;;
+
FJMPI)
export FOAM_MPI=fjmpi
export MPI_ARCH_PATH=/opt/FJSVmpi2
- _foamAddPath $MPI_ARCH_PATH/bin
- _foamAddLib $MPI_ARCH_PATH/lib/sparcv9
+ _foamAddPath "$MPI_ARCH_PATH"/bin
+ _foamAddLib "$MPI_ARCH_PATH"/lib/sparcv9
_foamAddLib /opt/FSUNf90/lib/sparcv9
_foamAddLib /opt/FJSVpnidt/lib
;;
+
QSMPI)
export FOAM_MPI=qsmpi
export MPI_ARCH_PATH=/usr/lib/mpi
- _foamAddPath $MPI_ARCH_PATH/bin
- _foamAddLib $MPI_ARCH_PATH/lib
+ _foamAddPath "$MPI_ARCH_PATH"/bin
+ _foamAddLib "$MPI_ARCH_PATH"/lib
;;
+
SGIMPI)
- # No trailing slash
+ : "${MPI_ROOT:=/dummy}"
MPI_ROOT="${MPI_ROOT%/}" # Remove trailing slash
- export MPI_ARCH_PATH="${MPI_ROOT%/}" # Remove trailing slash
- export FOAM_MPI="${MPI_ARCH_PATH##*/}"
+ export FOAM_MPI="${MPI_ROOT##*/}"
+ export MPI_ARCH_PATH="$MPI_ROOT"
[ -d "$MPI_ARCH_PATH" ] || {
- echo "Warning in $WM_PROJECT_DIR/etc/config.sh/mpi:" 1>&2
- echo " Not a valid $WM_MPLIB installation directory." 1>&2
- echo " Please set MPI_ROOT properly (usually via the mpt module)" 1>&2
- echo " Currently using '$MPI_ARCH_PATH'" 1>&2
+ echo "%{_foamMpiWarning}invalid $WM_MPLIB directory" 1>&2
+ echo " => ${MPI_ARCH_PATH:-unset}" 1>&2
+ echo "Please set MPI_ROOT correctly" 1>&2
}
- if [ -n "$FOAM_VERBOSE" ] && [ -n "$PS1" ]
- then
- echo "Using $WM_MPLIB" 1>&2
- echo " FOAM_MPI : $FOAM_MPI" 1>&2
- echo " MPI_ROOT : $MPI_ARCH_PATH" 1>&2
- fi
-
- _foamAddPath $MPI_ARCH_PATH/bin
- _foamAddLib $MPI_ARCH_PATH/lib
+ _foamAddPath "$MPI_ARCH_PATH"/bin
+ _foamAddLib "$MPI_ARCH_PATH"/lib
;;
+
INTELMPI*)
+ _foamEtc -config prefs.intelmpi ## Optional adjustments
+
if [ -n "$I_MPI_ROOT" ]
then
- # I_MPI_ROOT: The Intel MPI Library installation directory
+ # I_MPI_ROOT: Intel MPI Library installation directory
+ MPI_ARCH_PATH="${I_MPI_ROOT}" # Remove trailing slash
- MPI_ARCH_PATH="${I_MPI_ROOT%/}" # Remove trailing slash
- FOAM_MPI="${MPI_ARCH_PATH##*/}"
-
- # If subdirectory is version number only, prefix with 'impi-'
- case "$FOAM_MPI" in ([0-9]*) FOAM_MPI="impi-$FOAM_MPI";; esac
-
- if [ -n "$FOAM_VERBOSE" ] && [ -n "$PS1" ]
- then
- echo "Using $WM_MPLIB" 1>&2
- echo " FOAM_MPI : $FOAM_MPI" 1>&2
- echo " I_MPI_ROOT : $MPI_ARCH_PATH" 1>&2
- fi
- else
+ elif [ -n "$MPI_ROOT" ]
+ then
# MPI_ROOT: General specification
- MPI_ARCH_PATH="${MPI_ROOT%/}" # Remove trailing slash
- FOAM_MPI="${MPI_ARCH_PATH##*/}"
+ MPI_ARCH_PATH="${MPI_ROOT}"
- # If subdirectory is version number only, prefix with 'impi-'
- case "$FOAM_MPI" in ([0-9]*) FOAM_MPI="impi-$FOAM_MPI";; esac
-
- if [ -n "$FOAM_VERBOSE" ] && [ -n "$PS1" ]
- then
- echo "Using $WM_MPLIB" 1>&2
- echo " FOAM_MPI : $FOAM_MPI" 1>&2
- echo " MPI_ROOT : $MPI_ARCH_PATH" 1>&2
- fi
+ elif [ -n "$MPI_ARCH_PATH" ]
+ then
+ # MPI_ARCH_PATH: Set I_MPI_ROOT accordingly
+ export I_MPI_ROOT="$MPI_ARCH_PATH"
fi
- [ -d "$MPI_ARCH_PATH" ] || {
- echo "Warning in $WM_PROJECT_DIR/etc/config.sh/mpi:" 1>&2
- echo " Not a valid $WM_MPLIB installation directory." 1>&2
- echo " Please set I_MPI_ROOT or MPI_ROOT properly." 1>&2
- echo " Currently using '$MPI_ARCH_PATH'" 1>&2
- }
+
+ if [ -d "$MPI_ARCH_PATH" ]
+ then
+ # Remove trailing slash
+ MPI_ARCH_PATH="${MPI_ARCH_PATH%/}"
+
+ # Subdir name is FOAM_MPI name
+ FOAM_MPI="${MPI_ARCH_PATH##*/}"
+
+ # If subdir is version number only, prefix with 'impi-'
+ case "$FOAM_MPI" in ([0-9]*) FOAM_MPI="impi-$FOAM_MPI";; esac
+ else
+ echo "${_foamMpiWarning}invalid $WM_MPLIB directory" 1>&2
+ echo " => ${MPI_ARCH_PATH:-unset}" 1>&2
+ echo "Please set I_MPI_ROOT or MPI_ROOT correctly." 1>&2
+ fi
export FOAM_MPI MPI_ARCH_PATH
- _foamAddPath "$MPI_ARCH_PATH/intel64/bin"
- _foamAddLib "$MPI_ARCH_PATH/intel64/lib"
+ _foamAddPath "$MPI_ARCH_PATH"/intel64/bin
+ _foamAddLib "$MPI_ARCH_PATH"/intel64/lib
;;
+
MSMPI)
export FOAM_MPI=msmpi
- _foamEtc -config mpi-msmpi # <- Adjustments (optional)
+ _foamEtc -config prefs.msmpi ## Optional adjustments
- # Respect MPI_ARCH_PATH if set to a valid directory (eg, user adjustments)
+ # MPI_ARCH_PATH (prefs) if a valid dir, or ThirdParty location
if [ ! -d "$MPI_ARCH_PATH" ]
then
- export MPI_ARCH_PATH=$WM_THIRD_PARTY_DIR/platforms/$WM_ARCH$WM_COMPILER/$FOAM_MPI
+ export MPI_ARCH_PATH="$_foamMpiPrefixDir/$FOAM_MPI"
fi
- # _foamAddPath "$MPI_ARCH_PATH/bin"
- _foamAddLib "$MPI_ARCH_PATH/lib/x64"
+ # _foamAddPath "$MPI_ARCH_PATH"/bin
+ _foamAddLib "$MPI_ARCH_PATH"/lib/x64
;;
+
esac
-# Add (non-dummy) MPI implementation
-# - dummy MPI already in lib-path and has no external libraries
-if [ "$FOAM_MPI" != dummy ] && command -v _foamAddLib >/dev/null
+#------------------------------------------------------------------------------
+# Final
+
+# Ensure MPI_ARCH_PATH is marked for export
+[ -n "$MPI_ARCH_PATH" ] && export MPI_ARCH_PATH
+
+
+# A real (non-dummy) MPI
+if [ "$FOAM_MPI" != dummy ]
then
- if [ -n "$FOAM_EXT_LIBBIN" ] # External libraries (allowed to be unset)
+ if [ -n "$FOAM_VERBOSE" ] && [ -n "$PS1" ]
then
- _foamAddLib $FOAM_EXT_LIBBIN/$FOAM_MPI
+ echo "Using mpi type : $WM_MPLIB" 1>&2
+ echo " FOAM_MPI : $FOAM_MPI" 1>&2
+ echo " prefix : ${MPI_ARCH_PATH:-unset}" 1>&2
+ fi
+
+ # OpenFOAM library interface to (non-dummy) MPI
+ if command -v _foamAddLib >/dev/null
+ then
+ if [ -n "$FOAM_EXT_LIBBIN" ]
+ then
+ # External libraries (can be unset)
+ _foamAddLib "$FOAM_EXT_LIBBIN/$FOAM_MPI"
+ fi
+ _foamAddLib "$FOAM_LIBBIN/$FOAM_MPI"
fi
- _foamAddLib $FOAM_LIBBIN/$FOAM_MPI
fi
-# Alternative to 'mpiBufferSize' controlDict entry
-#export MPI_BUFFER_SIZE=20000000
+
+# Cleanup
+unset _foamMpiPrefixDir _foamMpiVersion _foamMpiWarning
#------------------------------------------------------------------------------
diff --git a/etc/cshrc b/etc/cshrc
index f15bf120a2..8959453798 100644
--- a/etc/cshrc
+++ b/etc/cshrc
@@ -91,7 +91,8 @@ setenv WM_COMPILE_OPTION Opt
# [WM_MPLIB] - MPI implementation:
# = SYSTEMOPENMPI | OPENMPI | SYSTEMMPI | MPI | MPICH | MPICH-GM |
# HPMPI | CRAY-MPICH | FJMPI | QSMPI | SGIMPI | INTELMPI | USERMPI
-# Also possible to use INTELMPI-xyz etc and define your own wmake rule
+# Specify SYSTEMOPENMPI1, SYSTEMOPENMPI2 for internal tracking (if desired)
+# Can also use INTELMPI-xyz etc and define your own wmake rule
setenv WM_MPLIB SYSTEMOPENMPI
diff --git a/src/Pstream/mpi/Make/options b/src/Pstream/mpi/Make/options
index 4c2b466f6f..69669a94ac 100644
--- a/src/Pstream/mpi/Make/options
+++ b/src/Pstream/mpi/Make/options
@@ -1,10 +1,4 @@
-sinclude $(GENERAL_RULES)/mplib$(WM_MPLIB)
-sinclude $(DEFAULT_RULES)/mplib$(WM_MPLIB)
-
-/* Default is PROJECT LIB/mpi target */
-ifeq (,$(FOAM_MPI_LIBBIN))
- FOAM_MPI_LIBBIN := $(FOAM_LIBBIN)/$(FOAM_MPI)
-endif
+include $(GENERAL_RULES)/mpi-rules
EXE_INC = $(PFLAGS) $(PINC) $(c++LESSWARN)
diff --git a/src/parallel/decompose/ptscotchDecomp/Make/options b/src/parallel/decompose/ptscotchDecomp/Make/options
index da9e22b8af..6f302fb634 100644
--- a/src/parallel/decompose/ptscotchDecomp/Make/options
+++ b/src/parallel/decompose/ptscotchDecomp/Make/options
@@ -1,14 +1,8 @@
/*
- * NB: mplib PINC must appear after PTSCOTCH_INC_DIR to ensure we
- * do not accidentally get a ptscotch header from the MPI distribution.
+ * NB: PINC must appear after PTSCOTCH_INC_DIR to ensure we do not
+ * accidentally get a ptscotch header from the MPI distribution
*/
-sinclude $(GENERAL_RULES)/mplib$(WM_MPLIB)
-sinclude $(DEFAULT_RULES)/mplib$(WM_MPLIB)
-
-/* Default is PROJECT LIB/mpi target */
-ifeq (,$(strip $(FOAM_MPI_LIBBIN)))
- FOAM_MPI_LIBBIN := $(FOAM_LIBBIN)/$(FOAM_MPI)
-endif
+include $(GENERAL_RULES)/mpi-rules
EXE_INC = \
-I$(PTSCOTCH_INC_DIR) \
diff --git a/src/renumber/zoltanRenumber/Make/options b/src/renumber/zoltanRenumber/Make/options
index f16794e34d..cf3b1e7226 100644
--- a/src/renumber/zoltanRenumber/Make/options
+++ b/src/renumber/zoltanRenumber/Make/options
@@ -1,5 +1,4 @@
-sinclude $(GENERAL_RULES)/mplib$(WM_MPLIB)
-sinclude $(DEFAULT_RULES)/mplib$(WM_MPLIB)
+include $(GENERAL_RULES)/mpi-rules
EXE_INC = \
-I$(ZOLTAN_INC_DIR) \
diff --git a/wmake/rules/General/ADIOS2 b/wmake/rules/General/ADIOS2
index 62f467196f..0efd7a3632 100644
--- a/wmake/rules/General/ADIOS2
+++ b/wmake/rules/General/ADIOS2
@@ -1,8 +1,7 @@
-#-------------------------------*- makefile -*---------------------------------
+#------------------------------------------------------------------------------
# ADIOS2 includes/libraries
-sinclude $(GENERAL_RULES)/mplib$(WM_MPLIB)
-sinclude $(DEFAULT_RULES)/mplib$(WM_MPLIB)
+include $(GENERAL_RULES)/mpi-rules
# May need to strip off rpath ...
diff --git a/wmake/rules/General/mpi-mpicc-openmpi b/wmake/rules/General/mpi-mpicc-openmpi
new file mode 100644
index 0000000000..7f898a577f
--- /dev/null
+++ b/wmake/rules/General/mpi-mpicc-openmpi
@@ -0,0 +1,8 @@
+#------------------------------------------------------------------------------
+# Use openmpi mpicc to obtain settings - mpicc must be available
+
+PFLAGS = -DMPICH_SKIP_MPICXX -DOMPI_SKIP_MPICXX
+PINC = $(shell mpicc --showme:compile)
+PLIBS = $(shell mpicc --showme:link)
+
+#------------------------------------------------------------------------------
diff --git a/wmake/rules/General/mpi-rules b/wmake/rules/General/mpi-rules
new file mode 100644
index 0000000000..cfcd6d6eca
--- /dev/null
+++ b/wmake/rules/General/mpi-rules
@@ -0,0 +1,32 @@
+#------------------------------------------------------------------------------
+# MPI rules
+
+PFLAGS =
+PINC =
+PLIBS =
+
+# The stem name. Eg, SYSTEMOPENMPI4 -> SYSTEMOPENMPI
+MPLIB_STEMNAME = $(shell echo "$(WM_MPLIB)" | sed -e 's/[-.0-9]*$$//')
+
+ifneq ("$(MPLIB_STEMNAME)","$(WM_MPLIB)")
+ sinclude $(GENERAL_RULES)/mplib$(MPLIB_STEMNAME)
+ sinclude $(DEFAULT_RULES)/mplib$(MPLIB_STEMNAME)
+ ifneq ("$(RULES)","$(DEFAULT_RULES)")
+ sinclude $(RULES)/mplib$(MPLIB_STEMNAME)
+ endif
+endif
+
+ifneq ("","$(WM_MPLIB)")
+ sinclude $(GENERAL_RULES)/mplib$(WM_MPLIB)
+ sinclude $(DEFAULT_RULES)/mplib$(WM_MPLIB)
+ ifneq ("$(RULES)","$(DEFAULT_RULES)")
+ sinclude $(RULES)/mplib$(WM_MPLIB)
+ endif
+endif
+
+# Default target MPI libdir is PROJECT lib/
+ifeq (,$(FOAM_MPI_LIBBIN))
+ FOAM_MPI_LIBBIN := $(FOAM_LIBBIN)/$(FOAM_MPI)
+endif
+
+#------------------------------------------------------------------------------
diff --git a/wmake/rules/General/mplib b/wmake/rules/General/mplib
deleted file mode 120000
index b7d2b8d468..0000000000
--- a/wmake/rules/General/mplib
+++ /dev/null
@@ -1 +0,0 @@
-no-mpi
\ No newline at end of file
diff --git a/wmake/rules/General/mplib-ia32-HPMPI b/wmake/rules/General/mplib-ia32-HPMPI
new file mode 100644
index 0000000000..1e0c1f0574
--- /dev/null
+++ b/wmake/rules/General/mplib-ia32-HPMPI
@@ -0,0 +1,8 @@
+#------------------------------------------------------------------------------
+# HPMPI rules (i386)
+
+PFLAGS =
+PINC = -isystem $(MPI_ARCH_PATH)/include -D_MPICC_H
+PLIBS = -L$(MPI_ARCH_PATH)/lib/linux_ia32 -lmpi
+
+#------------------------------------------------------------------------------
diff --git a/wmake/rules/General/mplib-ia64-HPMPI b/wmake/rules/General/mplib-ia64-HPMPI
new file mode 100644
index 0000000000..db327ee7e9
--- /dev/null
+++ b/wmake/rules/General/mplib-ia64-HPMPI
@@ -0,0 +1,8 @@
+#------------------------------------------------------------------------------
+# HPMPI rules (ia64)
+
+PFLAGS =
+PINC = -isystem $(MPI_ARCH_PATH)/include -D_MPICC_H
+PLIBS = -L$(MPI_ARCH_PATH)/lib/linux_ia64 -lmpi
+
+#------------------------------------------------------------------------------
diff --git a/wmake/rules/General/mplib-x86_64-HPMPI b/wmake/rules/General/mplib-x86_64-HPMPI
new file mode 100644
index 0000000000..e5e24b0c6a
--- /dev/null
+++ b/wmake/rules/General/mplib-x86_64-HPMPI
@@ -0,0 +1,8 @@
+#------------------------------------------------------------------------------
+# HPMPI rules (x86_64)
+
+PFLAGS =
+PINC = -isystem $(MPI_ARCH_PATH)/include -D_MPICC_H
+PLIBS = -L$(MPI_ARCH_PATH)/lib/linux_amd64 -lmpi
+
+#------------------------------------------------------------------------------
diff --git a/wmake/rules/General/mplib-x86_64-INTELMPI b/wmake/rules/General/mplib-x86_64-INTELMPI
new file mode 100644
index 0000000000..6d10729d73
--- /dev/null
+++ b/wmake/rules/General/mplib-x86_64-INTELMPI
@@ -0,0 +1,8 @@
+#------------------------------------------------------------------------------
+# INTELMPI rules (x86_64)
+
+PFLAGS = -DMPICH_SKIP_MPICXX -DOMPI_SKIP_MPICXX
+PINC = -isystem $(MPI_ARCH_PATH)/intel64/include
+PLIBS = -L$(MPI_ARCH_PATH)/intel64/lib -lmpi
+
+#------------------------------------------------------------------------------
diff --git a/wmake/rules/General/mplibCRAY-MPICH b/wmake/rules/General/mplibCRAY-MPICH
index fdb91c1379..3b8d7ef71d 100644
--- a/wmake/rules/General/mplibCRAY-MPICH
+++ b/wmake/rules/General/mplibCRAY-MPICH
@@ -1 +1,6 @@
+#------------------------------------------------------------------------------
+# Cray MPICH is MPICH
+
include $(GENERAL_RULES)/mplibMPICH
+
+#------------------------------------------------------------------------------
diff --git a/wmake/rules/General/mplibMPICH b/wmake/rules/General/mplibMPICH
index 8d3fee45b2..7809f2d929 100644
--- a/wmake/rules/General/mplibMPICH
+++ b/wmake/rules/General/mplibMPICH
@@ -1,3 +1,7 @@
+#------------------------------------------------------------------------------
+
PFLAGS = -DMPICH_SKIP_MPICXX -DOMPI_SKIP_MPICXX
PINC = -isystem $(MPI_ARCH_PATH)/include
PLIBS = -L$(MPI_ARCH_PATH)/lib$(WM_COMPILER_LIB_ARCH) -L$(MPI_ARCH_PATH)/lib -lmpi -lrt
+
+#------------------------------------------------------------------------------
diff --git a/wmake/rules/General/mplibMPICH-GM b/wmake/rules/General/mplibMPICH-GM
index 1921318a0a..1402f50ce6 100644
--- a/wmake/rules/General/mplibMPICH-GM
+++ b/wmake/rules/General/mplibMPICH-GM
@@ -1,3 +1,7 @@
+#------------------------------------------------------------------------------
+
PFLAGS =
PINC = -isystem $(MPI_ARCH_PATH)/include
PLIBS = -L$(MPI_ARCH_PATH)/lib$(WM_COMPILER_LIB_ARCH) -L$(MPI_ARCH_PATH)/lib -lmpich -L$(GM_LIB_PATH) -lgm
+
+#------------------------------------------------------------------------------
diff --git a/wmake/rules/General/mplibMVA2MPI b/wmake/rules/General/mplibMVA2MPI
index 112f510b3c..1ad587f59e 100644
--- a/wmake/rules/General/mplibMVA2MPI
+++ b/wmake/rules/General/mplibMVA2MPI
@@ -1,3 +1,7 @@
+#------------------------------------------------------------------------------
+
PFLAGS = -DMPICH_SKIP_MPICXX -DOMPI_SKIP_MPICXX
PINC = -isystem $(MPI_ARCH_PATH)/include
PLIBS = -L$(MPI_ARCH_PATH)/lib$(WM_COMPILER_LIB_ARCH) -L$(MPI_ARCH_PATH)/lib -lmpi
+
+#------------------------------------------------------------------------------
diff --git a/wmake/rules/General/mplibOPENMPI b/wmake/rules/General/mplibOPENMPI
index 112f510b3c..1ad587f59e 100644
--- a/wmake/rules/General/mplibOPENMPI
+++ b/wmake/rules/General/mplibOPENMPI
@@ -1,3 +1,7 @@
+#------------------------------------------------------------------------------
+
PFLAGS = -DMPICH_SKIP_MPICXX -DOMPI_SKIP_MPICXX
PINC = -isystem $(MPI_ARCH_PATH)/include
PLIBS = -L$(MPI_ARCH_PATH)/lib$(WM_COMPILER_LIB_ARCH) -L$(MPI_ARCH_PATH)/lib -lmpi
+
+#------------------------------------------------------------------------------
diff --git a/wmake/rules/General/mplibQSMPI b/wmake/rules/General/mplibQSMPI
index aa15c1b511..c9f8da312b 100644
--- a/wmake/rules/General/mplibQSMPI
+++ b/wmake/rules/General/mplibQSMPI
@@ -1,3 +1,7 @@
+#------------------------------------------------------------------------------
+
PFLAGS =
PINC = -isystem $(MPI_ARCH_PATH)/include
PLIBS = -L$(MPI_ARCH_PATH)/lib -lmpi
+
+#------------------------------------------------------------------------------
diff --git a/wmake/rules/General/mplibSGIMPI b/wmake/rules/General/mplibSGIMPI
index 205b9406c0..dcc15fa9d8 100644
--- a/wmake/rules/General/mplibSGIMPI
+++ b/wmake/rules/General/mplibSGIMPI
@@ -1,3 +1,7 @@
+#------------------------------------------------------------------------------
+
PFLAGS = -DSGIMPI -DMPI_NO_CPPBIND
PINC = -isystem $(MPI_ARCH_PATH)/include
PLIBS = -L$(MPI_ARCH_PATH)/lib -lmpi
+
+#------------------------------------------------------------------------------
diff --git a/wmake/rules/General/mplibSYSTEMMPI b/wmake/rules/General/mplibSYSTEMMPI
index a0d3d3098d..8c1fb49191 100644
--- a/wmake/rules/General/mplibSYSTEMMPI
+++ b/wmake/rules/General/mplibSYSTEMMPI
@@ -1,5 +1,8 @@
-# Use the system installed mpi - depends on specific environment variables
+#------------------------------------------------------------------------------
+# System installed mpi - depends on specific environment variables
PFLAGS = $(MPI_ARCH_FLAGS)
PINC = $(MPI_ARCH_INC)
PLIBS = $(MPI_ARCH_LIBS)
+
+#------------------------------------------------------------------------------
diff --git a/wmake/rules/General/mplibSYSTEMOPENMPI b/wmake/rules/General/mplibSYSTEMOPENMPI
index 9245c3ee99..d6ea275e79 100644
--- a/wmake/rules/General/mplibSYSTEMOPENMPI
+++ b/wmake/rules/General/mplibSYSTEMOPENMPI
@@ -1,5 +1,6 @@
-# Use the system installed openmpi - mpicc must be installed
+#------------------------------------------------------------------------------
+# Use openmpi mpicc to obtain settings - mpicc must be available
-PFLAGS = -DMPICH_SKIP_MPICXX -DOMPI_SKIP_MPICXX
-PINC = $(shell mpicc --showme:compile)
-PLIBS = $(shell mpicc --showme:link)
+include $(GENERAL_RULES)/mpi-mpicc-openmpi
+
+#------------------------------------------------------------------------------
diff --git a/wmake/rules/General/no-mpi b/wmake/rules/General/no-mpi
index a682d31029..21c81b9a37 100644
--- a/wmake/rules/General/no-mpi
+++ b/wmake/rules/General/no-mpi
@@ -1,7 +1,8 @@
+#------------------------------------------------------------------------------
# Empty flags for not compiling/linking MPI
-PFLAGS =
-PINC =
-PLIBS =
+PFLAGS =
+PINC =
+PLIBS =
#------------------------------------------------------------------------------
diff --git a/wmake/rules/linux64Amd/mplibHPMPI b/wmake/rules/linux64Amd/mplibHPMPI
index d7c4b0cf30..08c6149fec 100644
--- a/wmake/rules/linux64Amd/mplibHPMPI
+++ b/wmake/rules/linux64Amd/mplibHPMPI
@@ -1,3 +1 @@
-PFLAGS =
-PINC = -isystem $(MPI_ARCH_PATH)/include -D_MPICC_H
-PLIBS = -L$(MPI_ARCH_PATH)/lib/linux_amd64 -lmpi
+include $(GENERAL_RULES)/mplib-x86_64-HPMPI
diff --git a/wmake/rules/linux64Amd/mplibINTELMPI b/wmake/rules/linux64Amd/mplibINTELMPI
index b3667bce26..6fc8ce3c36 100644
--- a/wmake/rules/linux64Amd/mplibINTELMPI
+++ b/wmake/rules/linux64Amd/mplibINTELMPI
@@ -1,3 +1 @@
-PFLAGS = -DMPICH_SKIP_MPICXX -DOMPI_SKIP_MPICXX
-PINC = -isystem $(MPI_ARCH_PATH)/intel64/include
-PLIBS = -L$(MPI_ARCH_PATH)/intel64/lib -lmpi
+include $(GENERAL_RULES)/mplib-x86_64-INTELMPI
diff --git a/wmake/rules/linux64Clang/mplibHPMPI b/wmake/rules/linux64Clang/mplibHPMPI
index d7c4b0cf30..08c6149fec 100644
--- a/wmake/rules/linux64Clang/mplibHPMPI
+++ b/wmake/rules/linux64Clang/mplibHPMPI
@@ -1,3 +1 @@
-PFLAGS =
-PINC = -isystem $(MPI_ARCH_PATH)/include -D_MPICC_H
-PLIBS = -L$(MPI_ARCH_PATH)/lib/linux_amd64 -lmpi
+include $(GENERAL_RULES)/mplib-x86_64-HPMPI
diff --git a/wmake/rules/linux64Clang/mplibINTELMPI b/wmake/rules/linux64Clang/mplibINTELMPI
index b3667bce26..6fc8ce3c36 100644
--- a/wmake/rules/linux64Clang/mplibINTELMPI
+++ b/wmake/rules/linux64Clang/mplibINTELMPI
@@ -1,3 +1 @@
-PFLAGS = -DMPICH_SKIP_MPICXX -DOMPI_SKIP_MPICXX
-PINC = -isystem $(MPI_ARCH_PATH)/intel64/include
-PLIBS = -L$(MPI_ARCH_PATH)/intel64/lib -lmpi
+include $(GENERAL_RULES)/mplib-x86_64-INTELMPI
diff --git a/wmake/rules/linux64Cray/mplibINTELMPI b/wmake/rules/linux64Cray/mplibINTELMPI
index b3667bce26..6fc8ce3c36 100644
--- a/wmake/rules/linux64Cray/mplibINTELMPI
+++ b/wmake/rules/linux64Cray/mplibINTELMPI
@@ -1,3 +1 @@
-PFLAGS = -DMPICH_SKIP_MPICXX -DOMPI_SKIP_MPICXX
-PINC = -isystem $(MPI_ARCH_PATH)/intel64/include
-PLIBS = -L$(MPI_ARCH_PATH)/intel64/lib -lmpi
+include $(GENERAL_RULES)/mplib-x86_64-INTELMPI
diff --git a/wmake/rules/linux64Gcc/mplibHPMPI b/wmake/rules/linux64Gcc/mplibHPMPI
index d7c4b0cf30..08c6149fec 100644
--- a/wmake/rules/linux64Gcc/mplibHPMPI
+++ b/wmake/rules/linux64Gcc/mplibHPMPI
@@ -1,3 +1 @@
-PFLAGS =
-PINC = -isystem $(MPI_ARCH_PATH)/include -D_MPICC_H
-PLIBS = -L$(MPI_ARCH_PATH)/lib/linux_amd64 -lmpi
+include $(GENERAL_RULES)/mplib-x86_64-HPMPI
diff --git a/wmake/rules/linux64Gcc/mplibINTELMPI b/wmake/rules/linux64Gcc/mplibINTELMPI
index b3667bce26..6fc8ce3c36 100644
--- a/wmake/rules/linux64Gcc/mplibINTELMPI
+++ b/wmake/rules/linux64Gcc/mplibINTELMPI
@@ -1,3 +1 @@
-PFLAGS = -DMPICH_SKIP_MPICXX -DOMPI_SKIP_MPICXX
-PINC = -isystem $(MPI_ARCH_PATH)/intel64/include
-PLIBS = -L$(MPI_ARCH_PATH)/intel64/lib -lmpi
+include $(GENERAL_RULES)/mplib-x86_64-INTELMPI
diff --git a/wmake/rules/linux64Icc/mplibHPMPI b/wmake/rules/linux64Icc/mplibHPMPI
index d7c4b0cf30..08c6149fec 100644
--- a/wmake/rules/linux64Icc/mplibHPMPI
+++ b/wmake/rules/linux64Icc/mplibHPMPI
@@ -1,3 +1 @@
-PFLAGS =
-PINC = -isystem $(MPI_ARCH_PATH)/include -D_MPICC_H
-PLIBS = -L$(MPI_ARCH_PATH)/lib/linux_amd64 -lmpi
+include $(GENERAL_RULES)/mplib-x86_64-HPMPI
diff --git a/wmake/rules/linux64Icc/mplibINTELMPI b/wmake/rules/linux64Icc/mplibINTELMPI
index b3667bce26..6fc8ce3c36 100644
--- a/wmake/rules/linux64Icc/mplibINTELMPI
+++ b/wmake/rules/linux64Icc/mplibINTELMPI
@@ -1,3 +1 @@
-PFLAGS = -DMPICH_SKIP_MPICXX -DOMPI_SKIP_MPICXX
-PINC = -isystem $(MPI_ARCH_PATH)/intel64/include
-PLIBS = -L$(MPI_ARCH_PATH)/intel64/lib -lmpi
+include $(GENERAL_RULES)/mplib-x86_64-INTELMPI
diff --git a/wmake/rules/linuxClang/mplibHPMPI b/wmake/rules/linuxClang/mplibHPMPI
index c44b3dd3f9..baab3a9d90 100644
--- a/wmake/rules/linuxClang/mplibHPMPI
+++ b/wmake/rules/linuxClang/mplibHPMPI
@@ -1,3 +1 @@
-PFLAGS =
-PINC = -isystem $(MPI_ARCH_PATH)/include -D_MPICC_H
-PLIBS = -L$(MPI_ARCH_PATH)/lib/linux_ia32 -lmpi
+include $(GENERAL_RULES)/mplib-ia32-HPMPI
diff --git a/wmake/rules/linuxGcc/mplibHPMPI b/wmake/rules/linuxGcc/mplibHPMPI
index c44b3dd3f9..baab3a9d90 100644
--- a/wmake/rules/linuxGcc/mplibHPMPI
+++ b/wmake/rules/linuxGcc/mplibHPMPI
@@ -1,3 +1 @@
-PFLAGS =
-PINC = -isystem $(MPI_ARCH_PATH)/include -D_MPICC_H
-PLIBS = -L$(MPI_ARCH_PATH)/lib/linux_ia32 -lmpi
+include $(GENERAL_RULES)/mplib-ia32-HPMPI
diff --git a/wmake/rules/linuxIA64Gcc/mplibHPMPI b/wmake/rules/linuxIA64Gcc/mplibHPMPI
index cd11010d24..36c64a3719 100644
--- a/wmake/rules/linuxIA64Gcc/mplibHPMPI
+++ b/wmake/rules/linuxIA64Gcc/mplibHPMPI
@@ -1,3 +1 @@
-PFLAGS =
-PINC = -isystem $(MPI_ARCH_PATH)/include -D_MPICC_H
-PLIBS = -L$(MPI_ARCH_PATH)/lib/linux_ia64 -lmpi
+include $(GENERAL_RULES)/mplib-ia64-HPMPI
diff --git a/wmake/rules/linuxIA64Icc/mplibHPMPI b/wmake/rules/linuxIA64Icc/mplibHPMPI
index cd11010d24..36c64a3719 100644
--- a/wmake/rules/linuxIA64Icc/mplibHPMPI
+++ b/wmake/rules/linuxIA64Icc/mplibHPMPI
@@ -1,3 +1 @@
-PFLAGS =
-PINC = -isystem $(MPI_ARCH_PATH)/include -D_MPICC_H
-PLIBS = -L$(MPI_ARCH_PATH)/lib/linux_ia64 -lmpi
+include $(GENERAL_RULES)/mplib-ia64-HPMPI
diff --git a/wmake/rules/linuxIA64Icc/mplibINTELMPI b/wmake/rules/linuxIA64Icc/mplibINTELMPI
deleted file mode 100644
index b3667bce26..0000000000
--- a/wmake/rules/linuxIA64Icc/mplibINTELMPI
+++ /dev/null
@@ -1,3 +0,0 @@
-PFLAGS = -DMPICH_SKIP_MPICXX -DOMPI_SKIP_MPICXX
-PINC = -isystem $(MPI_ARCH_PATH)/intel64/include
-PLIBS = -L$(MPI_ARCH_PATH)/intel64/lib -lmpi
diff --git a/wmake/rules/linuxIcc/mplibHPMPI b/wmake/rules/linuxIcc/mplibHPMPI
index c44b3dd3f9..baab3a9d90 100644
--- a/wmake/rules/linuxIcc/mplibHPMPI
+++ b/wmake/rules/linuxIcc/mplibHPMPI
@@ -1,3 +1 @@
-PFLAGS =
-PINC = -isystem $(MPI_ARCH_PATH)/include -D_MPICC_H
-PLIBS = -L$(MPI_ARCH_PATH)/lib/linux_ia32 -lmpi
+include $(GENERAL_RULES)/mplib-ia32-HPMPI