ENH: improved granularity for MPI-rebuilds

- Provide Allwmake-mpi scripts for handling the MPI-only build segment.

- Adjust Make/options to support a FOAM_MPI_LIBBIN target location.
  This will simply default to FOAM_LIBBIN/FOAM_MPI, but allows different
  types of builds with out-of-tree targets.

Example,

Build OpenFOAM with default MPI settings (eg, system openmpi)
```
./Allwmake
```

Rebuild MPI-layers with different MPI.

For any given and known type
```
othermpi()
{
    export WM_MPLIB=OPENMPI
    export FOAM_MPI=openmpi-3.1.3

    export OPAL_PREFIX=$WM_THIRD_PARTY_DIR/platforms/$WM_ARCH$WM_COMPILER/$FOAM_MPI
    export MPI_ARCH_PATH="$OPAL_PREFIX"
}
```

May wish to build/rebuild ptscotch
```
(
    othermpi
    $WM_THIRD_PARTY_DIR/makeSCOTCH
)
```

Rebuild the mpi-dependent parts. Can also provide install location
```
for script in $(find src -name Allwmake-mpi)
do
(
    other_mpi
    export FOAM_MPI_LIBBIN="$HOME/tmp/install-prefix/lib/$FOAM_MPI"
)
done
```

src/Pstream/Allwclean

@@ -1,11 +1,10 @@
 #!/bin/sh
 cd "${0%/*}" || exit                                # Run from this directory
-. ${WM_PROJECT_DIR:?}/wmake/scripts/wmakeFunctions  # Require wmake functions
 
 #------------------------------------------------------------------------------
 
 wclean dummy
 
-wcleanLibMpi mpi
+./Allwclean-mpi
 
 #------------------------------------------------------------------------------

src/Pstream/Allwclean-mpi

@@ -0,0 +1,9 @@
+#!/bin/sh
+cd "${0%/*}" || exit                                # Run from this directory
+. ${WM_PROJECT_DIR:?}/wmake/scripts/wmakeFunctions  # Require wmake functions
+
+#------------------------------------------------------------------------------
+
+wcleanLibMpi mpi
+
+#------------------------------------------------------------------------------

src/Pstream/Allwmake

@@ -1,13 +1,12 @@
 #!/bin/sh
 cd "${0%/*}" || exit                                # Run from this directory
 . ${WM_PROJECT_DIR:?}/wmake/scripts/AllwmakeParseArguments
-. ${WM_PROJECT_DIR:?}/wmake/scripts/wmakeFunctions  # Require wmake functions
 
 #------------------------------------------------------------------------------
 
 echo "wmake $targetType dummy (mpi=$WM_MPLIB)"
 wmake $targetType dummy
 
-wmakeLibMpi mpi
+./Allwmake-mpi $targetType $*
 
 #------------------------------------------------------------------------------

src/Pstream/Allwmake-mpi

@@ -0,0 +1,13 @@
+#!/bin/sh
+cd "${0%/*}" || exit                                # Run from this directory
+# No AllwmakeParseArguments [infinite loop via Allwmake]
+. ${WM_PROJECT_DIR:?}/wmake/scripts/wmake.wmake-args
+. ${WM_PROJECT_DIR:?}/wmake/scripts/wmakeFunctions  # Require wmake functions
+
+#------------------------------------------------------------------------------
+# Environment
+#   - FOAM_MPI_LIBBIN (optional: defaults to FOAM_LIBBIN/FOAM_MPI)
+
+wmakeLibMpi mpi
+
+#------------------------------------------------------------------------------

src/Pstream/mpi/Make/files

@@ -3,4 +3,4 @@ UIPread.C
 UPstream.C
 PstreamGlobals.C
 
-LIB = $(FOAM_LIBBIN)/$(FOAM_MPI)/libPstream
+LIB = $(FOAM_MPI_LIBBIN)/libPstream

src/Pstream/mpi/Make/options

@@ -1,6 +1,11 @@
 sinclude $(GENERAL_RULES)/mplib$(WM_MPLIB)
 sinclude $(DEFAULT_RULES)/mplib$(WM_MPLIB)
 
+/* Default is PROJECT LIB/mpi target */
+ifeq (,$(FOAM_MPI_LIBBIN))
+    FOAM_MPI_LIBBIN := $(FOAM_LIBBIN)/$(FOAM_MPI)
+endif
+
 EXE_INC  = $(PFLAGS) $(PINC) $(c++LESSWARN)
 
 LIB_LIBS = $(PLIBS)

src/parallel/decompose/Allwclean

@@ -1,6 +1,5 @@
 #!/bin/sh
 cd "${0%/*}" || exit                                # Run from this directory
-. ${WM_PROJECT_DIR:?}/wmake/scripts/wmakeFunctions  # Require wmake functions
 
 #------------------------------------------------------------------------------
 
@@ -10,6 +9,6 @@ wclean scotchDecomp
 wclean decompositionMethods
 wclean decompose
 
-wcleanLibMpi ptscotchDecomp
+./Allwclean-mpi
 
 #------------------------------------------------------------------------------

src/parallel/decompose/Allwclean-mpi

@@ -0,0 +1,9 @@
+#!/bin/sh
+cd "${0%/*}" || exit                                # Run from this directory
+. ${WM_PROJECT_DIR:?}/wmake/scripts/wmakeFunctions  # Require wmake functions
+
+#------------------------------------------------------------------------------
+
+wcleanLibMpi ptscotchDecomp
+
+#------------------------------------------------------------------------------

src/parallel/decompose/Allwmake

@@ -1,7 +1,6 @@
 #!/bin/sh
 cd "${0%/*}" || exit                                # Run from this directory
 . ${WM_PROJECT_DIR:?}/wmake/scripts/AllwmakeParseArguments
-. ${WM_PROJECT_DIR:?}/wmake/scripts/wmakeFunctions  # Require wmake functions
 . ${WM_PROJECT_DIR:?}/wmake/scripts/have_kahip
 . ${WM_PROJECT_DIR:?}/wmake/scripts/have_metis
 . ${WM_PROJECT_DIR:?}/wmake/scripts/have_scotch
@@ -27,12 +26,8 @@ fi
 if have_scotch
 then
     wmake $targetType scotchDecomp
-
-    if have_ptscotch
-    then
-        wmakeLibMpi ptscotchDecomp "scotch=$SCOTCH_VERSION"
-    fi
 fi
 
+./Allwmake-mpi $targetType $*
 
 #------------------------------------------------------------------------------

src/parallel/decompose/Allwmake-mpi

@@ -0,0 +1,24 @@
+#!/bin/sh
+cd "${0%/*}" || exit                                # Run from this directory
+# No AllwmakeParseArguments [infinite loop via Allwmake]
+. ${WM_PROJECT_DIR:?}/wmake/scripts/wmake.wmake-args
+. ${WM_PROJECT_DIR:?}/wmake/scripts/wmakeFunctions  # Require wmake functions
+. ${WM_PROJECT_DIR:?}/wmake/scripts/have_scotch
+
+#------------------------------------------------------------------------------
+# Environment
+#   - FOAM_MPI_LIBBIN (optional: defaults to FOAM_LIBBIN/FOAM_MPI)
+
+: "${FOAM_EXT_LIBBIN:=/usr/lib$WM_COMPILER_LIB_ARCH}"  # Extra safety?
+export FOAM_EXT_LIBBIN
+
+if have_scotch
+then
+    if have_ptscotch
+    then
+        wmakeLibMpi ptscotchDecomp "scotch=$SCOTCH_VERSION"
+    fi
+fi
+
+
+#------------------------------------------------------------------------------

src/parallel/decompose/ptscotchDecomp/Make/files

@@ -1,3 +1,3 @@
 ptscotchDecomp.C
 
-LIB = $(FOAM_LIBBIN)/$(FOAM_MPI)/libptscotchDecomp
+LIB = $(FOAM_MPI_LIBBIN)/libptscotchDecomp

src/parallel/decompose/ptscotchDecomp/Make/options

@@ -5,6 +5,11 @@
 sinclude $(GENERAL_RULES)/mplib$(WM_MPLIB)
 sinclude $(DEFAULT_RULES)/mplib$(WM_MPLIB)
 
+/* Default is PROJECT LIB/mpi target */
+ifeq (,$(strip $(FOAM_MPI_LIBBIN)))
+    FOAM_MPI_LIBBIN := $(FOAM_LIBBIN)/$(FOAM_MPI)
+endif
+
 EXE_INC = \
     -I$(PTSCOTCH_INC_DIR) \
     -I$(SCOTCH_INC_DIR) \
@@ -22,8 +27,8 @@ LIB_LIBS = \
     -lscotch \
     -lptscotch
 
-/* Combined into libptscotch for windows compile */
+/* errexit, except for windows compile (already in library) */
-ifneq ($(WM_OSTYPE),MSwindows)
+ifeq (,$(findstring windows,$(WM_OSTYPE)))
     LIB_LIBS += -lptscotcherrexit
 endif
 

src/parallel/decompose/scotchDecomp/Make/options

@@ -7,8 +7,8 @@ LIB_LIBS = \
     -L$(SCOTCH_LIB_DIR) \
     -lscotch
 
-/* Combined into libscotch for windows compile */
+/* errexit, except for windows compile (already in library) */
-ifneq ($(WM_OSTYPE),MSwindows)
+ifeq (,$(findstring windows,$(WM_OSTYPE)))
     LIB_LIBS += -lscotcherrexit
 endif