ENH: improve separation of OpenFOAM and ThirdParty mpi-libraries

- drop FOAM_MPI_LIBBIN variable in favour of FOAM_MPI from which
  two different locations can be determined.

  Previously there was something like this:

      FOAM_MPI_LIBBIN = $FOAM_LIBBIN/openmpi-1.5.1

  We now instead determine the locations from the FOAM_MPI directly:

      FOAM_MPI=openmpi-1.5.1

          $FOAM_LIBBIN/$FOAM_MPI     -> OpenFOAM  mpi-related
          $FOAM_EXT_LIBBIN/$FOAM_MPI -> ThirdPary mpi-related

ENH: relocate ParaView libraries and plugins back under FOAM_LIBBIN

- as discussed with Mattijs

applications/utilities/postProcessing/graphics/PV3Readers/PV3FoamReader/vtkPV3Foam/Make/files

@@ -9,4 +9,4 @@ vtkPV3FoamMeshZone.C
 vtkPV3FoamUpdateInfo.C
 vtkPV3FoamUtils.C
 
-LIB = $(FOAM_EXT_LIBBIN)/libvtkPV3Foam
+LIB = $(FOAM_LIBBIN)/libvtkPV3Foam

applications/utilities/postProcessing/graphics/PV3Readers/PV3FoamReader/vtkPV3Foam/Make/options

@@ -19,5 +19,5 @@ LIB_LIBS = \
     -lfiniteVolume \
     -lgenericPatchFields \
     -llagrangian \
-    -L$(FOAM_EXT_LIBBIN) -lvtkPV3Readers \
+    -L$(FOAM_LIBBIN) -lvtkPV3Readers \
     $(GLIBS)

applications/utilities/postProcessing/graphics/PV3Readers/PV3blockMeshReader/vtkPV3blockMesh/Make/files

@@ -2,4 +2,4 @@ vtkPV3blockMesh.C
 vtkPV3blockMeshConvert.C
 vtkPV3blockMeshUtils.C
 
-LIB = $(FOAM_EXT_LIBBIN)/libvtkPV3blockMesh
+LIB = $(FOAM_LIBBIN)/libvtkPV3blockMesh

applications/utilities/postProcessing/graphics/PV3Readers/PV3blockMeshReader/vtkPV3blockMesh/Make/options

@@ -10,5 +10,5 @@ EXE_INC = \
 LIB_LIBS = \
     -lmeshTools \
     -lblockMesh \
-    -L$(FOAM_EXT_LIBBIN) -lvtkPV3Readers \
+    -L$(FOAM_LIBBIN) -lvtkPV3Readers \
     $(GLIBS)

applications/utilities/postProcessing/graphics/PV3Readers/vtkPV3Readers/Make/files

@@ -1,3 +1,3 @@
 vtkPV3Readers.C
 
-LIB = $(FOAM_EXT_LIBBIN)/libvtkPV3Readers
+LIB = $(FOAM_LIBBIN)/libvtkPV3Readers

etc/apps/paraview3/bashrc

@@ -2,7 +2,7 @@
 # =========                 |
 # \\      /  F ield         | OpenFOAM: The Open Source CFD Toolbox
 #  \\    /   O peration     |
-#   \\  /    A nd           | Copyright (C) 1991-2010 OpenCFD Ltd.
+#   \\  /    A nd           | Copyright (C) 1991-2011 OpenCFD Ltd.
 #    \\/     M anipulation  |
 #------------------------------------------------------------------------------
 # License
@@ -82,7 +82,7 @@ if [ -r $ParaView_DIR -o -r $paraviewInstDir ]
 then
     export PATH=$ParaView_DIR/bin:$PATH
     export LD_LIBRARY_PATH=$ParaView_DIR/lib/paraview-$ParaView_MAJOR:$LD_LIBRARY_PATH
-    export PV_PLUGIN_PATH=$FOAM_EXT_LIBBIN/paraview-$ParaView_MAJOR
+    export PV_PLUGIN_PATH=$FOAM_LIBBIN/paraview-$ParaView_MAJOR
 
     # add in python libraries if required
     paraviewPython=$ParaView_DIR/Utilities/VTKPythonWrapping

etc/apps/paraview3/cshrc

@@ -2,7 +2,7 @@
 # =========                 |
 # \\      /  F ield         | OpenFOAM: The Open Source CFD Toolbox
 #  \\    /   O peration     |
-#   \\  /    A nd           | Copyright (C) 1991-2010 OpenCFD Ltd.
+#   \\  /    A nd           | Copyright (C) 1991-2011 OpenCFD Ltd.
 #    \\/     M anipulation  |
 #------------------------------------------------------------------------------
 # License
@@ -80,7 +80,7 @@ setenv ParaView_DIR $WM_THIRD_PARTY_DIR/platforms/$WM_ARCH$WM_COMPILER/paraview-
 if ( -r $ParaView_DIR || -r $paraviewInstDir ) then
     setenv PATH ${ParaView_DIR}/bin:${PATH}
     setenv LD_LIBRARY_PATH "${ParaView_DIR}/lib/paraview-${ParaView_MAJOR}:${LD_LIBRARY_PATH}"
-    setenv PV_PLUGIN_PATH $FOAM_EXT_LIBBIN/paraview-${ParaView_MAJOR}
+    setenv PV_PLUGIN_PATH $FOAM_LIBBIN/paraview-${ParaView_MAJOR}
 
     # add in python libraries if required
     set paraviewPython=$ParaView_DIR/Utilities/VTKPythonWrapping

etc/apps/scotch/bashrc

@@ -36,10 +36,5 @@
 export SCOTCH_VERSION=scotch_5.1.11
 export SCOTCH_ARCH_PATH=$WM_THIRD_PARTY_DIR/platforms/$WM_ARCH$WM_COMPILER/$SCOTCH_VERSION
 
-# mpi-qualified include directory
-export SCOTCH_MPI_INCLUDE=$SCOTCH_ARCH_PATH/include/${FOAM_MPI_LIBBIN##*/}
-
-# mpi-qualified lib directory
-export SCOTCH_MPI_LIBBIN=$SCOTCH_ARCH_PATH/lib/${FOAM_MPI_LIBBIN##*/}
 
 # -----------------------------------------------------------------------------

etc/bashrc

@@ -2,7 +2,7 @@
 # =========                 |
 # \\      /  F ield         | OpenFOAM: The Open Source CFD Toolbox
 #  \\    /   O peration     |
-#   \\  /    A nd           | Copyright (C) 1991-2010 OpenCFD Ltd.
+#   \\  /    A nd           | Copyright (C) 1991-2011 OpenCFD Ltd.
 #    \\/     M anipulation  |
 #------------------------------------------------------------------------------
 # License

etc/cshrc

@@ -2,7 +2,7 @@
 # =========                 |
 # \\      /  F ield         | OpenFOAM: The Open Source CFD Toolbox
 #  \\    /   O peration     |
-#   \\  /    A nd           | Copyright (C) 1991-2010 OpenCFD Ltd.
+#   \\  /    A nd           | Copyright (C) 1991-2011 OpenCFD Ltd.
 #    \\/     M anipulation  |
 #------------------------------------------------------------------------------
 # License
@@ -256,9 +256,7 @@ if (! $?MANPATH) setenv MANPATH ''
 
 #- Clean PATH (path)
 set cleaned=`$foamClean "$PATH" "$foamOldDirs"`
-if ( $status == 0 ) then
+if ( $status == 0 ) setenv PATH $cleaned
-    setenv PATH $cleaned
-endif
 
 #- Clean LD_LIBRARY_PATH
 set cleaned=`$foamClean "$LD_LIBRARY_PATH" "$foamOldDirs"`

etc/settings.csh

@@ -2,7 +2,7 @@
 # =========                 |
 # \\      /  F ield         | OpenFOAM: The Open Source CFD Toolbox
 #  \\    /   O peration     |
-#   \\  /    A nd           | Copyright (C) 1991-2010 OpenCFD Ltd.
+#   \\  /    A nd           | Copyright (C) 1991-2011 OpenCFD Ltd.
 #    \\/     M anipulation  |
 #------------------------------------------------------------------------------
 # License
@@ -90,11 +90,10 @@ unsetenv MPFR_ARCH_PATH
 # Location of compiler installation
 # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 if ( ! $?foamCompiler ) then
-then
     foamCompiler=system
     echo "Warning in $WM_PROJECT_DIR/etc/settings.csh:"
     echo "    foamCompiler not set, using '$foamCompiler'"
-fi
+endif
 
 switch ("$foamCompiler")
 case OpenFOAM:
@@ -253,13 +252,13 @@ unset boost_version cgal_version
 # Communications library
 # ~~~~~~~~~~~~~~~~~~~~~~
 
-unsetenv MPI_ARCH_PATH MPI_HOME
+unsetenv MPI_ARCH_PATH MPI_HOME FOAM_MPI_LIBBIN
 
 switch ("$WM_MPLIB")
 case OPENMPI:
-    #set mpi_version=openmpi-1.4.1
+    #setenv FOAM_MPI openmpi-1.4.1
-    set mpi_version=openmpi-1.5.1
+    setenv FOAM_MPI openmpi-1.5.1
-    setenv MPI_ARCH_PATH $WM_THIRD_PARTY_DIR/platforms/$WM_ARCH$WM_COMPILER/$mpi_version
+    setenv MPI_ARCH_PATH $WM_THIRD_PARTY_DIR/platforms/$WM_ARCH$WM_COMPILER/$FOAM_MPI
 
     # Tell OpenMPI where to find its install directory
     setenv OPAL_PREFIX $MPI_ARCH_PATH
@@ -267,14 +266,11 @@ case OPENMPI:
     _foamAddPath    $MPI_ARCH_PATH/bin
     _foamAddLib     $MPI_ARCH_PATH/lib
     _foamAddMan     $MPI_ARCH_PATH/man
-
-    setenv FOAM_MPI_LIBBIN $FOAM_EXT_LIBBIN/$mpi_version
-    unset mpi_version
     breaksw
 
 case SYSTEMOPENMPI:
     # Use the system installed openmpi, get library directory via mpicc
-    set mpi_version=openmpi-system
+    setenv FOAM_MPI openmpi-system
 
     # Set compilation flags here instead of in wmake/rules/../mplibSYSTEMOPENMPI
     setenv PINC "`mpicc --showme:compile`"
@@ -289,25 +285,21 @@ case SYSTEMOPENMPI:
     endif
 
     _foamAddLib     $libDir
-
+    unset libDir
-    setenv FOAM_MPI_LIBBIN $FOAM_EXT_LIBBIN/$mpi_version
-    unset mpi_version libDir
     breaksw
 
 case MPICH:
-    set mpi_version=mpich2-1.1.1p1
+    setenv FOAM_MPI mpich2-1.1.1p1
-    setenv MPI_HOME $WM_THIRD_PARTY_DIR/$mpi_version
+    setenv MPI_HOME $WM_THIRD_PARTY_DIR/$FOAM_MPI
-    setenv MPI_ARCH_PATH $WM_THIRD_PARTY_DIR/platforms/$WM_ARCH$WM_COMPILER/$mpi_version
+    setenv MPI_ARCH_PATH $WM_THIRD_PARTY_DIR/platforms/$WM_ARCH$WM_COMPILER/$FOAM_MPI
 
     _foamAddPath    $MPI_ARCH_PATH/bin
     _foamAddLib     $MPI_ARCH_PATH/lib
     _foamAddMan     $MPI_ARCH_PATH/share/man
-
-    setenv FOAM_MPI_LIBBIN $FOAM_EXT_LIBBIN/$mpi_version
-    unset mpi_version
     breaksw
 
 case MPICH-GM:
+    setenv FOAM_MPI mpich-gm
     setenv MPI_ARCH_PATH /opt/mpi
     setenv MPICH_PATH $MPI_ARCH_PATH
     setenv GM_LIB_PATH /opt/gm/lib64
@@ -315,11 +307,10 @@ case MPICH-GM:
     _foamAddPath    $MPI_ARCH_PATH/bin
     _foamAddLib     $MPI_ARCH_PATH/lib
     _foamAddLib     $GM_LIB_PATH
-
-    setenv FOAM_MPI_LIBBIN $FOAM_EXT_LIBBIN/mpich-gm
     breaksw
 
 case HPMPI:
+    setenv FOAM_MPI hpmpi
     setenv MPI_HOME /opt/hpmpi
     setenv MPI_ARCH_PATH $MPI_HOME
 
@@ -339,23 +330,22 @@ case HPMPI:
         echo Unknown processor type `uname -m` for Linux
         breaksw
     endsw
-
-    setenv FOAM_MPI_LIBBIN $FOAM_EXT_LIBBIN/hpmpi
     breaksw
 
 case GAMMA:
+    setenv FOAM_MPI gamma
     setenv MPI_ARCH_PATH /usr
-    setenv FOAM_MPI_LIBBIN $FOAM_EXT_LIBBIN/gamma
     breaksw
 
 case MPI:
+    setenv FOAM_MPI mpi
     setenv MPI_ARCH_PATH /opt/mpi
-    setenv FOAM_MPI_LIBBIN $FOAM_EXT_LIBBIN/mpi
     breaksw
 
 case FJMPI:
+    setenv FOAM_MPI fjmpi
     setenv MPI_ARCH_PATH /opt/FJSVmpi2
-    setenv FOAM_MPI_LIBBIN $FOAM_EXT_LIBBIN/mpi
+
     _foamAddPath    $MPI_ARCH_PATH/bin
     _foamAddLib     $MPI_ARCH_PATH/lib/sparcv9
     _foamAddLib     /opt/FSUNf90/lib/sparcv9
@@ -363,20 +353,25 @@ case FJMPI:
     breaksw
 
 case QSMPI:
+    setenv FOAM_MPI qsmpi
     setenv MPI_ARCH_PATH /usr/lib/mpi
-    setenv FOAM_MPI_LIBBIN FOAM_LIBBIN/qsmpi
 
     _foamAddPath    $MPI_ARCH_PATH/bin
     _foamAddLib     $MPI_ARCH_PATH/lib
-
     breaksw
 
 default:
-    setenv FOAM_MPI_LIBBIN $FOAM_LIBBIN/dummy
+    setenv FOAM_MPI dummy
     breaksw
 endsw
 
-_foamAddLib $FOAM_MPI_LIBBIN
+
+# add (non-dummy) MPI implementation
+# dummy MPI already added to LD_LIBRARY_PATH and has no external libraries
+if ( "$FOAM_MPI" != dummy ) then
+    _foamAddLib ${FOAM_LIBBIN}/${FOAM_MPI}:${FOAM_EXT_LIBBIN}/${FOAM_MPI}
+endif
+
 
 
 # Set the minimum MPI buffer size (used by all platforms except SGI MPI)

etc/settings.sh

@@ -2,7 +2,7 @@
 # =========                 |
 # \\      /  F ield         | OpenFOAM: The Open Source CFD Toolbox
 #  \\    /   O peration     |
-#   \\  /    A nd           | Copyright (C) 1991-2010 OpenCFD Ltd.
+#   \\  /    A nd           | Copyright (C) 1991-2011 OpenCFD Ltd.
 #    \\/     M anipulation  |
 #------------------------------------------------------------------------------
 # License
@@ -273,13 +273,13 @@ unset boost_version cgal_version
 # Communications library
 # ~~~~~~~~~~~~~~~~~~~~~~
 
-unset MPI_ARCH_PATH MPI_HOME
+unset MPI_ARCH_PATH MPI_HOME FOAM_MPI_LIBBIN
 
 case "$WM_MPLIB" in
 OPENMPI)
-    #mpi_version=openmpi-1.4.1
+    #export FOAM_MPI=openmpi-1.4.1
-    mpi_version=openmpi-1.5.1
+    export FOAM_MPI=openmpi-1.5.1
-    export MPI_ARCH_PATH=$WM_THIRD_PARTY_DIR/platforms/$WM_ARCH$WM_COMPILER/$mpi_version
+    export MPI_ARCH_PATH=$WM_THIRD_PARTY_DIR/platforms/$WM_ARCH$WM_COMPILER/$FOAM_MPI
 
     # Tell OpenMPI where to find its install directory
     export OPAL_PREFIX=$MPI_ARCH_PATH
@@ -287,14 +287,11 @@ OPENMPI)
     _foamAddPath    $MPI_ARCH_PATH/bin
     _foamAddLib     $MPI_ARCH_PATH/lib
     _foamAddMan     $MPI_ARCH_PATH/man
-
-    export FOAM_MPI_LIBBIN=$FOAM_EXT_LIBBIN/$mpi_version
-    unset mpi_version
     ;;
 
 SYSTEMOPENMPI)
     # Use the system installed openmpi, get library directory via mpicc
-    mpi_version=openmpi-system
+    export FOAM_MPI=openmpi-system
 
     # Set compilation flags here instead of in wmake/rules/../mplibSYSTEMOPENMPI
     export PINC="`mpicc --showme:compile`"
@@ -310,25 +307,21 @@ SYSTEMOPENMPI)
     fi
 
     _foamAddLib     $libDir
-
+    unset libDir
-    export FOAM_MPI_LIBBIN=$FOAM_EXT_LIBBIN/$mpi_version
-    unset mpi_version libDir
     ;;
 
 MPICH)
-    mpi_version=mpich2-1.1.1p1
+    export FOAM_MPI=mpich2-1.1.1p1
-    export MPI_HOME=$WM_THIRD_PARTY_DIR/$mpi_version
+    export MPI_HOME=$WM_THIRD_PARTY_DIR/$FOAM_MPI
-    export MPI_ARCH_PATH=$WM_THIRD_PARTY_DIR/platforms/$WM_ARCH$WM_COMPILER/$mpi_version
+    export MPI_ARCH_PATH=$WM_THIRD_PARTY_DIR/platforms/$WM_ARCH$WM_COMPILER/$FOAM_MPI
 
     _foamAddPath    $MPI_ARCH_PATH/bin
     _foamAddLib     $MPI_ARCH_PATH/lib
     _foamAddMan     $MPI_ARCH_PATH/share/man
-
-    export FOAM_MPI_LIBBIN=$FOAM_EXT_LIBBIN/$mpi_version
-    unset mpi_version
     ;;
 
 MPICH-GM)
+    export FOAM_MPI=mpich-gm
     export MPI_ARCH_PATH=/opt/mpi
     export MPICH_PATH=$MPI_ARCH_PATH
     export GM_LIB_PATH=/opt/gm/lib64
@@ -336,11 +329,10 @@ MPICH-GM)
     _foamAddPath    $MPI_ARCH_PATH/bin
     _foamAddLib     $MPI_ARCH_PATH/lib
     _foamAddLib     $GM_LIB_PATH
-
-    export FOAM_MPI_LIBBIN=$FOAM_EXT_LIBBIN/mpich-gm
     ;;
 
 HPMPI)
+    export FOAM_MPI=hpmpi
     export MPI_HOME=/opt/hpmpi
     export MPI_ARCH_PATH=$MPI_HOME
 
@@ -361,23 +353,21 @@ HPMPI)
         echo Unknown processor type `uname -m` for Linux
         ;;
     esac
-
-    export FOAM_MPI_LIBBIN=$FOAM_EXT_LIBBIN/hpmpi
     ;;
 
 GAMMA)
+    export FOAM_MPI=gamma
     export MPI_ARCH_PATH=/usr
-    export FOAM_MPI_LIBBIN=$FOAM_EXT_LIBBIN/gamma
     ;;
 
 MPI)
+    export FOAM_MPI=mpi
     export MPI_ARCH_PATH=/opt/mpi
-    export FOAM_MPI_LIBBIN=$FOAM_EXT_LIBBIN/mpi
     ;;
 
 FJMPI)
+    export FOAM_MPI=fjmpi
     export MPI_ARCH_PATH=/opt/FJSVmpi2
-    export FOAM_MPI_LIBBIN=$FOAM_EXT_LIBBIN/mpi
 
     _foamAddPath    $MPI_ARCH_PATH/bin
     _foamAddLib     $MPI_ARCH_PATH/lib/sparcv9
@@ -386,20 +376,25 @@ FJMPI)
     ;;
 
 QSMPI)
+    export FOAM_MPI=qsmpi
     export MPI_ARCH_PATH=/usr/lib/mpi
-    export FOAM_MPI_LIBBIN=$FOAM_EXT_LIBBIN/qsmpi
 
     _foamAddPath    $MPI_ARCH_PATH/bin
     _foamAddLib     $MPI_ARCH_PATH/lib
-
     ;;
 
 *)
-    export FOAM_MPI_LIBBIN=$FOAM_LIBBIN/dummy
+    export FOAM_MPI=dummy
     ;;
 esac
 
-_foamAddLib $FOAM_MPI_LIBBIN
+# add (non-dummy) MPI implementation
+# dummy MPI already added to LD_LIBRARY_PATH and has no external libraries
+if [ "$FOAM_MPI" != dummy ]
+then
+    _foamAddLib $FOAM_LIBBIN/$FOAM_MPI:$FOAM_EXT_LIBBIN/$FOAM_MPI
+fi
+
 
 
 # Set the minimum MPI buffer size (used by all platforms except SGI MPI)

src/Pstream/gamma/Make/files

@@ -3,4 +3,4 @@ Pstream.C
 OPwrite.C
 IPread.C
 
-LIB = $(FOAM_MPI_LIBBIN)/libPstream
+LIB = $(FOAM_LIBBIN)/$(FOAM_MPI)/libPstream

src/Pstream/mpi/Make/files

@@ -3,4 +3,4 @@ UIPread.C
 UPstream.C
 PstreamGlobals.C
 
-LIB = $(FOAM_MPI_LIBBIN)/libPstream
+LIB = $(FOAM_LIBBIN)/$(FOAM_MPI)/libPstream

src/parallel/decompose/Allwmake

@@ -1,7 +1,7 @@
 #!/bin/sh
 cd ${0%/*} || exit 1    # run from this directory
 
-# get SCOTCH_VERSION, SCOTCH_ARCH_PATH, SCOTCH_MPI_INCLUDE
+# get SCOTCH_VERSION, SCOTCH_ARCH_PATH
 settings=`$WM_PROJECT_DIR/bin/foamEtcFile apps/scotch/bashrc`
 if [ -f "$settings" ]
 then
@@ -19,7 +19,7 @@ wmakeLnInclude decompositionMethods
 wmake libso scotchDecomp
 wmake libso metisDecomp
 
-if [ -d "$FOAM_MPI_LIBBIN" ]
+if [ -d "$FOAM_LIBBIN/$FOAM_MPI" ]
 then
 (
     WM_OPTIONS=${WM_OPTIONS}$WM_MPLIB

src/parallel/decompose/metisDecomp/Make/files

@@ -1,3 +1,3 @@
 metisDecomp.C
 
-LIB = $(FOAM_EXT_LIBBIN)/libmetisDecomp
+LIB = $(FOAM_LIBBIN)/libmetisDecomp

src/parallel/decompose/ptscotchDecomp/Make/files

@@ -1,3 +1,3 @@
 ptscotchDecomp.C
 
-LIB = $(FOAM_MPI_LIBBIN)/libptscotchDecomp
+LIB = $(FOAM_LIBBIN)/$(FOAM_MPI)/libptscotchDecomp

src/parallel/decompose/ptscotchDecomp/Make/options

@@ -3,9 +3,9 @@ sinclude $(RULES)/mplib$(WM_MPLIB)
 
 EXE_INC = \
     $(PFLAGS) $(PINC) \
-    -I$(SCOTCH_MPI_INCLUDE) \
+    -I$(SCOTCH_ARCH_PATH)/include/$(FOAM_MPI) \
     -I/usr/include/scotch \
     -I../decompositionMethods/lnInclude
 
 LIB_LIBS = \
-    -L$(FOAM_MPI_LIBBIN) -lptscotch -lptscotcherrexit -lrt
+    -L$(FOAM_EXT_LIBBIN)/$(FOAM_MPI) -lptscotch -lptscotcherrexit -lrt

src/parallel/decompose/scotchDecomp/Make/files

@@ -1,3 +1,3 @@
 scotchDecomp.C
 
-LIB = $(FOAM_EXT_LIBBIN)/libscotchDecomp
+LIB = $(FOAM_LIBBIN)/libscotchDecomp