MPI configuration: Added support for SYSTEMMPI

Provided by Bruno Santos
Also some general cleaning and update of comment.
Resolves bug-report http://www.openfoam.org/mantisbt/view.php?id=1167

etc/bashrc

@@ -83,8 +83,8 @@ export WM_LABEL_SIZE=32
 export WM_COMPILE_OPTION=Opt
 
 #- MPI implementation:
-#    WM_MPLIB = SYSTEMOPENMPI | OPENMPI | MPICH | MPICH-GM | HPMPI
+#    WM_MPLIB = SYSTEMOPENMPI | OPENMPI | SYSTEMMPI | MPICH | MPICH-GM | HPMPI
-#               | GAMMA | MPI | QSMPI | SGIMPI
+#               | MPI | QSMPI | SGIMPI
 export WM_MPLIB=SYSTEMOPENMPI
 
 #- Operating System:

etc/config/settings.csh

@@ -30,11 +30,11 @@
 #
 #------------------------------------------------------------------------------
 
-# prefix to PATH
+# Prefix to PATH
 alias _foamAddPath 'setenv PATH \!*\:${PATH}'
-# prefix to LD_LIBRARY_PATH
+# Prefix to LD_LIBRARY_PATH
 alias _foamAddLib 'setenv LD_LIBRARY_PATH \!*\:${LD_LIBRARY_PATH}'
-# prefix to MANPATH
+# Prefix to MANPATH
 alias _foamAddMan 'setenv MANPATH \!*\:${MANPATH}'
 
 #------------------------------------------------------------------------------
@@ -136,7 +136,7 @@ endsw
 
 #------------------------------------------------------------------------------
 
-# location of the jobControl directory
+# Location of the jobControl directory
 setenv FOAM_JOB_DIR $WM_PROJECT_INST_DIR/jobControl
 
 # wmake configuration
@@ -145,34 +145,34 @@ setenv WM_LINK_LANGUAGE c++
 setenv WM_LABEL_OPTION Int$WM_LABEL_SIZE
 setenv WM_OPTIONS $WM_ARCH$WM_COMPILER$WM_PRECISION_OPTION$WM_LABEL_OPTION$WM_COMPILE_OPTION
 
-# base executables/libraries
+# Base executables/libraries
 setenv FOAM_APPBIN $WM_PROJECT_DIR/platforms/$WM_OPTIONS/bin
 setenv FOAM_LIBBIN $WM_PROJECT_DIR/platforms/$WM_OPTIONS/lib
 
-# external (ThirdParty) libraries
+# External (ThirdParty) libraries
 setenv FOAM_EXT_LIBBIN $WM_THIRD_PARTY_DIR/platforms/$WM_OPTIONS/lib
 
-# site-specific directory
+# Site-specific directory
 if ( $?WM_PROJECT_SITE ) then
     set siteDir=$WM_PROJECT_SITE
 else
     set siteDir=$WM_PROJECT_INST_DIR/site
 endif
 
-# shared site executables/libraries
+# Shared site executables/libraries
-# similar naming convention as ~OpenFOAM expansion
+# Similar naming convention as ~OpenFOAM expansion
 setenv FOAM_SITE_APPBIN $siteDir/$WM_PROJECT_VERSION/platforms/$WM_OPTIONS/bin
 setenv FOAM_SITE_LIBBIN $siteDir/$WM_PROJECT_VERSION/platforms/$WM_OPTIONS/lib
 
-# user executables/libraries
+# User executables/libraries
 setenv FOAM_USER_APPBIN $WM_PROJECT_USER_DIR/platforms/$WM_OPTIONS/bin
 setenv FOAM_USER_LIBBIN $WM_PROJECT_USER_DIR/platforms/$WM_OPTIONS/lib
 
-# dynamicCode templates
+# DynamicCode templates
 # - default location is the "~OpenFOAM/codeTemplates/dynamicCode" expansion
 # setenv FOAM_CODE_TEMPLATES $WM_PROJECT_DIR/etc/codeTemplates/dynamicCode
 
-# convenience
+# Convenience
 setenv FOAM_APP $WM_PROJECT_DIR/applications
 setenv FOAM_SRC $WM_PROJECT_DIR/src
 setenv FOAM_TUTORIALS $WM_PROJECT_DIR/tutorials
@@ -180,12 +180,12 @@ setenv FOAM_UTILITIES $FOAM_APP/utilities
 setenv FOAM_SOLVERS $FOAM_APP/solvers
 setenv FOAM_RUN $WM_PROJECT_USER_DIR/run
 
-# add wmake to the path - not required for runtime-only environment
+# Add wmake to the path - not required for runtime-only environment
 if ( -d "${WM_DIR}" ) setenv PATH ${WM_DIR}:${PATH}
-# add OpenFOAM scripts to the path
+# Add OpenFOAM scripts to the path
 setenv PATH ${WM_PROJECT_DIR}/bin:${PATH}
 
-# add site-specific scripts to path - only if they exist
+# Add site-specific scripts to path - only if they exist
 if ( -d "$siteDir/bin" ) then                       # generic
     _foamAddPath "$siteDir/bin"
 endif
@@ -254,7 +254,7 @@ case ThirdParty:
         set mpc_version=mpc-1.0.1
         breaksw
     case Clang:
-        # using clang - not gcc
+        # Using clang - not gcc
         setenv WM_CC 'clang'
         setenv WM_CXX 'clang++'
         set clang_version=llvm-3.5.1
@@ -268,7 +268,7 @@ case ThirdParty:
         breaksw
     endsw
 
-    # optional configuration tweaks:
+    # Optional configuration tweaks:
     _foamSource `$WM_PROJECT_DIR/bin/foamEtcFile config/compiler.csh`
 
     if ( $?gcc_version ) then
@@ -284,7 +284,8 @@ case ThirdParty:
             echo
             echo "Warning in $WM_PROJECT_DIR/etc/config/settings.csh:"
             echo "    Cannot find $gccDir installation."
-            echo "    Please install this compiler version or if you wish to use the system compiler,"
+            echo "    Please install this compiler version or if you wish to" \
+                 " use the system compiler,"
             echo "    change the 'foamCompiler' setting to 'system'"
             echo
         endif
@@ -292,19 +293,19 @@ case ThirdParty:
         _foamAddMan     $gccDir/man
         _foamAddPath    $gccDir/bin
 
-        # add compiler libraries to run-time environment
+        # Add compiler libraries to run-time environment
         _foamAddLib     $gccDir/lib$WM_COMPILER_LIB_ARCH
 
-        # add gmp/mpfr libraries to run-time environment
+        # Add gmp/mpfr libraries to run-time environment
         _foamAddLib     $gmpDir/lib
         _foamAddLib     $mpfrDir/lib
 
-        # add mpc libraries (not need for older gcc) to run-time environment
+        # Add mpc libraries (not need for older gcc) to run-time environment
         if ( $?mpc_version ) then
             _foamAddLib     $mpcDir/lib
         endif
 
-        # used by boost/CGAL:
+        # Used by boost/CGAL:
         setenv MPFR_ARCH_PATH $mpfrDir
         setenv GMP_ARCH_PATH $gmpDir
     endif
@@ -319,7 +320,8 @@ case ThirdParty:
             echo
             echo "Warning in $WM_PROJECT_DIR/etc/config/settings.csh:"
             echo "    Cannot find $clangDir installation."
-            echo "    Please install this compiler version or if you wish to use the system compiler,"
+            echo "    Please install this compiler version or if you wish to" \
+                 " use the system compiler,"
             echo "    change the 'foamCompiler' setting to 'system'"
             echo
         endif
@@ -332,7 +334,7 @@ case ThirdParty:
     breaksw
 
 case system:
-    # okay, use system compiler
+    # Use system compiler
     breaksw
 
 default:
@@ -343,7 +345,7 @@ endsw
 
 
 #
-# add c++0x flags for external programs
+# Add c++0x flags for external programs
 #
 if ( $?WM_CXXFLAGS ) then
     switch ("$WM_COMPILER")
@@ -368,7 +370,7 @@ case SYSTEMOPENMPI:
     set libDir=`mpicc --showme:link | sed -e 's/.*-L\([^ ]*\).*/\1/'`
 
     # Bit of a hack: strip off 'lib' and hope this is the path to openmpi
-    # include files and libraries.
+    # Include files and libraries.
     setenv MPI_ARCH_PATH "${libDir:h}"
 
     _foamAddLib     $libDir
@@ -377,7 +379,7 @@ case SYSTEMOPENMPI:
 
 case OPENMPI:
     setenv FOAM_MPI openmpi-1.8.4
-    # optional configuration tweaks:
+    # Optional configuration tweaks:
     _foamSource `$WM_PROJECT_DIR/bin/foamEtcFile config/openmpi.csh`
 
     setenv MPI_ARCH_PATH $WM_THIRD_PARTY_DIR/platforms/$WM_ARCH$WM_COMPILER/$FOAM_MPI
@@ -394,6 +396,50 @@ case OPENMPI:
     _foamAddMan     $MPI_ARCH_PATH/share/man
     breaksw
 
+case SYSTEMMPI:
+    setenv FOAM_MPI mpi-system
+
+    if ( ! ($?MPI_ROOT) ) then
+        echo
+        echo "Warning in $WM_PROJECT_DIR/etc/config/settings.csh:"
+        echo "    Please set the environment variable MPI_ROOT to point to" \
+             " the base folder for the system MPI in use."
+        echo "    Example:"
+        echo
+        echo "        setenv MPI_ROOT /opt/mpi"
+        echo
+    else
+        setenv MPI_ARCH_PATH $MPI_ROOT
+
+        if ( ! ($?MPI_ARCH_FLAGS) ) then
+            echo
+            echo "Warning in $WM_PROJECT_DIR/etc/config/settings.csh:"
+            echo "    MPI_ARCH_FLAGS is not set. Example:"
+            echo
+            echo '        setenv MPI_ARCH_FLAGS "-DOMPI_SKIP_MPICXX"'
+            echo
+        endif
+
+        if ( ! ($?MPI_ARCH_INC) ) then
+            echo
+            echo "Warning in $WM_PROJECT_DIR/etc/config/settings.csh:"
+            echo "    MPI_ARCH_INC is not set. Example:"
+            echo
+            echo '        setenv MPI_ARCH_INC "-I$MPI_ROOT/include"'
+            echo
+        endif
+
+        if ( ! ($?MPI_ARCH_LIBS) ) then
+            echo
+            echo "Warning in $WM_PROJECT_DIR/etc/config/settings.csh:"
+            echo "    MPI_ARCH_LIBS is not set. Example:"
+            echo
+            echo '        setenv MPI_ARCH_LIBS "-L$MPI_ROOT/lib -lmpi"'
+            echo
+        endif
+    endif
+    breaksw
+
 case MPICH:
     setenv FOAM_MPI mpich2-1.1.1p1
     setenv MPI_HOME $WM_THIRD_PARTY_DIR/$FOAM_MPI
@@ -446,11 +492,6 @@ case HPMPI:
     endsw
     breaksw
 
-case GAMMA:
-    setenv FOAM_MPI gamma
-    setenv MPI_ARCH_PATH /usr
-    breaksw
-
 case MPI:
     setenv FOAM_MPI mpi
     setenv MPI_ARCH_PATH /opt/mpi
@@ -540,8 +581,8 @@ default:
 endsw
 
 
-# add (non-dummy) MPI implementation
+# Add (non-dummy) MPI implementation
-# dummy MPI already added to LD_LIBRARY_PATH and has no external libraries
+# Dummy MPI already added to LD_LIBRARY_PATH and has no external libraries
 if ( "$FOAM_MPI" != dummy ) then
     _foamAddLib ${FOAM_LIBBIN}/${FOAM_MPI}:${FOAM_EXT_LIBBIN}/${FOAM_MPI}
 endif
@@ -561,7 +602,7 @@ else
 endif
 
 
-# cleanup environment:
+# Cleanup environment:
 # ~~~~~~~~~~~~~~~~~~~~
 #- keep _foamAddPath _foamAddLib _foamAddMan
 unset minBufferSize

etc/config/settings.sh

@@ -30,7 +30,7 @@
 #
 #------------------------------------------------------------------------------
 
-# prefix to PATH
+# Prefix to PATH
 _foamAddPath()
 {
     while [ $# -ge 1 ]
@@ -40,7 +40,7 @@ _foamAddPath()
     done
 }
 
-# prefix to LD_LIBRARY_PATH
+# Prefix to LD_LIBRARY_PATH
 _foamAddLib()
 {
     while [ $# -ge 1 ]
@@ -50,7 +50,7 @@ _foamAddLib()
     done
 }
 
-# prefix to MANPATH
+# Prefix to MANPATH
 _foamAddMan()
 {
     while [ $# -ge 1 ]
@@ -93,7 +93,8 @@ Linux)
             export WM_LDFLAGS='-m64'
             ;;
         *)
-            echo "Unknown WM_ARCH_OPTION '$WM_ARCH_OPTION', should be 32 or 64" 1>&2
+            echo "Unknown WM_ARCH_OPTION '$WM_ARCH_OPTION', should be 32 or 64"\
+                 1>&2
             ;;
         esac
         ;;
@@ -164,7 +165,7 @@ esac
 
 #------------------------------------------------------------------------------
 
-# location of the jobControl directory
+# Location of the jobControl directory
 export FOAM_JOB_DIR=$WM_PROJECT_INST_DIR/jobControl
 
 # wmake configuration
@@ -173,30 +174,30 @@ export WM_LINK_LANGUAGE=c++
 export WM_LABEL_OPTION=Int$WM_LABEL_SIZE
 export WM_OPTIONS=$WM_ARCH$WM_COMPILER$WM_PRECISION_OPTION$WM_LABEL_OPTION$WM_COMPILE_OPTION
 
-# base executables/libraries
+# Base executables/libraries
 export FOAM_APPBIN=$WM_PROJECT_DIR/platforms/$WM_OPTIONS/bin
 export FOAM_LIBBIN=$WM_PROJECT_DIR/platforms/$WM_OPTIONS/lib
 
-# external (ThirdParty) libraries
+# External (ThirdParty) libraries
 export FOAM_EXT_LIBBIN=$WM_THIRD_PARTY_DIR/platforms/$WM_OPTIONS/lib
 
-# site-specific directory
+# Site-specific directory
 siteDir="${WM_PROJECT_SITE:-$WM_PROJECT_INST_DIR/site}"
 
-# shared site executables/libraries
+# Shared site executables/libraries
-# similar naming convention as ~OpenFOAM expansion
+# Similar naming convention as ~OpenFOAM expansion
 export FOAM_SITE_APPBIN=$siteDir/$WM_PROJECT_VERSION/platforms/$WM_OPTIONS/bin
 export FOAM_SITE_LIBBIN=$siteDir/$WM_PROJECT_VERSION/platforms/$WM_OPTIONS/lib
 
-# user executables/libraries
+# User executables/libraries
 export FOAM_USER_APPBIN=$WM_PROJECT_USER_DIR/platforms/$WM_OPTIONS/bin
 export FOAM_USER_LIBBIN=$WM_PROJECT_USER_DIR/platforms/$WM_OPTIONS/lib
 
-# dynamicCode templates
+# DynamicCode templates
 # - default location is the "~OpenFOAM/codeTemplates/dynamicCode" expansion
 # export FOAM_CODE_TEMPLATES=$WM_PROJECT_DIR/etc/codeTemplates/dynamicCode
 
-# convenience
+# Convenience
 export FOAM_APP=$WM_PROJECT_DIR/applications
 export FOAM_SRC=$WM_PROJECT_DIR/src
 export FOAM_TUTORIALS=$WM_PROJECT_DIR/tutorials
@@ -204,9 +205,9 @@ export FOAM_UTILITIES=$FOAM_APP/utilities
 export FOAM_SOLVERS=$FOAM_APP/solvers
 export FOAM_RUN=$WM_PROJECT_USER_DIR/run
 
-# add wmake to the path - not required for runtime-only environment
+# Add wmake to the path - not required for runtime-only environment
 [ -d "$WM_DIR" ] && PATH=$WM_DIR:$PATH
-# add OpenFOAM scripts to the path
+# Add OpenFOAM scripts to the path
 export PATH=$WM_PROJECT_DIR/bin:$PATH
 
 # add site-specific scripts to path - only if they exist
@@ -292,7 +293,7 @@ OpenFOAM | ThirdParty)
         ;;
     esac
 
-    # optional configuration tweaks:
+    # Optional configuration tweaks:
     _foamSource `$WM_PROJECT_DIR/bin/foamEtcFile config/compiler.sh`
 
     if [ -n "$gcc_version" ]
@@ -307,7 +308,8 @@ OpenFOAM | ThirdParty)
             echo 1>&2
             echo "Warning in $WM_PROJECT_DIR/etc/config/settings.sh:" 1>&2
             echo "    Cannot find $gccDir installation." 1>&2
-            echo "    Please install this compiler version or if you wish to use the system compiler," 1>&2
+            echo "    Please install this compiler version or if you wish to" \
+                 " use the system compiler," 1>&2
             echo "    change the 'foamCompiler' setting to 'system'" 1>&2
             echo
         }
@@ -315,20 +317,20 @@ OpenFOAM | ThirdParty)
         _foamAddMan     $gccDir/man
         _foamAddPath    $gccDir/bin
 
-        # add compiler libraries to run-time environment
+        # Add compiler libraries to run-time environment
         _foamAddLib     $gccDir/lib$WM_COMPILER_LIB_ARCH
 
-        # add gmp/mpfr libraries to run-time environment
+        # Add gmp/mpfr libraries to run-time environment
         _foamAddLib     $gmpDir/lib
         _foamAddLib     $mpfrDir/lib
 
-        # add mpc libraries (not need for older gcc) to run-time environment
+        # Add mpc libraries (not need for older gcc) to run-time environment
         if [ -n "$mpc_version" ]
         then
             _foamAddLib     $mpcDir/lib
         fi
 
-        # used by boost/CGAL:
+        # Used by boost/CGAL:
         export MPFR_ARCH_PATH=$mpfrDir
         export GMP_ARCH_PATH=$gmpDir
     fi
@@ -344,7 +346,8 @@ OpenFOAM | ThirdParty)
             echo 1>&2
             echo "Warning in $WM_PROJECT_DIR/etc/config/settings.sh:" 1>&2
             echo "    Cannot find $clangDir installation." 1>&2
-            echo "    Please install this compiler version or if you wish to use the system compiler," 1>&2
+            echo "    Please install this compiler version or if you wish to" \
+                 " use the system compiler," 1>&2
             echo "    change the 'foamCompiler' setting to 'system'" 1>&2
             echo 1>&2
         }
@@ -355,7 +358,7 @@ OpenFOAM | ThirdParty)
     unset clang_version clangDir
     ;;
 system)
-    # okay, use system compiler
+    # Use system compiler
     ;;
 *)
     echo "Warn: foamCompiler='$foamCompiler' is unsupported" 1>&2
@@ -365,7 +368,7 @@ esac
 
 
 #
-# add c++0x flags for external programs
+# Add c++0x flags for external programs
 #
 if [ -n "$WM_CXXFLAGS" ]
 then
@@ -400,7 +403,7 @@ SYSTEMOPENMPI)
 
 OPENMPI)
     export FOAM_MPI=openmpi-1.8.4
-    # optional configuration tweaks:
+    # Optional configuration tweaks:
     _foamSource `$WM_PROJECT_DIR/bin/foamEtcFile config/openmpi.sh`
 
     export MPI_ARCH_PATH=$WM_THIRD_PARTY_DIR/platforms/$WM_ARCH$WM_COMPILER/$FOAM_MPI
@@ -417,6 +420,54 @@ OPENMPI)
     _foamAddMan     $MPI_ARCH_PATH/share/man
     ;;
 
+SYSTEMMPI)
+    export FOAM_MPI=mpi-system
+
+    if [ -z "$MPI_ROOT" ]
+    then
+        echo 1>&2
+        echo "Warning in $WM_PROJECT_DIR/etc/config/settings.sh:" 1>&2
+        echo "    Please set the environment variable MPI_ROOT to point to" \
+             " the base folder for the system MPI in use." 1>&2
+        echo "    Example:" 1>&2
+        echo 1>&2
+        echo "        export MPI_ROOT=/opt/mpi" 1>&2
+        echo 1>&2
+    else
+        export MPI_ARCH_PATH=$MPI_ROOT
+
+        if [ -z "$MPI_ARCH_FLAGS" ]
+        then
+            echo 1>&2
+            echo "Warning in $WM_PROJECT_DIR/etc/config/settings.sh:" 1>&2
+            echo "    MPI_ARCH_FLAGS is not set. Example:" 1>&2
+            echo 1>&2
+            echo "        export MPI_ARCH_FLAGS=\"-DOMPI_SKIP_MPICXX\"" 1>&2
+            echo 1>&2
+        fi
+
+        if [ -z "$MPI_ARCH_INC" ]
+        then
+            echo 1>&2
+            echo "Warning in $WM_PROJECT_DIR/etc/config/settings.sh:" 1>&2
+            echo "    MPI_ARCH_INC is not set. Example:" 1>&2
+            echo 1>&2
+            echo "        export MPI_ARCH_INC=\"-I\$MPI_ROOT/include\"" 1>&2
+            echo 1>&2
+        fi
+
+        if [ -z "$MPI_ARCH_LIBS" ]
+        then
+            echo 1>&2
+            echo "Warning in $WM_PROJECT_DIR/etc/config/settings.sh:" 1>&2
+            echo "    MPI_ARCH_LIBS is not set. Example:" 1>&2
+            echo 1>&2
+            echo "        export MPI_ARCH_LIBS=\"-L\$MPI_ROOT/lib -lmpi\"" 1>&2
+            echo 1>&2
+        fi
+    fi
+    ;;
+
 MPICH)
     export FOAM_MPI=mpich2-1.1.1p1
     export MPI_HOME=$WM_THIRD_PARTY_DIR/$FOAM_MPI
@@ -470,11 +521,6 @@ HPMPI)
     esac
     ;;
 
-GAMMA)
-    export FOAM_MPI=gamma
-    export MPI_ARCH_PATH=/usr
-    ;;
-
 MPI)
     export FOAM_MPI=mpi
     export MPI_ARCH_PATH=/opt/mpi
@@ -499,7 +545,7 @@ QSMPI)
     ;;
 
 SGIMPI)
-    # no trailing slash
+    # No trailing slash
     [ "${MPI_ROOT%/}" = "${MPI_ROOT}" ] || MPI_ROOT="${MPI_ROOT%/}"
 
     export FOAM_MPI="${MPI_ROOT##*/}"
@@ -508,7 +554,8 @@ SGIMPI)
     if [ ! -d "$MPI_ROOT" -o -z "$MPI_ARCH_PATH" ]
     then
         echo "Warning in $WM_PROJECT_DIR/etc/config/settings.sh:" 1>&2
-        echo "    MPI_ROOT not a valid mpt installation directory or ending in a '/'." 1>&2
+        echo "    MPI_ROOT not a valid mpt installation directory or ending" \
+             " in a '/'." 1>&2
         echo "    Please set MPI_ROOT to the mpt installation directory." 1>&2
         echo "    MPI_ROOT currently set to '$MPI_ROOT'" 1>&2
     fi
@@ -525,7 +572,7 @@ SGIMPI)
     ;;
 
 INTELMPI)
-    # no trailing slash
+    # No trailing slash
     [ "${MPI_ROOT%/}" = "${MPI_ROOT}" ] || MPI_ROOT="${MPI_ROOT%/}"
 
     export FOAM_MPI="${MPI_ROOT##*/}"
@@ -534,7 +581,8 @@ INTELMPI)
     if [ ! -d "$MPI_ROOT" -o -z "$MPI_ARCH_PATH" ]
     then
         echo "Warning in $WM_PROJECT_DIR/etc/config/settings.sh:" 1>&2
-        echo "    MPI_ROOT not a valid mpt installation directory or ending in a '/'." 1>&2
+        echo "    MPI_ROOT not a valid mpt installation directory or ending" \
+             " in a '/'." 1>&2
         echo "    Please set MPI_ROOT to the mpt installation directory." 1>&2
         echo "    MPI_ROOT currently set to '$MPI_ROOT'" 1>&2
     fi
@@ -554,8 +602,8 @@ INTELMPI)
     ;;
 esac
 
-# add (non-dummy) MPI implementation
+# Add (non-dummy) MPI implementation
-# dummy MPI already added to LD_LIBRARY_PATH and has no external libraries
+# Dummy MPI already added to LD_LIBRARY_PATH and has no external libraries
 if [ "$FOAM_MPI" != dummy ]
 then
     _foamAddLib $FOAM_LIBBIN/$FOAM_MPI:$FOAM_EXT_LIBBIN/$FOAM_MPI
@@ -574,7 +622,7 @@ fi
 export MPI_BUFFER_SIZE
 
 
-# cleanup environment:
+# Cleanup environment:
 # ~~~~~~~~~~~~~~~~~~~~
 #keep _foamAddPath _foamAddLib _foamAddMan
 unset foamCompiler minBufferSize

etc/cshrc

@@ -83,8 +83,8 @@ setenv WM_LABEL_SIZE 32
 setenv WM_COMPILE_OPTION Opt
 
 #- MPI implementation:
-#    WM_MPLIB = SYSTEMOPENMPI | OPENMPI | MPICH | MPICH-GM | HPMPI
+#    WM_MPLIB = SYSTEMOPENMPI | OPENMPI | SYSTEMMPI | MPICH | MPICH-GM | HPMPI
-#               | GAMMA | MPI | QSMPI | SGIMPI
+#               | MPI | QSMPI | SGIMPI
 setenv WM_MPLIB SYSTEMOPENMPI
 
 #- Operating System:

src/Pstream/gamma/IPread.C

@@ -1,135 +0,0 @@
-/*---------------------------------------------------------------------------*\
-  =========                 |
-  \\      /  F ield         | OpenFOAM: The Open Source CFD Toolbox
-   \\    /   O peration     |
-    \\  /    A nd           | Copyright (C) 2011-2014 OpenFOAM Foundation
-     \\/     M anipulation  |
--------------------------------------------------------------------------------
-License
-    This file is part of OpenFOAM.
-
-    OpenFOAM is free software: you can redistribute it and/or modify it
-    under the terms of the GNU General Public License as published by
-    the Free Software Foundation, either version 3 of the License, or
-    (at your option) any later version.
-
-    OpenFOAM is distributed in the hope that it will be useful, but WITHOUT
-    ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
-    FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
-    for more details.
-
-    You should have received a copy of the GNU General Public License
-    along with OpenFOAM.  If not, see <http://www.gnu.org/licenses/>.
-
-Description
-    Read token and binary block from IPstream
-
-\*---------------------------------------------------------------------------*/
-
-#include "IPstream.H"
-#include "PstreamGlobals.H"
-
-extern "C"
-{
-    #include <linux/gamma/libgamma.h>
-}
-
-// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
-
-namespace Foam
-{
-
-// * * * * * * * * * * * * * * * * Constructor * * * * * * * * * * * * * * * //
-
-IPstream::IPstream
-(
-    const commsTypes commsType,
-    const int fromProcNo,
-    const label bufSize,
-    streamFormat format,
-    versionNumber version
-)
-:
-    Pstream(commsType, bufSize),
-    Istream(format, version),
-    fromProcNo_(fromProcNo),
-    messageSize_(0)
-{
-    // Blocking read.
-
-    setOpened();
-    setGood();
-
-    if (Pstream::debug)
-    {
-        Pout<< "IPstream::IPstream : Starting receive from " << fromProcNo_
-            << " recvIndex:" << PstreamGlobals::recvIndex[fromProcNo_]
-            << Foam::endl;
-    }
-
-    PstreamGlobals::gammaWait(fromProcNo_);
-
-    label ready = PstreamGlobals::consumeIndex[fromProcNo_];
-    messageSize_ = PstreamGlobals::recvBufLen[ready][fromProcNo_];
-
-    if (!bufSize)
-    {
-        if (Pstream::debug)
-        {
-            Pout<< "IPstream::IPstream : sizing buffer to " << messageSize_
-                << endl;
-        }
-
-        buf_.setSize(messageSize_);
-    }
-
-    PstreamGlobals::copyReceive(fromProcNo_, buf_.begin(), buf_.size());
-
-    if (Pstream::debug)
-    {
-        Pout<< "IPstream::IPstream : Received " << messageSize_
-            << " from " << fromProcNo_
-            << Foam::endl;
-    }
-}
-
-
-// * * * * * * * * * * * * * * * Member Functions  * * * * * * * * * * * * * //
-
-label IPstream::read
-(
-    const commsTypes commsType,
-    const int fromProcNo,
-    char* buf,
-    const std::streamsize bufSize
-)
-{
-    // Blocking read.
-    label messageSize;
-
-    if (Pstream::debug)
-    {
-        Pout<< "IPstream::read : Starting receive from " << fromProcNo
-            << " recvIndex:" << PstreamGlobals::recvIndex[fromProcNo]
-            << Foam::endl;
-    }
-
-    PstreamGlobals::gammaWait(fromProcNo);
-    messageSize = PstreamGlobals::copyReceive(fromProcNo, buf, bufSize);
-
-    if (Pstream::debug)
-    {
-        Pout<< "IPstream::read : Received " << messageSize
-            << " from " << fromProcNo
-            << Foam::endl;
-    }
-
-    return messageSize;
-}
-
-
-// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
-
-} // End namespace Foam
-
-// ************************************************************************* //

src/Pstream/gamma/Make/files

@@ -1,6 +0,0 @@
-PstreamGlobals.C
-Pstream.C
-OPwrite.C
-IPread.C
-
-LIB = $(FOAM_LIBBIN)/$(FOAM_MPI)/libPstream

src/Pstream/gamma/Make/options

@@ -1,5 +0,0 @@
-sinclude $(GENERAL_RULES)/mplib$(WM_MPLIB)
-sinclude $(RULES)/mplib$(WM_MPLIB)
-
-EXE_INC  = $(PFLAGS) $(PINC)
-LIB_LIBS = $(PLIBS)

src/Pstream/gamma/OPwrite.C

@@ -1,183 +0,0 @@
-/*---------------------------------------------------------------------------*\
-  =========                 |
-  \\      /  F ield         | OpenFOAM: The Open Source CFD Toolbox
-   \\    /   O peration     |
-    \\  /    A nd           | Copyright (C) 2011-2014 OpenFOAM Foundation
-     \\/     M anipulation  |
--------------------------------------------------------------------------------
-License
-    This file is part of OpenFOAM.
-
-    OpenFOAM is free software: you can redistribute it and/or modify it
-    under the terms of the GNU General Public License as published by
-    the Free Software Foundation, either version 3 of the License, or
-    (at your option) any later version.
-
-    OpenFOAM is distributed in the hope that it will be useful, but WITHOUT
-    ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
-    FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
-    for more details.
-
-    You should have received a copy of the GNU General Public License
-    along with OpenFOAM.  If not, see <http://www.gnu.org/licenses/>.
-
-Description
-    Write primitive and binary block from OPstream gamma-mpi
-
-\*---------------------------------------------------------------------------*/
-
-#include "OPstream.H"
-#include "PstreamGlobals.H"
-
-extern "C"
-{
-    #include <linux/gamma/libgamma.h>
-}
-
-// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
-
-namespace Foam
-{
-
-// * * * * * * * * * * * * * * Static Data Members * * * * * * * * * * * * * //
-
-// Largest message sent so far. This tracks the size of the receive
-// buffer on the receiving end. Done so we only send out resize messages
-// if necessary
-//! \cond fileScope
-labelList maxSendSize;
-//! \endcond
-
-
-// * * * * * * * * * * * * * * * * Destructor  * * * * * * * * * * * * * * * //
-
-OPstream::~OPstream()
-{
-    if (Pstream::debug)
-    {
-        Pout<< "OPstream::~OPstream() to processor " << toProcNo_
-            << Foam::endl;
-    }
-
-    if
-    (
-       !write
-        (
-            commsType_,
-            toProcNo_,
-            buf_.begin(),
-            bufPosition_
-        )
-    )
-    {
-        FatalErrorIn("OPstream::~OPstream()")
-            << "GAMMA cannot send outgoing message"
-            << Foam::abort(FatalError);
-    }
-}
-
-
-// * * * * * * * * * * * * * * * Member Functions  * * * * * * * * * * * * * //
-
-bool OPstream::write
-(
-    const commsTypes commsType,
-    const int toProcNo,
-    const char* buf,
-    const std::streamsize bufSize
-)
-{
-    if (PstreamGlobals::getSizeFromHeader(buf, bufSize) != -1)
-    {
-        FatalErrorIn("OPstream::write")
-            << "Problem: Trying to send message of size " << bufSize
-            << " that corresponds to the special resizeMessage."
-            << Foam::abort(FatalError);
-    }
-
-    if (maxSendSize.empty())
-    {
-        // Intialize maxSendSize to the initial size of the receive buffers.
-        maxSendSize.setSize(Pstream::nProcs());
-        maxSendSize = PstreamGlobals::initialBufferLen;
-        maxSendSize[Pstream::myProcNo()] = 0;
-
-        if (Pstream::debug)
-        {
-            forAll(maxSendSize, procNo)
-            {
-                Pout<< "OPstream::write() : for toProcNo:" << procNo
-                    << " set maxSendSize to " << maxSendSize[procNo]
-                    << Foam::endl;
-            }
-        }
-    }
-
-    if (Pstream::debug)
-    {
-        Pout<< "OPstream::write() : proc:" << toProcNo
-            << " maxSendSize:" << maxSendSize[toProcNo]
-            << Foam::endl;
-    }
-
-    if (bufSize > maxSendSize[toProcNo])
-    {
-        // Send resize message.
-        if (Pstream::debug)
-        {
-            Pout<< "OPstream::write() : Sending resize message to proc "
-            << toProcNo
-            << " for size:" << bufSize
-            << Foam::endl;
-        }
-
-        PstreamGlobals::setResizeMessage(bufSize);
-        gamma_send_flowctl
-        (
-            toProcNo,
-            reinterpret_cast<char*>(PstreamGlobals::resizeMessage),
-            PstreamGlobals::resizeMessageLen*sizeof(uint64_t)
-        );
-
-        maxSendSize[toProcNo] = bufSize;
-    }
-
-
-    // Do normal send
-    // ~~~~~~~~~~~~~~
-
-    // Note: could be put into allocation of buf.
-    //gamma_mlock(const_cast<char*>(buf), bufSize);
-
-    if (Pstream::debug)
-    {
-        Pout<< "OPstream::write() : Sending to proc " << toProcNo
-            << " bytes:" << bufSize << Foam::endl;
-    }
-
-    gamma_send_flowctl
-    (
-        toProcNo,
-        const_cast<char*>(buf),
-        bufSize
-    );
-
-    //gamma_munlock(const_cast<char*>(buf), bufSize);
-
-    if (Pstream::debug)
-    {
-        Pout<< "OPstream::write() : Sent " << bufSize
-            << " to proc " << toProcNo
-            << Foam::endl;
-    }
-
-
-    return true;
-}
-
-
-// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
-
-} // End namespace Foam
-
-// ************************************************************************* //

src/Pstream/gamma/Pstream.C

@@ -1,474 +0,0 @@
-/*---------------------------------------------------------------------------*\
-  =========                 |
-  \\      /  F ield         | OpenFOAM: The Open Source CFD Toolbox
-   \\    /   O peration     |
-    \\  /    A nd           | Copyright (C) 2011-2012 OpenFOAM Foundation
-     \\/     M anipulation  |
--------------------------------------------------------------------------------
-License
-    This file is part of OpenFOAM.
-
-    OpenFOAM is free software: you can redistribute it and/or modify it
-    under the terms of the GNU General Public License as published by
-    the Free Software Foundation, either version 3 of the License, or
-    (at your option) any later version.
-
-    OpenFOAM is distributed in the hope that it will be useful, but WITHOUT
-    ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
-    FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
-    for more details.
-
-    You should have received a copy of the GNU General Public License
-    along with OpenFOAM.  If not, see <http://www.gnu.org/licenses/>.
-
-Description
-    Pstream for GAMMA
-
-    GAMMA has a (polling) receive handler which gets called every time a
-    received message is complete. Ours stores the length of the currently
-    received message and sets up the next buffer to store the next message
-    in.
-    Note that the pattern between two processors can be
-    - send
-    - receive
-    - receive
-    - send
-    since the first swap might belong to a local exchange and the second to
-    a reduce. Since gamma has to have the receive buffers already set up we
-    have to allocate them big enough. To prevent excessive amounts needed we
-    dynamically resize them (never shrink) by sending special 'resize' messages
-    before sending a largish message.
-
-    Because of this we actually need four receive buffers:
-    - send
-    - receive resize message
-    - receive normal message
-    - receive resize message
-    - receive normal message
-    - send
-
-    The special resize message is a message with a special header which
-    (hopefully) should never appear in normal exchanges (it actually checks
-    for this in the OPstream::send)
-
-\*---------------------------------------------------------------------------*/
-
-#include "Pstream.H"
-#include "PstreamReduceOps.H"
-#include "OSspecific.H"
-#include "PstreamGlobals.H"
-
-#include <cstring>
-#include <cstdlib>
-#include <csignal>
-
-extern "C"
-{
-#   include <linux/gamma/libgamma.h>
-}
-
-
-// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
-
-namespace Foam
-{
-
-// Receive handler to copy out received message length and switch buffers.
-static void handler(void)
-{
-    label current = PstreamGlobals::recvIndex[gamma_active_port];
-
-    List<char>& buf = PstreamGlobals::recvBuf[current][gamma_active_port];
-    label bufLen = PstreamGlobals::recvBufLen[current][gamma_active_port];
-
-    if (bufLen != -1)
-    {
-        FatalErrorIn("Pstream::handler(void)")
-            << "Buffer length not reset : "
-            << bufLen
-            << " when receiving message of size " << gamma_msglen
-            << " from processor " << gamma_active_port << endl
-            << "This means that the existing data has not been consumed yet"
-            << " (by IPstream::read) and means your communication pattern"
-            << " is probably not balanced (a receive for every send)"
-            << endl
-            << "This can happen if you have e.g. gather without scatter."
-            << endl
-            << "A workaround is to increase the depth of the circular"
-            << " receive buffers in PstreamGlobals.H"
-            << abort(FatalError);
-    }
-
-
-    // Some checks
-    if
-    (
-        gamma_msglen < 0
-     || gamma_msglen > buf.size()
-    )
-    {
-        FatalErrorIn("Pstream::handler(void)")
-            << "Received message of size " << gamma_msglen
-            << " from processor " << gamma_active_port
-            << Foam::endl
-            << "but global receive buffer is only of size "
-            << buf.size()
-            << abort(FatalError);
-    }
-
-    // Check for resize message
-    label resizeLen = PstreamGlobals::getSizeFromHeader
-    (
-        buf.begin(),
-        gamma_msglen
-    );
-
-    if (resizeLen != -1)
-    {
-        if (Pstream::debug)
-        {
-            Pout<< "Pstream::handler : Resize message:" << resizeLen
-                << " from proc " << gamma_active_port
-                << " current size:"
-                << PstreamGlobals::getMaxBufSize(gamma_active_port)
-                << Foam::endl;
-        }
-
-        // Saved current buffer.
-        List<char> savedBuf;
-
-        if (resizeLen > PstreamGlobals::getMaxBufSize(gamma_active_port))
-        {
-            if (Pstream::debug)
-            {
-                Pout<< "Pstream::handler :"
-                    << " resizing receive buffer for processor "
-                    << gamma_active_port
-                    << " from "
-                    << PstreamGlobals::getMaxBufSize(gamma_active_port)
-                    << " to " << resizeLen << Foam::endl;
-            }
-
-            // Save the pointer (that gamma knows about) so we can safely
-            // gamma_switch_to_buffer with a valid pointer.
-            // Not sure if necessary but do anyway.
-            savedBuf.transfer(buf);
-
-            // Resize all the buffers
-            forAll(PstreamGlobals::recvBuf, i)
-            {
-                List<char>& chars =
-                    PstreamGlobals::recvBuf[i][gamma_active_port];
-
-//                gamma_munlock(chars.begin(), chars.size());
-                chars.setSize(resizeLen);
-//                gamma_mlock(chars.begin(), chars.size());
-            }
-        }
-
-        // Update length with special value to denote resize was done.
-        PstreamGlobals::recvBufLen[current][gamma_active_port] = -2;
-    }
-    else
-    {
-        // Update length with actual message length
-        PstreamGlobals::recvBufLen[current][gamma_active_port] = gamma_msglen;
-    }
-
-    // Go to next buffer.
-    label next = PstreamGlobals::recvBuf.fcIndex(current);
-    PstreamGlobals::recvIndex[gamma_active_port] = next;
-
-//    gamma_switch_to_buffer
-    gamma_post_recv
-    (
-        gamma_active_port,
-        PstreamGlobals::recvBuf[next][gamma_active_port].begin(),
-        PstreamGlobals::recvBuf[next][gamma_active_port].size()
-    );
-}
-
-
-// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
-
-void Pstream::addValidParOptions(HashTable<string>& validParOptions)
-{
-    validParOptions.insert("np", "");
-    validParOptions.insert("p4pg", "PI file");
-    validParOptions.insert("p4wd", "directory");
-    validParOptions.insert("p4amslave", "");
-    validParOptions.insert("p4yourname", "hostname");
-
-    validParOptions.insert("machinefile", "machine file");
-    validParOptions.insert("GAMMANP", "numProcs");
-    validParOptions.insert("GAMMAHOME", "gamma cwd");
-    validParOptions.insert("GAMMA", "1(enable) or 0(disable)");
-}
-
-
-bool Pstream::init(int& argc, char**& argv)
-{
-    int numprocs = 0;
-
-    string npString("-GAMMANP");
-
-    for (label i = 0; i < argc; i++)
-    {
-        if (argv[i] == npString)
-        {
-            if (i+1 < argc)
-            {
-                numprocs = atoi(argv[i+1]);
-                break;
-            }
-        }
-    }
-
-    // Initialize GAMMA
-    unsigned char smallNumprocs = numprocs;
-
-    gamma_init(smallNumprocs, argc, argv);
-
-    myProcNo_ = gamma_my_node();
-
-    // Make sure printing with prefix.
-    setParRun();
-
-    procIDs_.setSize(numprocs);
-
-    forAll(procIDs_, procNo)
-    {
-        procIDs_[procNo] = procNo;
-    }
-
-
-    // Allocate receive buffers.
-    // ~~~~~~~~~~~~~~~~~~~~~~~~~
-
-    // Make sure each receive buffer is at least large enough to receive
-    // the resize message.
-
-    // Current active buffer
-    PstreamGlobals::recvIndex.setSize(numprocs);
-    PstreamGlobals::recvIndex = 0;
-    PstreamGlobals::consumeIndex.setSize(numprocs);
-    PstreamGlobals::consumeIndex = 0;
-
-    forAll(PstreamGlobals::recvBuf, i)
-    {
-        PstreamGlobals::recvBufLen[i].setSize(numprocs);
-        PstreamGlobals::recvBufLen[i] = -1;
-
-        List<List<char> >& buffers = PstreamGlobals::recvBuf[i];
-
-        buffers.setSize(numprocs);
-        forAll(buffers, procNo)
-        {
-            if (procNo != myProcNo_)
-            {
-                buffers[procNo].setSize(PstreamGlobals::initialBufferLen);
-
-                // Acc. to gamma sources all buffers need to be in memory.
-                // Either locked or "write touched".
-//                gamma_mlock
- //               (
-  //                  buffers[procNo].begin(),
-   //                 buffers[procNo].size()
-    //            );
-            }
-        }
-    }
-
-
-    // Lock the special resize message
-    //    gamma_mlock
-    //    (
-    //       reinterpret_cast<char*>(PstreamGlobals::resizeMessage),
-    //      PstreamGlobals::resizeMessageLen*sizeof(uint64_t)
-    // );
-
-
-    // Attach current receive buffers
-    forAll(procIDs_, procNo)
-    {
-        if (procNo != myProcNo_)
-        {
-            // Buffer index (always 0 at this point)
-            label current = PstreamGlobals::recvIndex[procNo];
-
-            // Current buffer for this processor.
-            List<char>& buf = PstreamGlobals::recvBuf[current][procNo];
-
-            gamma_set_active_port
-            (
-                procNo,             //unsigned short port,
-                procNo,             //unsigned short dest_node,
-                gamma_my_par_pid(), //unsigned char dest_par_pid,
-                myProcNo_,          //unsigned short dest_port,
-                handler,            //callback
-                procNo,             //unsigned short semaphore,
-                GO_BACK,            //unsigned char buffer_kind,
-                buf.begin(),
-                buf.size()
-            );
-        }
-    }
-
-
-    // Make sure all have allocated the ports (so set the receive buffers)
-    gamma_sync();
-
-    Info<< "GAMMA Pstream initialized with:" << nl
-        << "    floatTransfer         : " << floatTransfer << nl
-        << "    nProcsSimpleSum       : " << nProcsSimpleSum << nl
-        << "    scheduledTransfer     : " << Pstream::scheduledTransfer << nl
-        << Foam::endl;
-
-    // Now that nprocs is known construct communication tables.
-    initCommunicationSchedule();
-
-    return true;
-}
-
-
-void Pstream::exit(int errnum)
-{
-    //    gamma_munlockall();
-    gamma_exit();
-    //gamma_abort();
-}
-
-
-void Pstream::abort()
-{
-    Pout<< "**Pstream::abort()**" << endl;
-    // gamma_munlockall();
-    gamma_abort();
-}
-
-
-void reduce(scalar& Value, const sumOp<scalar>& bop)
-{
-    if (!Pstream::parRun())
-    {
-        return;
-    }
-
-    if (Pstream::debug)
-    {
-        Pout<< "**entering Pstream::reduce for " << Value << Foam::endl;
-    }
-
-
-    if (Pstream::master())
-    {
-        for
-        (
-            int slave=Pstream::firstSlave();
-            slave<=Pstream::lastSlave();
-            slave++
-        )
-        {
-            scalar value;
-
-            if
-            (
-               !IPstream::read
-                (
-                    slave,
-                    reinterpret_cast<char*>(&value),    // buf
-                    sizeof(Value)                       // bufSize
-                )
-            )
-            {
-                FatalErrorIn
-                (
-                    "reduce(scalar& Value, const sumOp<scalar>& sumOp)"
-                )   << "IPstream::read failed"
-                    << Foam::abort(FatalError);
-            }
-
-            Value = bop(Value, value);
-        }
-    }
-    else
-    {
-        if
-        (
-           !OPstream::write
-            (
-                Pstream::masterNo(),
-                reinterpret_cast<const char*>(&Value),  // buf
-                sizeof(Value),                          // bufSize
-                false                                   // non-buffered
-            )
-        )
-        {
-            FatalErrorIn
-            (
-                "reduce(scalar& Value, const sumOp<scalar>& sumOp)"
-            )   << "OPstream::write failed"
-                << Foam::abort(FatalError);
-        }
-    }
-
-    if (Pstream::master())
-    {
-        for
-        (
-            int slave=Pstream::firstSlave();
-            slave<=Pstream::lastSlave();
-            slave++
-        )
-        {
-            if
-            (
-               !OPstream::write
-                (
-                    slave,
-                    reinterpret_cast<const char*>(&Value),  // buf
-                    sizeof(Value),                          // bufSize,
-                    false                                   // non-buffered
-                )
-            )
-            {
-                FatalErrorIn
-                (
-                    "reduce(scalar& Value, const sumOp<scalar>& sumOp)"
-                )   << "OPstream::write failed"
-                    << Foam::abort(FatalError);
-            }
-        }
-    }
-    else
-    {
-        if
-        (
-           !IPstream::read
-            (
-                Pstream::masterNo(),
-                reinterpret_cast<char*>(&Value),    // buf
-                sizeof(Value)                       // bufSize
-            )
-        )
-        {
-            FatalErrorIn
-            (
-                "reduce(scalar& Value, const sumOp<scalar>& sumOp)"
-            )   << "IPstream::read failed"
-                << Foam::abort(FatalError);
-        }
-    }
-
-    if (Pstream::debug)
-    {
-        Pout<< "**exiting Pstream::reduce with " << Value << Foam::endl;
-    }
-}
-
-
-// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
-
-} // End namespace Foam
-
-// ************************************************************************* //

src/Pstream/gamma/PstreamGlobals.C

@@ -1,206 +0,0 @@
-/*---------------------------------------------------------------------------*\
-  =========                 |
-  \\      /  F ield         | OpenFOAM: The Open Source CFD Toolbox
-   \\    /   O peration     |
-    \\  /    A nd           | Copyright (C) 2011 OpenFOAM Foundation
-     \\/     M anipulation  |
--------------------------------------------------------------------------------
-License
-    This file is part of OpenFOAM.
-
-    OpenFOAM is free software: you can redistribute it and/or modify it
-    under the terms of the GNU General Public License as published by
-    the Free Software Foundation, either version 3 of the License, or
-    (at your option) any later version.
-
-    OpenFOAM is distributed in the hope that it will be useful, but WITHOUT
-    ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
-    FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
-    for more details.
-
-    You should have received a copy of the GNU General Public License
-    along with OpenFOAM.  If not, see <http://www.gnu.org/licenses/>.
-
-\*---------------------------------------------------------------------------*/
-
-#include "PstreamGlobals.H"
-#include "IOstreams.H"
-#include "Pstream.H"
-
-extern "C" {
-
-#include <linux/gamma/libgamma.h>
-
-}
-
-// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
-
-namespace Foam
-{
-
-// * * * * * * * * * * * * * * * Member Functions  * * * * * * * * * * * * * //
-
-// Receive buffers
-FixedList<List<List<char> >, 4> PstreamGlobals::recvBuf;
-
-// Length of receive buffers
-FixedList<labelList, 4> PstreamGlobals::recvBufLen;
-
-labelList PstreamGlobals::recvIndex;
-labelList PstreamGlobals::consumeIndex;
-
-// These are all signalling nans and probably different from the ones that
-// the fpu might ever generate.
-uint64_t PstreamGlobals::resizeMessage[PstreamGlobals::resizeMessageLen] =
-{
-    0x7ff7ffffffffffABllu,
-    0x7ff7ffffffffffCDllu,
-    0x7ff7ffffffffff12llu,
-    0x7ff7ffffffffff30llu,
-    0x7ff7ffffffffff19llu,
-    0x0000000000000000llu       // this word gets overwritten with the length.
-};
-
-
-// Wrapper around gamma_wait
-void PstreamGlobals::gammaWait(const label procNo)
-{
-    // Last request. Block.
-    gamma_wait(procNo, 1);
-
-    // Currently unconsumed received message
-    label ready = PstreamGlobals::consumeIndex[procNo];
-
-    // Check received length
-    if (PstreamGlobals::recvBufLen[ready][procNo] == -2)
-    {
-        // Was resize message. Consume and rewait (is always followed by
-        // real message)
-
-        if (Pstream::debug)
-        {
-            Pout<< "PstreamGlobals::gammaWait : "
-                << "Resize event. consumeIndex:" << ready
-                << " Restarting receive from " << procNo << endl;
-        }
-        // Consume resize message
-        PstreamGlobals::recvBufLen[ready][procNo] = -1;
-        PstreamGlobals::consumeIndex[procNo] =
-            PstreamGlobals::recvBuf.fcIndex(ready);
-        // And rewait
-        gamma_wait(procNo, 1);
-    }
-}
-
-
-// Copies data from global receive buffer into buf.
-label PstreamGlobals::copyReceive
-(
-    const label procNo,
-    char* buf,
-    const label bufSize
-)
-{
-    // Get the ready buffer
-    label ready = consumeIndex[procNo];
-
-    // Actually received
-    label receivedLen = recvBufLen[ready][procNo];
-
-    if (Pstream::debug)
-    {
-        Pout<< "copyReceive : for proc " << procNo
-            << " copying " << receivedLen << " bytes out of buffer " << ready
-            << endl;
-    }
-
-    if (receivedLen < 0)
-    {
-        FatalErrorIn
-        (
-            "Pstream::copyReceive(const label, char*, const label)"
-        )   << "Illegal message length "
-            << receivedLen
-            << " received from proc " << procNo << " into buffer " << ready
-            << endl
-            << "This is probably caused by receiving more than is actually"
-            << " sent (e.g. gather without scatter)." << endl
-            << abort(FatalError);
-    }
-
-    if (receivedLen > bufSize)
-    {
-        FatalErrorIn
-        (
-            "Pstream::copyReceive(const label, char*, const label)"
-        )   << "buffer ("
-            << bufSize
-            << ") not large enough for incomming message ("
-            << receivedLen << ')'
-            << " received from proc " << procNo << " into buffer " << ready
-            << abort(FatalError);
-    }
-
-    // Copy out of receive buffer
-    memcpy
-    (
-        buf,
-        recvBuf[ready][procNo].begin(),
-        receivedLen
-    );
-    // Release receive buffer
-    recvBufLen[ready][procNo] = -1;
-    // Go to next buffer to consume
-    consumeIndex[procNo] = recvBuf.fcIndex(ready);
-
-    return receivedLen;
-}
-
-
-// Checks whether an incoming message is a resize message. If not returns -1,
-// otherwise returns size read from header.
-label PstreamGlobals::getSizeFromHeader(const char* buf, const label len)
-{
-    if (len != resizeMessageLen*sizeof(uint64_t))
-    {
-        return -1;
-    }
-
-    const uint64_t* dPtr = reinterpret_cast<const uint64_t*>(buf);
-
-    // Check all but the last word
-    for (label i = 0; i < resizeMessageLen-1; i++)
-    {
-        if (*dPtr++ != resizeMessage[i])
-        {
-            return -1;
-        }
-    }
-
-    return *reinterpret_cast<const label*>(dPtr);
-}
-
-
-void PstreamGlobals::setResizeMessage(const label len)
-{
-    reinterpret_cast<label&>(resizeMessage[resizeMessageLen-1]) = len;
-}
-
-
-label PstreamGlobals::getMaxBufSize(const int procNo)
-{
-    label maxSz = 0;
-
-    forAll(recvBuf, i)
-    {
-        maxSz = max(maxSz, recvBuf[i][procNo].size());
-    }
-    return maxSz;
-}
-
-
-// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
-
-} // End namespace Foam
-
-// ************************************************************************* //

src/Pstream/gamma/PstreamGlobals.H

@@ -1,105 +0,0 @@
-/*---------------------------------------------------------------------------*\
-  =========                 |
-  \\      /  F ield         | OpenFOAM: The Open Source CFD Toolbox
-   \\    /   O peration     |
-    \\  /    A nd           | Copyright (C) 2011 OpenFOAM Foundation
-     \\/     M anipulation  |
--------------------------------------------------------------------------------
-License
-    This file is part of OpenFOAM.
-
-    OpenFOAM is free software: you can redistribute it and/or modify it
-    under the terms of the GNU General Public License as published by
-    the Free Software Foundation, either version 3 of the License, or
-    (at your option) any later version.
-
-    OpenFOAM is distributed in the hope that it will be useful, but WITHOUT
-    ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
-    FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
-    for more details.
-
-    You should have received a copy of the GNU General Public License
-    along with OpenFOAM.  If not, see <http://www.gnu.org/licenses/>.
-
-Namespace
-    Foam::PstreamGlobals
-
-Description
-    Global functions and variables for working with parallel streams,
-    but principally for gamma/mpi
-
-SourceFiles
-    PstreamGlobals.C
-
-\*---------------------------------------------------------------------------*/
-
-#ifndef PstreamGlobals_H
-#define PstreamGlobals_H
-
-#include "FixedList.H"
-#include "labelList.H"
-#include "DynamicList.H"
-
-// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
-
-namespace Foam
-{
-
-/*---------------------------------------------------------------------------*\
-                           Class PstreamGlobals Declaration
-\*---------------------------------------------------------------------------*/
-
-namespace PstreamGlobals
-{
-
-//- Block wait for message on port procNo
-void gammaWait(const label procNo);
-
-//- Helper routine to copy out newly received data
-label copyReceive
-(
-    const label procNo,
-    char* buf,
-    const label bufSize
-);
-
-
-//- Receive buffers
-extern FixedList<List<List<char> >, 4> recvBuf;
-
-//- Length of receive buffers
-extern FixedList<labelList, 4> recvBufLen;
-
-//- Currently active buffer in receiving
-extern labelList recvIndex;
-//- Receive buffer that has to be consumed
-extern labelList consumeIndex;
-
-
-//- Special message to signal resizing
-const int resizeMessageLen = 6;
-extern uint64_t resizeMessage[];
-//- Initial buffer length. Should be able to contain the message comfortably.
-const int initialBufferLen = 2*resizeMessageLen*sizeof(uint64_t);
-
-//- Helper routine to check if a message is a resize message.
-//  Returns -1 if not or the new size.
-label getSizeFromHeader(const char* buf, const label len);
-//- Change the resize message to contain the new length
-void setResizeMessage(const label len);
-
-//- Get max size of all receive buffers to procNo
-label getMaxBufSize(const int procNo);
-
-};
-
-
-// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
-
-} // End namespace Foam
-
-// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
-
-#endif
-
-// ************************************************************************* //

wmake/rules/General/mplibSYSTEMMPI

@@ -0,0 +1,5 @@
+# Use the system installed mpi - depends on specific environment variables
+
+PFLAGS     = $(MPI_ARCH_FLAGS)
+PINC       = $(MPI_ARCH_INC)
+PLIBS      = $(MPI_ARCH_LIBS)