From a93a1abec755b5afa30a62674819a3d722704e7c Mon Sep 17 00:00:00 2001
From: Mark Olesen <Mark.Olesen@esi-group.com>
Date: Tue, 11 Dec 2018 18:50:19 +0100
Subject: [PATCH] COMP: define Scotch and Zoltan include dirs before mpi
 directories

- the local third-party headers are to be searched before any MPI
  include directories. This stops the situation where an mpi-specific
  version of ptscotch.h is installed into the MPI directories.

  This "system" version may well have a different SCOTCH_Num size
  (32 bit vs 64 bit) from the third-party library etc.
---
 .../decompose/metisDecomp/metisDecomp.C       | 21 ++++++++++++++-----
 .../decompose/ptscotchDecomp/Make/options     | 11 +++++++---
 .../decompose/ptscotchDecomp/ptscotchDecomp.C |  8 +++++++
 .../decompose/scotchDecomp/Make/options       |  8 -------
 .../decompose/scotchDecomp/scotchDecomp.C     | 17 ++++++++++-----
 src/renumber/zoltanRenumber/Make/options      |  4 ++--
 6 files changed, 46 insertions(+), 23 deletions(-)

diff --git a/src/parallel/decompose/metisDecomp/metisDecomp.C b/src/parallel/decompose/metisDecomp/metisDecomp.C
index 8889b48fb2..9fa36b1489 100644
--- a/src/parallel/decompose/metisDecomp/metisDecomp.C
+++ b/src/parallel/decompose/metisDecomp/metisDecomp.C
@@ -27,12 +27,23 @@ License
 #include "addToRunTimeSelectionTable.H"
 #include "Time.H"
 
-extern "C"
-{
-    #define OMPI_SKIP_MPICXX
-    #include "metis.h"
-}
+// Probably not needed...
+#define MPICH_SKIP_MPICXX
+#define OMPI_SKIP_MPICXX
 
+#include "metis.h"
+
+// Provide a clear error message if we have a size mismatch
+//
+// Metis has an 'idx_t' type, but the IDXTYPEWIDTH define is perhaps
+// more future-proof?
+#ifdef IDXTYPEWIDTH
+static_assert
+(
+    sizeof(Foam::label) == (IDXTYPEWIDTH/8),
+    "sizeof(Foam::label) == (IDXTYPEWIDTH/8), check your metis headers"
+);
+#endif
 
 // * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
 
diff --git a/src/parallel/decompose/ptscotchDecomp/Make/options b/src/parallel/decompose/ptscotchDecomp/Make/options
index 4d8690985a..e505182324 100644
--- a/src/parallel/decompose/ptscotchDecomp/Make/options
+++ b/src/parallel/decompose/ptscotchDecomp/Make/options
@@ -1,10 +1,15 @@
+/*
+ * NB: mplib PINC must appear after the SCOTCH_ARCH_PATH/include/FOAM_MPI
+ * to ensure we do not accidentally get a ptscotch header from the
+ * mpi distribution.
+ */
 sinclude $(GENERAL_RULES)/mplib$(WM_MPLIB)
 sinclude $(DEFAULT_RULES)/mplib$(WM_MPLIB)
 
 EXE_INC = \
-    $(PFLAGS) $(PINC) \
-    -I$(SCOTCH_INC_DIR) \
     -I$(SCOTCH_ARCH_PATH)/include/$(FOAM_MPI) \
+    -I$(SCOTCH_INC_DIR) \
+    $(PFLAGS) $(PINC) \
     -I../decompositionMethods/lnInclude
 
 /*
@@ -13,8 +18,8 @@ EXE_INC = \
  */
 LIB_LIBS = \
     -L$(SCOTCH_LIB_DIR) \
-    -L$(FOAM_EXT_LIBBIN) \
     -L$(FOAM_EXT_LIBBIN)/$(FOAM_MPI) \
+    -L$(FOAM_EXT_LIBBIN) \
     -lptscotch -lptscotcherrexit \
     -lscotch
 
diff --git a/src/parallel/decompose/ptscotchDecomp/ptscotchDecomp.C b/src/parallel/decompose/ptscotchDecomp/ptscotchDecomp.C
index f5c12994c7..56d7a022c5 100644
--- a/src/parallel/decompose/ptscotchDecomp/ptscotchDecomp.C
+++ b/src/parallel/decompose/ptscotchDecomp/ptscotchDecomp.C
@@ -207,6 +207,7 @@ License
 #include "globalIndex.H"
 #include "SubField.H"
 
+// Avoid too many warnings from mpi.h
 #pragma GCC diagnostic ignored "-Wold-style-cast"
 
 #include <cstdio>
@@ -222,6 +223,13 @@ License
     #include <fenv.h>
 #endif
 
+// Provide a clear error message if we have a size mismatch
+static_assert
+(
+    sizeof(Foam::label) == sizeof(SCOTCH_Num),
+    "sizeof(Foam::label) == sizeof(SCOTCH_Num), check your scotch headers"
+);
+
 
 // * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
 
diff --git a/src/parallel/decompose/scotchDecomp/Make/options b/src/parallel/decompose/scotchDecomp/Make/options
index 6846910c04..d4e3013991 100644
--- a/src/parallel/decompose/scotchDecomp/Make/options
+++ b/src/parallel/decompose/scotchDecomp/Make/options
@@ -1,12 +1,4 @@
-/*
- * Note including of mplib compilation rules.
- * This is purely to avoid scotch.h including mpicxx.h, which causes problems.
- */
-sinclude $(GENERAL_RULES)/mplib$(WM_MPLIB)
-sinclude $(DEFAULT_RULES)/mplib$(WM_MPLIB)
-
 EXE_INC = \
-    $(PFLAGS) $(PINC) \
     -I$(SCOTCH_INC_DIR) \
     -I../decompositionMethods/lnInclude
 
diff --git a/src/parallel/decompose/scotchDecomp/scotchDecomp.C b/src/parallel/decompose/scotchDecomp/scotchDecomp.C
index 77067cdce1..a5afaa5a81 100644
--- a/src/parallel/decompose/scotchDecomp/scotchDecomp.C
+++ b/src/parallel/decompose/scotchDecomp/scotchDecomp.C
@@ -127,13 +127,13 @@ License
 #include "Time.H"
 #include "OFstream.H"
 
-extern "C"
-{
+// Probably not needed, but in case we pickup a ptscotch.h ...
+#define MPICH_SKIP_MPICXX
+#define OMPI_SKIP_MPICXX
+
 #include "scotch.h"
-}
 
-
-// Hack: scotch generates floating point errors so need to switch of error
+// Hack: scotch generates floating point errors so need to switch off error
 //       trapping!
 #ifdef __GLIBC__
     #ifndef _GNU_SOURCE
@@ -142,6 +142,13 @@ extern "C"
     #include <fenv.h>
 #endif
 
+// Provide a clear error message if we have a size mismatch
+static_assert
+(
+    sizeof(Foam::label) == sizeof(SCOTCH_Num),
+    "sizeof(Foam::label) == sizeof(SCOTCH_Num), check your scotch headers"
+);
+
 
 // * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
 
diff --git a/src/renumber/zoltanRenumber/Make/options b/src/renumber/zoltanRenumber/Make/options
index 2d1fd2e105..f16794e34d 100644
--- a/src/renumber/zoltanRenumber/Make/options
+++ b/src/renumber/zoltanRenumber/Make/options
@@ -2,11 +2,11 @@ sinclude $(GENERAL_RULES)/mplib$(WM_MPLIB)
 sinclude $(DEFAULT_RULES)/mplib$(WM_MPLIB)
 
 EXE_INC = \
+    -I$(ZOLTAN_INC_DIR) \
     $(PFLAGS) $(PINC) \
     ${c++LESSWARN} \
     -I$(LIB_SRC)/renumber/renumberMethods/lnInclude \
-    -I$(LIB_SRC)/meshTools/lnInclude \
-    -I$(ZOLTAN_INC_DIR)
+    -I$(LIB_SRC)/meshTools/lnInclude
 
 LIB_LIBS = \
     -L$(ZOLTAN_LIB_DIR) -lzoltan \