wmakeCollect: Experimental scheduler for wmake to speed-up parallel compilations

wmakeCollect collects the compilation commands for the all of the object
files to be compiled into a single makefile which is passed to make to
schedule the compilations in parallel efficiently.

Before wmakeCollect can be called the lnInclude directories must be
up-to-date and after wmakeCollect the linkage stage of the compilation
must executed using wmake.

This entire process is now handled by wmake using the new '-queue' or
'-q' option to compile sections of the OpenFOAM source tree or the
entire tree efficiently.  The number of cores the compilation executes
on may be specified either using the WM_NCOMPPROCS variable or the '-j'
option.

To efficiently compile OpenFOAM after a 'git pull' the '-update' option
is provided which updates lnInclude directories, dep files and removes
deprecated files and directories.  This option may be used with '-q':

    wmake -q -update

wmake/wmake

@@ -49,7 +49,7 @@
 #         wdep -h
 #
 # See also
-#     wmakeLnInclude, wmakeLnIncludeAll, wdep, wrmdep, wrmo,
+#     wmakeLnInclude, wmakeLnIncludeAll, wmakeQueue, wdep, wrmdep, wrmo,
 #     wclean, wcleanPlatform, wcleanLnIncludeAll
 #
 #------------------------------------------------------------------------------
@@ -64,8 +64,15 @@ Usage: $Script [OPTION] [dir]
        $Script [OPTION] target [dir [MakeDir]]
 
 options:
-  -s | -silent      Run make in 'silent' mode (do not echo commands)
+  -s | -silent      'silent' mode (does not echo commands)
-  -a | -all         Run wmake in all subdirectories
+  -a | -all         wmake all sub-directories, running Allwmake if present
+  -q | -queue       wmakeQueue all sub-directories, running Allwmake if present
+  -k or -non-stop   Compile without stopping when errors occur
+  -j                Compile using all local cores/hyperthreads
+  -jN or -j N       Compile using N cores/hyperthreads
+  -no-scheduler     Compile without wmakeScheduler
+  -update           Update lnInclude directories, dep files, remove deprecated
+                    files and directories
   -h | -help        Print the usage
 
 
@@ -76,7 +83,8 @@ The 'target' is a Makefile target:
   e.g., platforms/linux64GccDPOpt/.../fvMesh.o
 
 or a special target:
-  all               wmake all subdirectories, running Allwmake files if present
+  all               wmake all sub-directories, running Allwmake if present
+  queue             wmakeQueue all sub-directories, running Allwmake if present
   exe               Compile statically linked executable
   lib               Compile statically linked archive lib (.a)
   libo              Compile statically linked lib (.o)
@@ -104,24 +112,65 @@ all=
 while [ "$#" -gt 0 ]
 do
     case "$1" in
-    -h | -help)
+        # Print help
-        usage
+        -h | -help)
-        ;;
+            usage
-    -s | -silent)
+            ;;
-        make="$make -s"
+        -s | -silent)
-        shift
+            make="$make -s"
-        ;;
+            ;;
-    -a | -all | all)
+        -a | -all | all)
-        all="all"
+            all="all"
-        shift
+            ;;
-        ;;
+        -q | -queue | queue)
-    -*)
+            all="queue"
-        usage "unknown option: '$*'"
+            ;;
-        ;;
+        # Parallel compilation on all cores of local machine
-    *)
+        -j)
-        break
+            setWM_NCOMPPROCS
-        ;;
+            test $# -ge 2 && expr $2 + 1 > /dev/null 2>&1 \
+                && shift && export WM_NCOMPPROCS=$1
+            echo "Compiling enabled on $WM_NCOMPPROCS cores"
+            ;;
+        # Parallel compilation on specified number of cores
+        -j*)
+            export WM_NCOMPPROCS=${1#-j}
+            echo "Compiling enabled on $WM_NCOMPPROCS cores"
+            ;;
+        # Non-stop compilation, ignoring errors
+        -k | -non-stop)
+            export WM_CONTINUE_ON_ERROR=1
+            ;;
+        # Disable scheduled parallel compilation
+        -no-scheduler)
+            unset WM_SCHEDULER
+            ;;
+        # Meant to be used following a pull, this will:
+        # - remove dep files that depend on deleted files;
+        # - remove stale dep files;
+        # - update lnInclude directories;
+        # - remove empty directories, along with deprecated object directories
+        #   and respective binaries.
+        -update)
+            wrmdep -update
+            wrmdep -old
+            wmakeLnIncludeAll
+            wclean empty
+            export WM_UPDATE_DEPENDENCIES=yes
+            [ -z "$all" ] && all="all"
+            ;;
+        --)
+            shift
+            break
+            ;;
+        -*)
+            usage "unknown option: '$*'"
+            ;;
+        *)
+            break
+            ;;
     esac
+    shift
 done
 
 
@@ -252,6 +301,27 @@ then
 fi
 
 
+#------------------------------------------------------------------------------
+# Recurse the source tree to compile "all" targets using wmakeQueue
+#------------------------------------------------------------------------------
+
+#scheduler="wmakeQueue"
+scheduler="wmakeCollect"
+
+if [ "$all" = "queue" ]
+then
+    (                                             \
+        wmakeLnIncludeAll -j$WM_NCOMPPROCS        \
+     && WM_ID=$(stat --format=%d.%i $PWD)         \
+        WM_SCHEDULER=$scheduler                   \
+        trap '$scheduler -kill' TERM INT;         \
+        wmake -all objects                        \
+     && $scheduler                                \
+    ) && wmake -all
+    exit $?
+fi
+
+
 #------------------------------------------------------------------------------
 # Search up the directory tree for the Make sub-directory,
 # check the existance of the 'files' file and build there if present

wmake/wmakeCollect

@@ -0,0 +1,160 @@
+#!/bin/bash
+#------------------------------------------------------------------------------
+# =========                 |
+# \\      /  F ield         | OpenFOAM: The Open Source CFD Toolbox
+#  \\    /   O peration     |
+#   \\  /    A nd           | Copyright (C) 2016 OpenFOAM Foundation
+#    \\/     M anipulation  |
+#-------------------------------------------------------------------------------
+# License
+#     This file is part of OpenFOAM.
+#
+#     OpenFOAM is free software: you can redistribute it and/or modify it
+#     under the terms of the GNU General Public License as published by
+#     the Free Software Foundation, either version 3 of the License, or
+#     (at your option) any later version.
+#
+#     OpenFOAM is distributed in the hope that it will be useful, but WITHOUT
+#     ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+#     FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+#     for more details.
+#
+#     You should have received a copy of the GNU General Public License
+#     along with OpenFOAM.  If not, see <http://www.gnu.org/licenses/>.
+#
+# Script
+#     wmakeCollect
+#
+# Usage
+#     wmakeCollect [OPTION] <command>
+#
+# Description
+#     wmake scheduler for efficient parallel compilations.
+#
+#-------------------------------------------------------------------------------
+Script=${0##*/}
+
+usage() {
+    exec 1>&2
+    while [ "$#" -ge 1 ]; do echo "$1"; shift; done
+    cat<<USAGE
+
+Usage: $Script [OPTION] <command>
+
+options:
+  -kill             Removes temporary makefiles
+  -h | -help        Print the usage
+
+Collecting scheduler for fast parallel compilation of large numbers of object
+files.  Can be used in compiling OpenFOAM by setting the WM_SCHEDULER variable.
+
+When called with a compilation command it is written into a file in the
+directory ~.\$WM_PROJECT/.collect/$WM_ID.
+
+When called without a command the files in the ~.\$WM_PROJECT/.collect/$WM_ID
+are combined into a single Makefile which is passed to make to compile all of
+the object files in parallel.
+
+Typical usage for compiling OpenFOAM:
+
+  - Ensure all lnInclude directories are up-to-date:
+    wmakeLnIncludeAll
+
+  - Compile all with this scheduler:
+    wmake -queue or wmake -q
+
+USAGE
+    exit 1
+}
+
+# Set true to clean-up file if interupted
+cleanup=
+
+while [ "$#" -gt 0 ]
+do
+    case "$1" in
+    -h | -help)
+        usage
+        ;;
+    -kill)
+        cleanup="true"
+        shift
+        ;;
+    -*)
+        usage "unknown option: '$*'"
+        break
+        ;;
+    *)
+        break
+        ;;
+    esac
+done
+
+
+if [ -z "$WM_ID" ]
+then
+    echo "$Script error: build ID variable WM_ID not set"
+    exit 1
+fi
+
+# Location of the wmakeCollect makefiles
+wmakeCollectDir=$HOME/.$WM_PROJECT/.collect
+
+# Make sure directories exist
+mkdir -p $wmakeCollectDir
+
+# Location of the wmakeCollect makefiles for this build
+collectDir=$wmakeCollectDir/$WM_ID
+
+# Collected makefile for this build
+makefile="$wmakeCollectDir/Makefile.$WM_ID"
+
+
+# Clean-up files and exit
+if [ -n "$cleanup" ]
+then
+    rm -rf $collectDir
+    rm -f $makefile
+    exit 0
+fi
+
+
+if [ "$#" -gt 0 ]
+then
+    # Make sure directories exist
+    mkdir -p $collectDir
+
+    # Unique file name for makefile for the current target
+    file="$collectDir/$$_${RANDOM}"
+
+    # The current target
+    object="${@: -1:1}"
+
+    # Add the current target to the list of objects
+    echo "OBJECTS += $object" >> $file
+
+    # Add the build rule for the current target
+    echo "$object: $makefile" >> $file
+    echo -e "\t cd $PWD && \\" >> $file
+    echo -e "\t${@:1:($#-1)} $object" >> $file
+    echo >> $file
+else
+    if [ -d $collectDir ]
+    then
+        # Collect all the makefiles into a single makefiles for this build
+        cat $collectDir/* > $makefile
+
+        # Add a build rule for all of the targets
+        echo 'all: $(OBJECTS)' >> $makefile
+
+        # Clear out all of the target makefiles
+        rm -rf $collectDir
+
+        # Run make on the collected makefile
+        make -j $WM_NCOMPPROCS -f $makefile all
+
+        rm -f $makefile
+    fi
+fi
+
+#------------------------------------------------------------------------------