mirror of
https://develop.openfoam.com/Development/openfoam.git
synced 2025-11-28 03:28:01 +00:00
wmakeCollect: Experimental scheduler for wmake to speed-up parallel compilations
wmakeCollect collects the compilation commands for the all of the object
files to be compiled into a single makefile which is passed to make to
schedule the compilations in parallel efficiently.
Before wmakeCollect can be called the lnInclude directories must be
up-to-date and after wmakeCollect the linkage stage of the compilation
must executed using wmake.
This entire process is now handled by wmake using the new '-queue' or
'-q' option to compile sections of the OpenFOAM source tree or the
entire tree efficiently. The number of cores the compilation executes
on may be specified either using the WM_NCOMPPROCS variable or the '-j'
option.
To efficiently compile OpenFOAM after a 'git pull' the '-update' option
is provided which updates lnInclude directories, dep files and removes
deprecated files and directories. This option may be used with '-q':
wmake -q -update
This commit is contained in:
Henry Weller
80
wmake/wmake
80
wmake/wmake
@ -49,7 +49,7 @@
|
||||
# wdep -h
|
||||
#
|
||||
# See also
|
||||
# wmakeLnInclude, wmakeLnIncludeAll, wdep, wrmdep, wrmo,
|
||||
# wmakeLnInclude, wmakeLnIncludeAll, wmakeQueue, wdep, wrmdep, wrmo,
|
||||
# wclean, wcleanPlatform, wcleanLnIncludeAll
|
||||
#
|
||||
#------------------------------------------------------------------------------
|
||||
@ -64,8 +64,15 @@ Usage: $Script [OPTION] [dir]
|
||||
$Script [OPTION] target [dir [MakeDir]]
|
||||
|
||||
options:
|
||||
-s | -silent Run make in 'silent' mode (do not echo commands)
|
||||
-a | -all Run wmake in all subdirectories
|
||||
-s | -silent 'silent' mode (does not echo commands)
|
||||
-a | -all wmake all sub-directories, running Allwmake if present
|
||||
-q | -queue wmakeQueue all sub-directories, running Allwmake if present
|
||||
-k or -non-stop Compile without stopping when errors occur
|
||||
-j Compile using all local cores/hyperthreads
|
||||
-jN or -j N Compile using N cores/hyperthreads
|
||||
-no-scheduler Compile without wmakeScheduler
|
||||
-update Update lnInclude directories, dep files, remove deprecated
|
||||
files and directories
|
||||
-h | -help Print the usage
|
||||
|
||||
|
||||
@ -76,7 +83,8 @@ The 'target' is a Makefile target:
|
||||
e.g., platforms/linux64GccDPOpt/.../fvMesh.o
|
||||
|
||||
or a special target:
|
||||
all wmake all subdirectories, running Allwmake files if present
|
||||
all wmake all sub-directories, running Allwmake if present
|
||||
queue wmakeQueue all sub-directories, running Allwmake if present
|
||||
exe Compile statically linked executable
|
||||
lib Compile statically linked archive lib (.a)
|
||||
libo Compile statically linked lib (.o)
|
||||
@ -104,16 +112,56 @@ all=
|
||||
while [ "$#" -gt 0 ]
|
||||
do
|
||||
case "$1" in
|
||||
# Print help
|
||||
-h | -help)
|
||||
usage
|
||||
;;
|
||||
-s | -silent)
|
||||
make="$make -s"
|
||||
shift
|
||||
;;
|
||||
-a | -all | all)
|
||||
all="all"
|
||||
;;
|
||||
-q | -queue | queue)
|
||||
all="queue"
|
||||
;;
|
||||
# Parallel compilation on all cores of local machine
|
||||
-j)
|
||||
setWM_NCOMPPROCS
|
||||
test $# -ge 2 && expr $2 + 1 > /dev/null 2>&1 \
|
||||
&& shift && export WM_NCOMPPROCS=$1
|
||||
echo "Compiling enabled on $WM_NCOMPPROCS cores"
|
||||
;;
|
||||
# Parallel compilation on specified number of cores
|
||||
-j*)
|
||||
export WM_NCOMPPROCS=${1#-j}
|
||||
echo "Compiling enabled on $WM_NCOMPPROCS cores"
|
||||
;;
|
||||
# Non-stop compilation, ignoring errors
|
||||
-k | -non-stop)
|
||||
export WM_CONTINUE_ON_ERROR=1
|
||||
;;
|
||||
# Disable scheduled parallel compilation
|
||||
-no-scheduler)
|
||||
unset WM_SCHEDULER
|
||||
;;
|
||||
# Meant to be used following a pull, this will:
|
||||
# - remove dep files that depend on deleted files;
|
||||
# - remove stale dep files;
|
||||
# - update lnInclude directories;
|
||||
# - remove empty directories, along with deprecated object directories
|
||||
# and respective binaries.
|
||||
-update)
|
||||
wrmdep -update
|
||||
wrmdep -old
|
||||
wmakeLnIncludeAll
|
||||
wclean empty
|
||||
export WM_UPDATE_DEPENDENCIES=yes
|
||||
[ -z "$all" ] && all="all"
|
||||
;;
|
||||
--)
|
||||
shift
|
||||
break
|
||||
;;
|
||||
-*)
|
||||
usage "unknown option: '$*'"
|
||||
@ -122,6 +170,7 @@ do
|
||||
break
|
||||
;;
|
||||
esac
|
||||
shift
|
||||
done
|
||||
|
||||
|
||||
@ -252,6 +301,27 @@ then
|
||||
fi
|
||||
|
||||
|
||||
#------------------------------------------------------------------------------
|
||||
# Recurse the source tree to compile "all" targets using wmakeQueue
|
||||
#------------------------------------------------------------------------------
|
||||
|
||||
#scheduler="wmakeQueue"
|
||||
scheduler="wmakeCollect"
|
||||
|
||||
if [ "$all" = "queue" ]
|
||||
then
|
||||
( \
|
||||
wmakeLnIncludeAll -j$WM_NCOMPPROCS \
|
||||
&& WM_ID=$(stat --format=%d.%i $PWD) \
|
||||
WM_SCHEDULER=$scheduler \
|
||||
trap '$scheduler -kill' TERM INT; \
|
||||
wmake -all objects \
|
||||
&& $scheduler \
|
||||
) && wmake -all
|
||||
exit $?
|
||||
fi
|
||||
|
||||
|
||||
#------------------------------------------------------------------------------
|
||||
# Search up the directory tree for the Make sub-directory,
|
||||
# check the existance of the 'files' file and build there if present
|
||||
|
||||
160
wmake/wmakeCollect
Executable file
160
wmake/wmakeCollect
Executable file
@ -0,0 +1,160 @@
|
||||
#!/bin/bash
|
||||
#------------------------------------------------------------------------------
|
||||
# ========= |
|
||||
# \\ / F ield | OpenFOAM: The Open Source CFD Toolbox
|
||||
# \\ / O peration |
|
||||
# \\ / A nd | Copyright (C) 2016 OpenFOAM Foundation
|
||||
# \\/ M anipulation |
|
||||
#-------------------------------------------------------------------------------
|
||||
# License
|
||||
# This file is part of OpenFOAM.
|
||||
#
|
||||
# OpenFOAM is free software: you can redistribute it and/or modify it
|
||||
# under the terms of the GNU General Public License as published by
|
||||
# the Free Software Foundation, either version 3 of the License, or
|
||||
# (at your option) any later version.
|
||||
#
|
||||
# OpenFOAM is distributed in the hope that it will be useful, but WITHOUT
|
||||
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
|
||||
# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
|
||||
# for more details.
|
||||
#
|
||||
# You should have received a copy of the GNU General Public License
|
||||
# along with OpenFOAM. If not, see <http://www.gnu.org/licenses/>.
|
||||
#
|
||||
# Script
|
||||
# wmakeCollect
|
||||
#
|
||||
# Usage
|
||||
# wmakeCollect [OPTION] <command>
|
||||
#
|
||||
# Description
|
||||
# wmake scheduler for efficient parallel compilations.
|
||||
#
|
||||
#-------------------------------------------------------------------------------
|
||||
Script=${0##*/}
|
||||
|
||||
usage() {
|
||||
exec 1>&2
|
||||
while [ "$#" -ge 1 ]; do echo "$1"; shift; done
|
||||
cat<<USAGE
|
||||
|
||||
Usage: $Script [OPTION] <command>
|
||||
|
||||
options:
|
||||
-kill Removes temporary makefiles
|
||||
-h | -help Print the usage
|
||||
|
||||
Collecting scheduler for fast parallel compilation of large numbers of object
|
||||
files. Can be used in compiling OpenFOAM by setting the WM_SCHEDULER variable.
|
||||
|
||||
When called with a compilation command it is written into a file in the
|
||||
directory ~.\$WM_PROJECT/.collect/$WM_ID.
|
||||
|
||||
When called without a command the files in the ~.\$WM_PROJECT/.collect/$WM_ID
|
||||
are combined into a single Makefile which is passed to make to compile all of
|
||||
the object files in parallel.
|
||||
|
||||
Typical usage for compiling OpenFOAM:
|
||||
|
||||
- Ensure all lnInclude directories are up-to-date:
|
||||
wmakeLnIncludeAll
|
||||
|
||||
- Compile all with this scheduler:
|
||||
wmake -queue or wmake -q
|
||||
|
||||
USAGE
|
||||
exit 1
|
||||
}
|
||||
|
||||
# Set true to clean-up file if interupted
|
||||
cleanup=
|
||||
|
||||
while [ "$#" -gt 0 ]
|
||||
do
|
||||
case "$1" in
|
||||
-h | -help)
|
||||
usage
|
||||
;;
|
||||
-kill)
|
||||
cleanup="true"
|
||||
shift
|
||||
;;
|
||||
-*)
|
||||
usage "unknown option: '$*'"
|
||||
break
|
||||
;;
|
||||
*)
|
||||
break
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
|
||||
if [ -z "$WM_ID" ]
|
||||
then
|
||||
echo "$Script error: build ID variable WM_ID not set"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Location of the wmakeCollect makefiles
|
||||
wmakeCollectDir=$HOME/.$WM_PROJECT/.collect
|
||||
|
||||
# Make sure directories exist
|
||||
mkdir -p $wmakeCollectDir
|
||||
|
||||
# Location of the wmakeCollect makefiles for this build
|
||||
collectDir=$wmakeCollectDir/$WM_ID
|
||||
|
||||
# Collected makefile for this build
|
||||
makefile="$wmakeCollectDir/Makefile.$WM_ID"
|
||||
|
||||
|
||||
# Clean-up files and exit
|
||||
if [ -n "$cleanup" ]
|
||||
then
|
||||
rm -rf $collectDir
|
||||
rm -f $makefile
|
||||
exit 0
|
||||
fi
|
||||
|
||||
|
||||
if [ "$#" -gt 0 ]
|
||||
then
|
||||
# Make sure directories exist
|
||||
mkdir -p $collectDir
|
||||
|
||||
# Unique file name for makefile for the current target
|
||||
file="$collectDir/$$_${RANDOM}"
|
||||
|
||||
# The current target
|
||||
object="${@: -1:1}"
|
||||
|
||||
# Add the current target to the list of objects
|
||||
echo "OBJECTS += $object" >> $file
|
||||
|
||||
# Add the build rule for the current target
|
||||
echo "$object: $makefile" >> $file
|
||||
echo -e "\t cd $PWD && \\" >> $file
|
||||
echo -e "\t${@:1:($#-1)} $object" >> $file
|
||||
echo >> $file
|
||||
else
|
||||
if [ -d $collectDir ]
|
||||
then
|
||||
# Collect all the makefiles into a single makefiles for this build
|
||||
cat $collectDir/* > $makefile
|
||||
|
||||
# Add a build rule for all of the targets
|
||||
echo 'all: $(OBJECTS)' >> $makefile
|
||||
|
||||
# Clear out all of the target makefiles
|
||||
rm -rf $collectDir
|
||||
|
||||
# Run make on the collected makefile
|
||||
make -j $WM_NCOMPPROCS -f $makefile all
|
||||
|
||||
rm -f $makefile
|
||||
fi
|
||||
fi
|
||||
|
||||
#------------------------------------------------------------------------------
|
||||
Reference in New Issue
Block a user