AllwmakeParseArguments: Moved all parallel processing options into wmake

and added support for queue scheduling option '-q', '-queue'

Now the 'Allwmake' scripts execute 'wmake -all' to handle parallel
processing in a general way, avoiding code duplication.
This commit is contained in:
Henry Weller
2016-07-04 22:30:20 +01:00
parent 8abdc36398
commit af8e0f1443
2 changed files with 70 additions and 109 deletions

View File

@ -28,9 +28,6 @@
# Allwmake argument parser # Allwmake argument parser
# #
# Usage # Usage
# # Declare genDoc and set to default if documentation building is supported
# genDoc=0 # 0 or 1
#
# # Parse the arguments by sourcing this script # # Parse the arguments by sourcing this script
# . $WM_PROJECT_DIR/wmake/scripts/AllwmakeParseArguments # . $WM_PROJECT_DIR/wmake/scripts/AllwmakeParseArguments
# #
@ -46,118 +43,51 @@ usage() {
Usage: $Script [OPTIONS] Usage: $Script [OPTIONS]
options: Executing $Script is equivalent to
-h or -help Print list of Allwmake options
-k or -non-stop Compile without stopping when errors occur wmake -all [OPTIONS]
-j Compile using all local cores/hyperthreads
-jN or -j N Compile using N cores/hyperthreads
-no-scheduler Compile without wmakeScheduler
-update Update lnInclude directories, dep files, remove deprecated
files and directories
USAGE USAGE
# Print options for building code documentation wmake -help
test -n "$genDoc" && cat<<USAGE_DOC
doc Compile code documentation (requires Doxygen)
USAGE_DOC
# Print options for building libraries
cat<<USAGE_LIB
lib Compile statically linked archive lib (.a)
libo Compile statically linked lib (.o)
libso Compile dynamically linked lib (.so)
dep Compile dependency files
objects Compile only
USAGE_LIB
exit 1 exit 1
} }
#------------------------------------------------------------------------------
# Set WM_NCOMPPROCS to number of cores on local machine
#------------------------------------------------------------------------------
setWM_NCOMPPROCS()
{
if [ -r /proc/cpuinfo ]
then
WM_NCOMPPROCS=$(egrep "^processor" /proc/cpuinfo | wc -l)
else
WM_NCOMPPROCS=1
fi
export WM_NCOMPPROCS
}
#------------------------------------------------------------------------------ #------------------------------------------------------------------------------
# Parse the arguments and options # Parse the arguments and options
#------------------------------------------------------------------------------ #------------------------------------------------------------------------------
while [ "$#" -gt 0 ] fromWmake=
for arg in "$@"
do do
case "$1" in case "$arg" in
# Print help
-h | -help) -h | -help)
usage usage
exit 0
;; ;;
# Parallel compilation on all cores of local machine # Check if called from wmake to avoid recusion
-j) -fromWmake)
setWM_NCOMPPROCS fromWmake="fromWmake"
test $# -ge 2 && expr $2 + 1 > /dev/null 2>&1 \
&& shift && export WM_NCOMPPROCS=$1
echo "Compiling enabled on $WM_NCOMPPROCS cores"
;; ;;
# Parallel compilation on specified number of cores # Target type
-j*)
export WM_NCOMPPROCS=${1#-j}
echo "Compiling enabled on $WM_NCOMPPROCS cores"
;;
# Non-stop compilation, ignoring errors
-k | -non-stop)
export WM_CONTINUE_ON_ERROR=1
;;
# Disable scheduled parallel compilation
-no-scheduler)
unset WM_SCHEDULER
;;
# Meant to be used following a pull, this will:
# - remove dep files that depend on deleted files;
# - remove stale dep files;
# - update lnInclude directories;
# - remove empty directories, along with deprecated object directories
# and respective binaries.
-update)
wrmdep -update
wrmdep -old
wmakeLnIncludeAll
wclean empty
# Set WM_UPDATE_DEPENDENCIES, so that wmake will pick up on it
export WM_UPDATE_DEPENDENCIES=yes
;;
# Generate documentation
doc)
test -n "$genDoc" || usage "invalid option '$1'"
genDoc=1
;;
# Specify target type
lib | libo | libso | dep | objects) lib | libo | libso | dep | objects)
targetType=$1 targetType=$arg
;;
--)
shift
break
;;
-* | *)
usage "invalid option '$1'"
;; ;;
esac esac
shift
done done
#------------------------------------------------------------------------------
# Execute wmake -all if not called from wmake
#------------------------------------------------------------------------------
if [ -z "$fromWmake" ]
then
wmake -all $*
fi
#------------------------------------------------------------------------------ #------------------------------------------------------------------------------
# If WM_CONTINUE_ON_ERROR not set activate the shell option "stop on error" # If WM_CONTINUE_ON_ERROR not set activate the shell option "stop on error"
#------------------------------------------------------------------------------ #------------------------------------------------------------------------------
@ -172,7 +102,7 @@ fi
# Cleanup local variables and functions # Cleanup local variables and functions
#------------------------------------------------------------------------------ #------------------------------------------------------------------------------
unset Script usage setWM_NCOMPPROCS unset Script usage fromWmake
#------------------------------------------------------------------------------ #------------------------------------------------------------------------------

View File

@ -102,12 +102,30 @@ USAGE
make="make" make="make"
#------------------------------------------------------------------------------
# Set WM_NCOMPPROCS to number of cores on local machine
#------------------------------------------------------------------------------
useAllCores()
{
if [ -r /proc/cpuinfo ]
then
WM_NCOMPPROCS=$(egrep "^processor" /proc/cpuinfo | wc -l)
else
WM_NCOMPPROCS=1
fi
export WM_NCOMPPROCS
}
#------------------------------------------------------------------------------ #------------------------------------------------------------------------------
# Parse arguments and options # Parse arguments and options
#------------------------------------------------------------------------------ #------------------------------------------------------------------------------
# Default to compiling the local target only # Default to compiling the local target only
all= all=
update=
while [ "$#" -gt 0 ] while [ "$#" -gt 0 ]
do do
@ -127,7 +145,7 @@ do
;; ;;
# Parallel compilation on all cores of local machine # Parallel compilation on all cores of local machine
-j) -j)
setWM_NCOMPPROCS useAllCores
test $# -ge 2 && expr $2 + 1 > /dev/null 2>&1 \ test $# -ge 2 && expr $2 + 1 > /dev/null 2>&1 \
&& shift && export WM_NCOMPPROCS=$1 && shift && export WM_NCOMPPROCS=$1
echo "Compiling enabled on $WM_NCOMPPROCS cores" echo "Compiling enabled on $WM_NCOMPPROCS cores"
@ -152,16 +170,7 @@ do
# - remove empty directories, along with deprecated object directories # - remove empty directories, along with deprecated object directories
# and respective binaries. # and respective binaries.
-update) -update)
wrmdep -update update="true"
wrmdep -old
if [ -n "$WM_NCOMPPROCS" ]
then
wmakeLnIncludeAll -j$WM_NCOMPPROCS
else
wmakeLnIncludeAll
fi
wclean empty
export WM_UPDATE_DEPENDENCIES=yes
[ -z "$all" ] && all="all" [ -z "$all" ] && all="all"
;; ;;
--) --)
@ -264,6 +273,27 @@ then
fi fi
#------------------------------------------------------------------------------
# Recurse the source tree to compile "all" targets
#------------------------------------------------------------------------------
if [ -n "$update" ]
then
wrmdep -update
wrmdep -old
if [ -n "$WM_NCOMPPROCS" ]
then
wmakeLnIncludeAll -j$WM_NCOMPPROCS
else
wmakeLnIncludeAll
fi
wclean empty
export WM_UPDATE_DEPENDENCIES=yes
fi
unset update
#------------------------------------------------------------------------------ #------------------------------------------------------------------------------
# Recurse the source tree to compile "all" targets # Recurse the source tree to compile "all" targets
#------------------------------------------------------------------------------ #------------------------------------------------------------------------------
@ -272,7 +302,7 @@ if [ "$all" = "all" ]
then then
if [ -e Allwmake ] if [ -e Allwmake ]
then then
./Allwmake $targetType ./Allwmake -fromWmake $targetType
exit $? exit $?
else else
# Have to keep track of the main exit code for the call to "make" # Have to keep track of the main exit code for the call to "make"
@ -310,7 +340,6 @@ fi
# Recurse the source tree to compile "all" targets using wmakeQueue # Recurse the source tree to compile "all" targets using wmakeQueue
#------------------------------------------------------------------------------ #------------------------------------------------------------------------------
#scheduler="wmakeQueue"
scheduler="wmakeCollect" scheduler="wmakeCollect"
if [ "$all" = "queue" ] if [ "$all" = "queue" ]
@ -326,6 +355,8 @@ then
exit $? exit $?
fi fi
unset scheduler
#------------------------------------------------------------------------------ #------------------------------------------------------------------------------
# Search up the directory tree for the Make sub-directory, # Search up the directory tree for the Make sub-directory,
@ -496,7 +527,7 @@ exec $make -f $WM_DIR/makefiles/general MAKE_DIR=$MakeDir \
# Cleanup local variables and functions # Cleanup local variables and functions
#------------------------------------------------------------------------------ #------------------------------------------------------------------------------
unset Script usage expandPath findTarget unset Script usage useAllCores update expandPath findTarget
#------------------------------------------------------------------------------ #------------------------------------------------------------------------------