zhyang-dev/openfoam
1
0
Fork 0
mirror of https://develop.openfoam.com/Development/openfoam.git synced 2025-11-28 03:28:01 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

wmake : set WM_NCOMPPROCS automatically when both WM_HOSTS and WM_SCHEDULER are set

Browse Source 
This substantially eases enabling/disabling distributed compiling
For example,
 - export WM_HOSTS="host1:N ... hostN:N"  as usual
 - simply set/unset WM_SCHEDULER

etc/aliases.{sh,csh} added aliases for enabling/disabling wmakeScheduler
 - alias wmSchedON='export WM_SCHEDULER=$WM_PROJECT_DIR/wmake/wmakeScheduler'
 - alias wmSchedOFF='unset WM_SCHEDULER'

Removed wmakeLockScheduler, identical to wmakeScheduler except a 'sleep'
This commit is contained in:
Mark Olesen
2008-06-16 11:46:11 +02:00
parent a382bdacea
commit dcb487fd0f
4 changed files with 20 additions and 208 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
etc/aliases.csh
View File

@ -38,6 +38,12 @@ alias wm32 'setenv WM_ARCH_OPTION 32; source $WM_PROJECT_DIR/etc/cshrc'
alias wmSP 'setenv WM_PRECISION_OPTION SP; source $WM_PROJECT_DIR/etc/cshrc'
alias wmDP 'setenv WM_PRECISION_OPTION DP; source $WM_PROJECT_DIR/etc/cshrc'
# Toggle wmakeScheduler on/off
# - also need to set WM_HOSTS
# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~
alias wmSchedON 'setenv WM_SCHEDULER $WM_PROJECT_DIR/wmake/wmakeScheduler'
alias wmSchedOFF 'unsetenv WM_SCHEDULER'
# Change directory aliases
# ~~~~~~~~~~~~~~~~~~~~~~~~
alias src 'cd $FOAM_SRC'

6
etc/aliases.sh
View File

@ -38,6 +38,12 @@ alias wm32='export WM_ARCH_OPTION=32; . $WM_PROJECT_DIR/etc/bashrc'
alias wmSP='export WM_PRECISION_OPTION=SP; . $WM_PROJECT_DIR/etc/bashrc'
alias wmDP='export WM_PRECISION_OPTION=DP; . $WM_PROJECT_DIR/etc/bashrc'
# Toggle wmakeScheduler on/off
# - also need to set WM_HOSTS
# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~
alias wmSchedON='export WM_SCHEDULER=$WM_PROJECT_DIR/wmake/wmakeScheduler'
alias wmSchedOFF='unset WM_SCHEDULER'
# Change directory aliases
# ~~~~~~~~~~~~~~~~~~~~~~~~
alias src='cd $FOAM_SRC'

9
wmake/wmake
View File

@ -89,9 +89,16 @@ done
make="make"
# set WM_NCOMPPROCS automatically when both WM_HOSTS and WM_SCHEDULER are set
if [ -z "$WM_NCOMPPROCS" -a -n "$WM_HOSTS" -a -n "$WM_SCHEDULER" ]
then
WM_NCOMPPROCS=$(wmakeScheduler -count)
[ $? -eq 0 ] || unset WM_NCOMPPROCS
fi
if [ "$WM_NCOMPPROCS" ]
then
if [ "$WM_NCOMPPROCS" -ne 1 -a ! "$MAKEFLAGS" ]
if [ "$WM_NCOMPPROCS" -gt 1 -a ! "$MAKEFLAGS" ]
then
lockDir=$HOME/.wmakeScheduler

207
wmake/wmakeLockScheduler
View File

@ -1,207 +0,0 @@
#!/bin/bash
#------------------------------------------------------------------------------
# ========= |
# \\ / F ield | OpenFOAM: The Open Source CFD Toolbox
# \\ / O peration |
# \\ / A nd | Copyright (C) 1991-2007 OpenCFD Ltd.
# \\/ M anipulation |
#-------------------------------------------------------------------------------
# License
# This file is part of OpenFOAM.
#
# OpenFOAM is free software; you can redistribute it and/or modify it
# under the terms of the GNU General Public License as published by the
# Free Software Foundation; either version 2 of the License, or (at your
# option) any later version.
#
# OpenFOAM is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
# for more details.
#
# You should have received a copy of the GNU General Public License
# along with OpenFOAM; if not, write to the Free Software Foundation,
# Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
#
# Script
# wmakeScheduler
#
# Description
# Scheduler for network distributed compilations using wmake.
# - WM_HOSTS contains a list of hosts and number of concurrent processes
# eg,
# export WM_HOSTS="hostA:1 hostB:2 hostC:1"
# - WM_COLOURS contains a list of colours to cycle through
# export WM_COLOURS="black blue green cyan red magenta yellow"
#
# Sources the relevant cshrc/bashrc if not set.
#
# WM_PROJECT_DIR, WM_PROJECT and WM_PROJECT_VERSION will have been set
# before calling this routine.
# FOAM_INST_DIR may possibly have been set (to find installation)
#
# Usage
# wmakeScheduler COMMAND
# run 'COMMAND' on one of the slots listed in WM_HOSTS
#
# wmakeScheduler -count
# count the total number of slots available in WM_HOSTS
# eg, export WM_NCOMPPROCS=$(wmakeScheduler -count)
#
#-------------------------------------------------------------------------------
lockDir=$HOME/.wmakeScheduler
# fallback - 1 core on current host
: ${WM_HOSTS:=$HOST:1}
# count the total number of slots available and exit
if [ "$1" = "-count" ]
then
expr $(
for slotGroup in $WM_HOSTS
do
n=${slotGroup##*:}
if [ "$n" = "${slotGroup%%:*}" ]; then n=1; fi # missing ':'
echo "+ ${n:-1}"
done)
exit 0
fi
# where to source WM_PROJECT settings in a remote shell
# This code tries to figure out which cshrc or bashrc to execute.
# !! Assumes remote computer running same shell and startup files
# in same location
sourceFoam=false # fallback command
case $SHELL in
*/csh | */tcsh ) # [t]csh vs bash|ksh|sh
shellRc=cshrc
;;
*)
shellRc=bashrc
;;
esac
# check ~/.$WM_PROJECT/$WM_PROJECT_VERSION/
# check ~/.$WM_PROJECT/
# check <installedProject>/etc/
if [ "$WM_PROJECT" ]; then
for i in \
$HOME/.$WM_PROJECT/$WM_PROJECT_VERSION \
$HOME/.$WM_PROJECT \
$WM_PROJECT_DIR/etc \
;
do
if [ -f "$i/$shellRc" ]; then
sourceFoam="$i/$shellRc"
break
fi
done
fi
# Construct test string for remote execution.
# Source WM_PROJECT settings if WM_PROJECT environment not set.
# attempt to preserve the installation directory 'FOAM_INST_DIR'
case $sourceFoam in
*/bashrc)
if [ "$FOAM_INST_DIR" ]; then
sourceFoam='[ "$WM_PROJECT" ] || '"FOAM_INST_DIR=$FOAM_INST_DIR . $sourceFoam"
else
sourceFoam='[ "$WM_PROJECT" ] || '". $sourceFoam"
fi
;;
*/cshrc)
# TODO: csh equivalent to bash code (preserving FOAM_INST_DIR)
sourceFoam='if ( ! $?WM_PROJECT ) source '"$sourceFoam"
;;
esac
# quote double-quotes for remote command line
rcmd=$(echo $* | sed -e s/\"/\'\"\'/g)
## the same, without forking (not ksh, maybe not /bin/sh either)
# rcmd=$(while [ "$#" -gt 0 ]; do echo "${1//\"/'\"'}"; shift; done)
# Convert WM_COLOURS into an array
declare colours
nColours=0
for col in $WM_COLOURS
do
colours[$nColours]=$col
((nColours = $nColours + 1))
done
# Bashism: make pipe fail early. This make sure return value of compilation
# is returned and not of colouring pipe.
set -o pipefail
# Define function to colour output by argument 1
colourPipe(){
if [ "$1" ]; then
(while read line; do setterm -foreground $1; echo "$line" ; done; setterm -foreground default)
else
cat
fi
}
colourIndex=0
while :
do
for slotGroup in $WM_HOSTS
do
# split 'host:N', but catch 'host:' and 'host' too
host=${slotGroup%%:*}
n=${slotGroup##*:}
if [ "$n" = "$host" ]; then n=1; fi # missing ':'
: ${n:=1}
i=0
while [ "$i" -lt "$n" ]
do
lockFile="$lockDir/$host:$i"
if lockfile -r0 "$lockFile" 2>/dev/null; then
# Set colour
colour=${colours[$colourIndex]}
## echo "** host=$host colourIndex=$colourIndex colour=$colour"
if [ "$host" = "$HOST" ]; then
if [ "$colour" ]; then
eval $* 2>&1 | colourPipe $colour
else
eval $*
fi
elif [ -n "$JOB_ID" ]; then
qrsh -inherit -v PWD $host "$rcmd"
else
if [ "$colour" ]; then
ssh $host "$sourceFoam 2>/dev/null; cd $PWD && $rcmd" 2>&1 | colourPipe $colour
else
ssh $host "$sourceFoam 2>/dev/null; cd $PWD && $rcmd"
fi
fi
retval=$?
# Release lock
rm -f "$lockFile" 2>/dev/null
exit $retval
fi
i=$(expr $i + 1)
# Cycle through colours
colourIndex=$(expr $colourIndex + 1)
if (( $colourIndex >= $nColours )); then
colourIndex=0
fi
done
done
done
if [ "$WM_COLOURS" ]; then
setterm -foreground default
fi
#------------------------------------------------------------------------------