diff --git a/lib/hdnnp/.gitignore b/lib/hdnnp/.gitignore new file mode 100644 index 0000000000..2a6742ed7a --- /dev/null +++ b/lib/hdnnp/.gitignore @@ -0,0 +1,3 @@ +/includelink +/liblink +/n2p2-* diff --git a/lib/hdnnp/Install.py b/lib/hdnnp/Install.py index 3b232f400b..45c14a5aff 100644 --- a/lib/hdnnp/Install.py +++ b/lib/hdnnp/Install.py @@ -10,7 +10,7 @@ import sys, os, platform, subprocess, shutil from argparse import ArgumentParser sys.path.append('..') -from install_helpers import get_cpus, fullpath, geturl, checkmd5sum +from install_helpers import get_cpus, fullpath, geturl, checkmd5sum, getfallback parser = ArgumentParser(prog='Install.py', description="LAMMPS library build wrapper script") @@ -76,14 +76,21 @@ if pathflag: if buildflag: url = "https://github.com/CompPhysVienna/n2p2/archive/v%s.tar.gz" % (version) - filename = "n2p2-%s.tar.gz" %version - print("Downloading n2p2 ...") - geturl(url, filename) + filename = "n2p2-%s.tar.gz" % version + fallback = getfallback('n2p2', url) + print("Downloading n2p2 from", url) + try: + geturl(url, filename) + except: + geturl(fallback, filename) # verify downloaded archive integrity via md5 checksum, if known. if version in checksums: if not checkmd5sum(checksums[version], filename): - sys.exit("Checksum for n2p2 library does not match") + print("Checksum did not match. Trying fallback URL", fallback) + geturl(fallback, filename) + if not checkmd5sum(checksums[version], filename): + sys.exit("Checksum for n2p2 library does not match for fallback, too.") print("Unpacking n2p2 source tarball ...") if os.path.exists("%s/n2p2-%s" % (homepath, version)): diff --git a/lib/install_helpers.py b/lib/install_helpers.py index e1ebd1cde7..59171ee4e0 100644 --- a/lib/install_helpers.py +++ b/lib/install_helpers.py @@ -1,4 +1,7 @@ -import hashlib,os,subprocess +import hashlib +import os +import re +import subprocess # try to auto-detect the maximum number of available CPUs def get_cpus(): @@ -32,31 +35,31 @@ def which(program): return None -def geturl(url,fname): +def geturl(url, fname): success = False if which('curl') != None: - cmd = 'curl -L -o "%s" %s' % (fname,url) + cmd = 'curl -L -o "%s" %s' % (fname, url) try: - subprocess.check_output(cmd,stderr=subprocess.STDOUT,shell=True) + subprocess.check_output(cmd, stderr=subprocess.STDOUT, shell=True) success = True except subprocess.CalledProcessError as e: print("Calling curl failed with: %s" % e.output.decode('UTF-8')) if not success and which('wget') != None: - cmd = 'wget -O "%s" %s' % (fname,url) + cmd = 'wget -O "%s" %s' % (fname, url) try: - subprocess.check_output(cmd,stderr=subprocess.STDOUT,shell=True) + subprocess.check_output(cmd, stderr=subprocess.STDOUT, shell=True) success = True except subprocess.CalledProcessError as e: print("Calling wget failed with: %s" % e.output.decode('UTF-8')) if not success: - error("Failed to download source code with 'curl' or 'wget'") + raise Exception("Failed to download source code with 'curl' or 'wget' from " + url) return -def checkmd5sum(md5sum,fname): - with open(fname,'rb') as fh: +def checkmd5sum(md5sum, fname): + with open(fname, 'rb') as fh: m = hashlib.md5() while True: data = fh.read(81920) @@ -66,3 +69,6 @@ def checkmd5sum(md5sum,fname): fh.close() return m.hexdigest() == md5sum +def getfallback(lib, url): + archive = re.sub(r'^https://.*/([^/]+gz)', r'-\1', url) + return 'https://download.lammps.org/thirdparty/' + lib + archive diff --git a/lib/latte/Install.py b/lib/latte/Install.py index 2e8f9bee8d..4e3b27859c 100644 --- a/lib/latte/Install.py +++ b/lib/latte/Install.py @@ -10,7 +10,7 @@ import sys, os, subprocess, shutil, tarfile from argparse import ArgumentParser sys.path.append('..') -from install_helpers import fullpath, geturl, checkmd5sum +from install_helpers import fullpath, geturl, checkmd5sum, getfallback parser = ArgumentParser(prog='Install.py', description="LAMMPS library build wrapper script") @@ -86,17 +86,25 @@ if buildflag: url = "https://github.com/lanl/LATTE/archive/v%s.tar.gz" % version lattepath = fullpath(homepath) lattedir = os.path.join(lattepath, homedir) + fallback = getfallback('latte', url) + filename = 'LATTE.tar.gz' # download and unpack LATTE tarball if buildflag: print("Downloading LATTE ...") - geturl(url, "LATTE.tar.gz") + try: + geturl(url, filename) + except: + geturl(fallback, filename) # verify downloaded archive integrity via md5 checksum, if known. if version in checksums: - if not checkmd5sum(checksums[version], 'LATTE.tar.gz'): - sys.exit("Checksum for LATTE library does not match") + if not checkmd5sum(checksums[version], filename): + print("Checksum did not match. Trying fallback URL", fallback) + geturl(fallback, filename) + if not checkmd5sum(checksums[version], filename): + sys.exit("Checksum for LATTE library does not match for fallback, too.") print("Unpacking LATTE ...") if os.path.exists(lattedir): diff --git a/lib/mdi/Install.py b/lib/mdi/Install.py index 59d218b1f9..77d13d3e8b 100644 --- a/lib/mdi/Install.py +++ b/lib/mdi/Install.py @@ -9,7 +9,7 @@ import sys,os,subprocess import glob sys.path.append('..') -from install_helpers import checkmd5sum +from install_helpers import fullpath, geturl, checkmd5sum, getfallback # help message @@ -51,49 +51,6 @@ def error(str=None): # expand to full path name # process leading '~' or relative path -def fullpath(path): - return os.path.abspath(os.path.expanduser(path)) - -def which(program): - def is_exe(fpath): - return os.path.isfile(fpath) and os.access(fpath, os.X_OK) - - fpath, fname = os.path.split(program) - if fpath: - if is_exe(program): - return program - else: - for path in os.environ["PATH"].split(os.pathsep): - path = path.strip('"') - exe_file = os.path.join(path, program) - if is_exe(exe_file): - return exe_file - - return None - -def geturl(url,fname): - success = False - - if which('curl') != None: - cmd = 'curl -L -o "%s" %s' % (fname,url) - try: - subprocess.check_output(cmd,stderr=subprocess.STDOUT,shell=True) - success = True - except subprocess.CalledProcessError as e: - print("Calling curl failed with: %s" % e.output.decode('UTF-8')) - - if not success and which('wget') != None: - cmd = 'wget -O "%s" %s' % (fname,url) - try: - subprocess.check_output(cmd,stderr=subprocess.STDOUT,shell=True) - success = True - except subprocess.CalledProcessError as e: - print("Calling wget failed with: %s" % e.output.decode('UTF-8')) - - if not success: - error("Failed to download source code with 'curl' or 'wget'") - return - # parse args args = sys.argv[1:] @@ -123,17 +80,24 @@ lib = os.path.basename(cwd) # download and unpack MDI_Library tarball -homepath = "." +homepath = fullpath('.') homedir = "%s/MDI_Library" % homepath print("Downloading MDI_Library ...") -mditar = "%s/v%s.tar.gz" % (homepath,version) -geturl(url, mditar) +mditar = "%s/v%s.tar.gz" % (homepath, version) +fallback = getfallback('mdi', url) +try: + geturl(url, mditar) +except: + geturl(fallback, mditar) # verify downloaded archive integrity via md5 checksum, if known. if version in checksums: if not checkmd5sum(checksums[version], mditar): - sys.exit("Checksum for MDI library does not match") + print("Checksum did not match. Trying fallback URL", fallback) + geturl(fallback, mditar) + if not checkmd5sum(checksums[version], mditar): + sys.exit("Checksum for MDI library does not match") print("Unpacking MDI_Library tarball ...") if os.path.exists("%s/v%s" % (homepath,version)): @@ -199,7 +163,6 @@ makefile_lammps = open(str(dir_path) + "/Makefile.lammps", "a") makefile_lammps.write(str(rpath_option) + "\n") makefile_lammps.close() - shared_files = glob.glob( os.path.join( homepath, "liblink", "lib%s.a" % lib) ) if len(shared_files) > 0: print("Build was successful") diff --git a/lib/mscg/Install.py b/lib/mscg/Install.py index 9d18b7c083..df17005dca 100644 --- a/lib/mscg/Install.py +++ b/lib/mscg/Install.py @@ -10,7 +10,7 @@ import sys, os, subprocess, shutil, tarfile from argparse import ArgumentParser sys.path.append('..') -from install_helpers import fullpath, geturl +from install_helpers import fullpath, geturl, checkmd5sum, getfallback parser = ArgumentParser(prog='Install.py', description="LAMMPS library build wrapper script") @@ -84,7 +84,19 @@ if pathflag: if buildflag: print("Downloading MS-CG ...") tarname = os.path.join(homepath, tarname) - geturl(url, tarname) + fallback = getfallback('mscg', url) + try: + geturl(url, tarname) + except: + geturl(fallback, tarname) + + # verify downloaded archive integrity via md5 checksum, if known. + if mscgver in checksums: + if not checkmd5sum(checksums[mscgver], tarname): + print("Checksum did not match. Trying fallback URL", fallback) + geturl(fallback, tarname) + if not checkmd5sum(checksums[mscgver], tarname): + sys.exit("Checksum for LATTE library does not match for fallback, too.") print("Unpacking MS-CG tarfile ...") diff --git a/lib/pace/Install.py b/lib/pace/Install.py index f7f50d4f6d..21cadbb20a 100644 --- a/lib/pace/Install.py +++ b/lib/pace/Install.py @@ -13,7 +13,7 @@ import sys from argparse import ArgumentParser sys.path.append('..') -from install_helpers import fullpath, geturl, checkmd5sum +from install_helpers import fullpath, geturl, checkmd5sum, getfallback # settings @@ -77,14 +77,21 @@ if buildflag: print("Downloading pace tarball ...") archive_filename = "%s.%s" % (version, archive_extension) download_filename = "%s/%s" % (thisdir, archive_filename) + fallback = getfallback('pacelib', url) print("Downloading from ", url, " to ", download_filename, end=" ") - geturl(url, download_filename) + try: + geturl(url, download_filename) + except: + geturl(fallback, download_filename) print(" done") # verify downloaded archive integrity via md5 checksum, if known. if version in checksums: - if not checkmd5sum(checksums[version], archive_filename): - sys.exit("Checksum for pace library does not match") + if not checkmd5sum(checksums[version], download_filename): + print("Checksum did not match. Trying fallback URL", fallback) + geturl(fallback, download_filename) + if not checkmd5sum(checksums[version], download_filename): + sys.exit("Checksum for pace library does not match for fallback, too.") print("Unpacking pace tarball ...") src_folder = thisdir + "/src" diff --git a/lib/plumed/Install.py b/lib/plumed/Install.py index bb3a158bd4..bb6f27fddb 100644 --- a/lib/plumed/Install.py +++ b/lib/plumed/Install.py @@ -10,7 +10,7 @@ import sys, os, platform, subprocess, shutil from argparse import ArgumentParser sys.path.append('..') -from install_helpers import get_cpus, fullpath, geturl, checkmd5sum +from install_helpers import get_cpus, fullpath, geturl, checkmd5sum, getfallback parser = ArgumentParser(prog='Install.py', description="LAMMPS library build wrapper script") @@ -86,6 +86,7 @@ buildflag = args.build pathflag = args.path is not None plumedpath = args.path mode = args.mode +version = args.version homepath = fullpath('.') homedir = "%s/plumed2" % (homepath) @@ -101,14 +102,21 @@ if pathflag: if buildflag: url = "https://github.com/plumed/plumed2/releases/download/v%s/plumed-src-%s.tgz" % (version, version) - filename = "plumed-src-%s.tar.gz" %version + filename = "plumed-src-%s.tar.gz" % version + fallback = getfallback('plumed', url) print("Downloading plumed ...") - geturl(url, filename) + try: + geturl(url, filename) + except: + geturl(fallback, filename) # verify downloaded archive integrity via md5 checksum, if known. if version in checksums: if not checkmd5sum(checksums[version], filename): - sys.exit("Checksum for plumed2 library does not match") + print("Checksum did not match. Trying fallback URL", fallback) + geturl(fallback, filename) + if not checkmd5sum(checksums[version], filename): + sys.exit("Checksum for plumed2 library does not match for fallback, too.") print("Unpacking plumed2 source tarball ...") if os.path.exists("%s/plumed-%s" % (homepath, version)): diff --git a/lib/scafacos/Install.py b/lib/scafacos/Install.py index 8044215d47..28bcf43086 100644 --- a/lib/scafacos/Install.py +++ b/lib/scafacos/Install.py @@ -10,15 +10,13 @@ import sys, os, subprocess, shutil, tarfile from argparse import ArgumentParser sys.path.append('..') -from install_helpers import fullpath, geturl, get_cpus, checkmd5sum +from install_helpers import fullpath, geturl, get_cpus, checkmd5sum, getfallback -parser = ArgumentParser(prog='Install.py', - description="LAMMPS library build wrapper script") +parser = ArgumentParser(prog='Install.py', description="LAMMPS library build wrapper script") # settings version = "1.0.1" -url = "https://github.com/scafacos/scafacos/releases/download/v%s/scafacos-%s.tar.gz" % (version, version) # known checksums for different ScaFaCoS versions. used to validate the download. checksums = { \ @@ -59,6 +57,7 @@ if not args.build and not args.path: buildflag = args.build pathflag = args.path is not None version = args.version +url = "https://github.com/scafacos/scafacos/releases/download/v%s/scafacos-%s.tar.gz" % (version, version) homepath = fullpath(".") scafacospath = os.path.join(homepath, "scafacos-%s" % version) @@ -76,12 +75,20 @@ if pathflag: if buildflag: print("Downloading ScaFaCoS ...") - geturl(url, "%s/scafacos-%s.tar.gz" % (homepath, version)) + filename = "%s/scafacos-%s.tar.gz" % (homepath, version) + fallback = getfallback('scafacos', url) + try: + geturl(url, filename) + except: + geturl(fallback, filename) # verify downloaded archive integrity via md5 checksum, if known. if version in checksums: - if not checkmd5sum(checksums[version], '%s/scafacos-%s.tar.gz' % (homepath, version)): - sys.exit("Checksum for ScaFaCoS library does not match") + if not checkmd5sum(checksums[version], filename): + print("Checksum did not match. Trying fallback URL", fallback) + geturl(fallback, filename) + if not checkmd5sum(checksums[version], filename): + sys.exit("Checksum for ScaFaCoS library does not match for fallback, too.") print("Unpacking ScaFaCoS tarball ...") if os.path.exists(scafacospath):