ICODE-ADC/lammps
4
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

implement download fallback for traditional make build

Browse Source
This commit is contained in:
Axel Kohlmeyer
2023-02-01 06:47:25 -05:00
parent 5d16bea899
commit 89b37c51df
9 changed files with 105 additions and 84 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
lib/hdnnp/.gitignore vendored Normal file
View File

@ -0,0 +1,3 @@
/includelink
/liblink
/n2p2-*

15
lib/hdnnp/Install.py
View File

@ -10,7 +10,7 @@ import sys, os, platform, subprocess, shutil
from argparse import ArgumentParser
sys.path.append('..')
from install_helpers import get_cpus, fullpath, geturl, checkmd5sum
from install_helpers import get_cpus, fullpath, geturl, checkmd5sum, getfallback
parser = ArgumentParser(prog='Install.py',
description="LAMMPS library build wrapper script")
@ -76,14 +76,21 @@ if pathflag:
if buildflag:
url = "https://github.com/CompPhysVienna/n2p2/archive/v%s.tar.gz" % (version)
filename = "n2p2-%s.tar.gz" %version
print("Downloading n2p2 ...")
filename = "n2p2-%s.tar.gz" % version
fallback = getfallback('n2p2', url)
print("Downloading n2p2 from", url)
try:
geturl(url, filename)
except:
geturl(fallback, filename)
# verify downloaded archive integrity via md5 checksum, if known.
if version in checksums:
if not checkmd5sum(checksums[version], filename):
sys.exit("Checksum for n2p2 library does not match")
print("Checksum did not match. Trying fallback URL", fallback)
geturl(fallback, filename)
if not checkmd5sum(checksums[version], filename):
sys.exit("Checksum for n2p2 library does not match for fallback, too.")
print("Unpacking n2p2 source tarball ...")
if os.path.exists("%s/n2p2-%s" % (homepath, version)):

24
lib/install_helpers.py
View File

@ -1,4 +1,7 @@
import hashlib,os,subprocess
import hashlib
import os
import re
import subprocess
# try to auto-detect the maximum number of available CPUs
def get_cpus():
@ -32,31 +35,31 @@ def which(program):
return None
def geturl(url,fname):
def geturl(url, fname):
success = False
if which('curl') != None:
cmd = 'curl -L -o "%s" %s' % (fname,url)
cmd = 'curl -L -o "%s" %s' % (fname, url)
try:
subprocess.check_output(cmd,stderr=subprocess.STDOUT,shell=True)
subprocess.check_output(cmd, stderr=subprocess.STDOUT, shell=True)
success = True
except subprocess.CalledProcessError as e:
print("Calling curl failed with: %s" % e.output.decode('UTF-8'))
if not success and which('wget') != None:
cmd = 'wget -O "%s" %s' % (fname,url)
cmd = 'wget -O "%s" %s' % (fname, url)
try:
subprocess.check_output(cmd,stderr=subprocess.STDOUT,shell=True)
subprocess.check_output(cmd, stderr=subprocess.STDOUT, shell=True)
success = True
except subprocess.CalledProcessError as e:
print("Calling wget failed with: %s" % e.output.decode('UTF-8'))
if not success:
error("Failed to download source code with 'curl' or 'wget'")
raise Exception("Failed to download source code with 'curl' or 'wget' from " + url)
return
def checkmd5sum(md5sum,fname):
with open(fname,'rb') as fh:
def checkmd5sum(md5sum, fname):
with open(fname, 'rb') as fh:
m = hashlib.md5()
while True:
data = fh.read(81920)
@ -66,3 +69,6 @@ def checkmd5sum(md5sum,fname):
fh.close()
return m.hexdigest() == md5sum
def getfallback(lib, url):
archive = re.sub(r'^https://.*/([^/]+gz)', r'-\1', url)
return 'https://download.lammps.org/thirdparty/' + lib + archive

16
lib/latte/Install.py
View File

@ -10,7 +10,7 @@ import sys, os, subprocess, shutil, tarfile
from argparse import ArgumentParser
sys.path.append('..')
from install_helpers import fullpath, geturl, checkmd5sum
from install_helpers import fullpath, geturl, checkmd5sum, getfallback
parser = ArgumentParser(prog='Install.py',
description="LAMMPS library build wrapper script")
@ -86,17 +86,25 @@ if buildflag:
url = "https://github.com/lanl/LATTE/archive/v%s.tar.gz" % version
lattepath = fullpath(homepath)
lattedir = os.path.join(lattepath, homedir)
fallback = getfallback('latte', url)
filename = 'LATTE.tar.gz'
# download and unpack LATTE tarball
if buildflag:
print("Downloading LATTE ...")
geturl(url, "LATTE.tar.gz")
try:
geturl(url, filename)
except:
geturl(fallback, filename)
# verify downloaded archive integrity via md5 checksum, if known.
if version in checksums:
if not checkmd5sum(checksums[version], 'LATTE.tar.gz'):
sys.exit("Checksum for LATTE library does not match")
if not checkmd5sum(checksums[version], filename):
print("Checksum did not match. Trying fallback URL", fallback)
geturl(fallback, filename)
if not checkmd5sum(checksums[version], filename):
sys.exit("Checksum for LATTE library does not match for fallback, too.")
print("Unpacking LATTE ...")
if os.path.exists(lattedir):

59
lib/mdi/Install.py
View File

@ -9,7 +9,7 @@ import sys,os,subprocess
import glob
sys.path.append('..')
from install_helpers import checkmd5sum
from install_helpers import fullpath, geturl, checkmd5sum, getfallback
# help message
@ -51,49 +51,6 @@ def error(str=None):
# expand to full path name
# process leading '~' or relative path
def fullpath(path):
return os.path.abspath(os.path.expanduser(path))
def which(program):
def is_exe(fpath):
return os.path.isfile(fpath) and os.access(fpath, os.X_OK)
fpath, fname = os.path.split(program)
if fpath:
if is_exe(program):
return program
else:
for path in os.environ["PATH"].split(os.pathsep):
path = path.strip('"')
exe_file = os.path.join(path, program)
if is_exe(exe_file):
return exe_file
return None
def geturl(url,fname):
success = False
if which('curl') != None:
cmd = 'curl -L -o "%s" %s' % (fname,url)
try:
subprocess.check_output(cmd,stderr=subprocess.STDOUT,shell=True)
success = True
except subprocess.CalledProcessError as e:
print("Calling curl failed with: %s" % e.output.decode('UTF-8'))
if not success and which('wget') != None:
cmd = 'wget -O "%s" %s' % (fname,url)
try:
subprocess.check_output(cmd,stderr=subprocess.STDOUT,shell=True)
success = True
except subprocess.CalledProcessError as e:
print("Calling wget failed with: %s" % e.output.decode('UTF-8'))
if not success:
error("Failed to download source code with 'curl' or 'wget'")
return
# parse args
args = sys.argv[1:]
@ -123,15 +80,22 @@ lib = os.path.basename(cwd)
# download and unpack MDI_Library tarball
homepath = "."
homepath = fullpath('.')
homedir = "%s/MDI_Library" % homepath
print("Downloading MDI_Library ...")
mditar = "%s/v%s.tar.gz" % (homepath,version)
geturl(url, mditar)
mditar = "%s/v%s.tar.gz" % (homepath, version)
fallback = getfallback('mdi', url)
try:
geturl(url, mditar)
except:
geturl(fallback, mditar)
# verify downloaded archive integrity via md5 checksum, if known.
if version in checksums:
if not checkmd5sum(checksums[version], mditar):
print("Checksum did not match. Trying fallback URL", fallback)
geturl(fallback, mditar)
if not checkmd5sum(checksums[version], mditar):
sys.exit("Checksum for MDI library does not match")
@ -199,7 +163,6 @@ makefile_lammps = open(str(dir_path) + "/Makefile.lammps", "a")
makefile_lammps.write(str(rpath_option) + "\n")
makefile_lammps.close()
shared_files = glob.glob( os.path.join( homepath, "liblink", "lib%s.a" % lib) )
if len(shared_files) > 0:
print("Build was successful")

14
lib/mscg/Install.py
View File

@ -10,7 +10,7 @@ import sys, os, subprocess, shutil, tarfile
from argparse import ArgumentParser
sys.path.append('..')
from install_helpers import fullpath, geturl
from install_helpers import fullpath, geturl, checkmd5sum, getfallback
parser = ArgumentParser(prog='Install.py',
description="LAMMPS library build wrapper script")
@ -84,7 +84,19 @@ if pathflag:
if buildflag:
print("Downloading MS-CG ...")
tarname = os.path.join(homepath, tarname)
fallback = getfallback('mscg', url)
try:
geturl(url, tarname)
except:
geturl(fallback, tarname)
# verify downloaded archive integrity via md5 checksum, if known.
if mscgver in checksums:
if not checkmd5sum(checksums[mscgver], tarname):
print("Checksum did not match. Trying fallback URL", fallback)
geturl(fallback, tarname)
if not checkmd5sum(checksums[mscgver], tarname):
sys.exit("Checksum for LATTE library does not match for fallback, too.")
print("Unpacking MS-CG tarfile ...")

13
lib/pace/Install.py
View File

@ -13,7 +13,7 @@ import sys
from argparse import ArgumentParser
sys.path.append('..')
from install_helpers import fullpath, geturl, checkmd5sum
from install_helpers import fullpath, geturl, checkmd5sum, getfallback
# settings
@ -77,14 +77,21 @@ if buildflag:
print("Downloading pace tarball ...")
archive_filename = "%s.%s" % (version, archive_extension)
download_filename = "%s/%s" % (thisdir, archive_filename)
fallback = getfallback('pacelib', url)
print("Downloading from ", url, " to ", download_filename, end=" ")
try:
geturl(url, download_filename)
except:
geturl(fallback, download_filename)
print(" done")
# verify downloaded archive integrity via md5 checksum, if known.
if version in checksums:
if not checkmd5sum(checksums[version], archive_filename):
sys.exit("Checksum for pace library does not match")
if not checkmd5sum(checksums[version], download_filename):
print("Checksum did not match. Trying fallback URL", fallback)
geturl(fallback, download_filename)
if not checkmd5sum(checksums[version], download_filename):
sys.exit("Checksum for pace library does not match for fallback, too.")
print("Unpacking pace tarball ...")
src_folder = thisdir + "/src"

14
lib/plumed/Install.py
View File

@ -10,7 +10,7 @@ import sys, os, platform, subprocess, shutil
from argparse import ArgumentParser
sys.path.append('..')
from install_helpers import get_cpus, fullpath, geturl, checkmd5sum
from install_helpers import get_cpus, fullpath, geturl, checkmd5sum, getfallback
parser = ArgumentParser(prog='Install.py',
description="LAMMPS library build wrapper script")
@ -86,6 +86,7 @@ buildflag = args.build
pathflag = args.path is not None
plumedpath = args.path
mode = args.mode
version = args.version
homepath = fullpath('.')
homedir = "%s/plumed2" % (homepath)
@ -101,14 +102,21 @@ if pathflag:
if buildflag:
url = "https://github.com/plumed/plumed2/releases/download/v%s/plumed-src-%s.tgz" % (version, version)
filename = "plumed-src-%s.tar.gz" %version
filename = "plumed-src-%s.tar.gz" % version
fallback = getfallback('plumed', url)
print("Downloading plumed ...")
try:
geturl(url, filename)
except:
geturl(fallback, filename)
# verify downloaded archive integrity via md5 checksum, if known.
if version in checksums:
if not checkmd5sum(checksums[version], filename):
sys.exit("Checksum for plumed2 library does not match")
print("Checksum did not match. Trying fallback URL", fallback)
geturl(fallback, filename)
if not checkmd5sum(checksums[version], filename):
sys.exit("Checksum for plumed2 library does not match for fallback, too.")
print("Unpacking plumed2 source tarball ...")
if os.path.exists("%s/plumed-%s" % (homepath, version)):

21
lib/scafacos/Install.py
View File

@ -10,15 +10,13 @@ import sys, os, subprocess, shutil, tarfile
from argparse import ArgumentParser
sys.path.append('..')
from install_helpers import fullpath, geturl, get_cpus, checkmd5sum
from install_helpers import fullpath, geturl, get_cpus, checkmd5sum, getfallback
parser = ArgumentParser(prog='Install.py',
description="LAMMPS library build wrapper script")
parser = ArgumentParser(prog='Install.py', description="LAMMPS library build wrapper script")
# settings
version = "1.0.1"
url = "https://github.com/scafacos/scafacos/releases/download/v%s/scafacos-%s.tar.gz" % (version, version)
# known checksums for different ScaFaCoS versions. used to validate the download.
checksums = { \
@ -59,6 +57,7 @@ if not args.build and not args.path:
buildflag = args.build
pathflag = args.path is not None
version = args.version
url = "https://github.com/scafacos/scafacos/releases/download/v%s/scafacos-%s.tar.gz" % (version, version)
homepath = fullpath(".")
scafacospath = os.path.join(homepath, "scafacos-%s" % version)
@ -76,12 +75,20 @@ if pathflag:
if buildflag:
print("Downloading ScaFaCoS ...")
geturl(url, "%s/scafacos-%s.tar.gz" % (homepath, version))
filename = "%s/scafacos-%s.tar.gz" % (homepath, version)
fallback = getfallback('scafacos', url)
try:
geturl(url, filename)
except:
geturl(fallback, filename)
# verify downloaded archive integrity via md5 checksum, if known.
if version in checksums:
if not checkmd5sum(checksums[version], '%s/scafacos-%s.tar.gz' % (homepath, version)):
sys.exit("Checksum for ScaFaCoS library does not match")
if not checkmd5sum(checksums[version], filename):
print("Checksum did not match. Trying fallback URL", fallback)
geturl(fallback, filename)
if not checkmd5sum(checksums[version], filename):
sys.exit("Checksum for ScaFaCoS library does not match for fallback, too.")
print("Unpacking ScaFaCoS tarball ...")
if os.path.exists(scafacospath):