implement download fallback for traditional make build

lib/hdnnp/.gitignore

@@ -0,0 +1,3 @@
+/includelink
+/liblink
+/n2p2-*

lib/hdnnp/Install.py

@@ -10,7 +10,7 @@ import sys, os, platform, subprocess, shutil
 from argparse import ArgumentParser
 
 sys.path.append('..')
-from install_helpers import get_cpus, fullpath, geturl, checkmd5sum
+from install_helpers import get_cpus, fullpath, geturl, checkmd5sum, getfallback
 
 parser = ArgumentParser(prog='Install.py',
                         description="LAMMPS library build wrapper script")
@@ -76,14 +76,21 @@ if pathflag:
 
 if buildflag:
   url = "https://github.com/CompPhysVienna/n2p2/archive/v%s.tar.gz" % (version)
-  filename = "n2p2-%s.tar.gz" %version
+  filename = "n2p2-%s.tar.gz" % version
-  print("Downloading n2p2 ...")
+  fallback = getfallback('n2p2', url)
-  geturl(url, filename)
+  print("Downloading n2p2 from", url)
+  try:
+    geturl(url, filename)
+  except:
+    geturl(fallback, filename)
 
   # verify downloaded archive integrity via md5 checksum, if known.
   if version in checksums:
     if not checkmd5sum(checksums[version], filename):
-      sys.exit("Checksum for n2p2 library does not match")
+      print("Checksum did not match. Trying fallback URL", fallback)
+      geturl(fallback, filename)
+      if not checkmd5sum(checksums[version], filename):
+        sys.exit("Checksum for n2p2 library does not match for fallback, too.")
 
   print("Unpacking n2p2 source tarball ...")
   if os.path.exists("%s/n2p2-%s" % (homepath, version)):

lib/install_helpers.py

@@ -1,4 +1,7 @@
-import hashlib,os,subprocess
+import hashlib
+import os
+import re
+import subprocess
 
 # try to auto-detect the maximum number of available CPUs
 def get_cpus():
@@ -32,31 +35,31 @@ def which(program):
 
   return None
 
-def geturl(url,fname):
+def geturl(url, fname):
   success = False
 
   if which('curl') != None:
-    cmd = 'curl -L -o "%s" %s' % (fname,url)
+    cmd = 'curl -L -o "%s" %s' % (fname, url)
     try:
-      subprocess.check_output(cmd,stderr=subprocess.STDOUT,shell=True)
+      subprocess.check_output(cmd, stderr=subprocess.STDOUT, shell=True)
       success = True
     except subprocess.CalledProcessError as e:
       print("Calling curl failed with: %s" % e.output.decode('UTF-8'))
 
   if not success and which('wget') != None:
-    cmd = 'wget -O "%s" %s' % (fname,url)
+    cmd = 'wget -O "%s" %s' % (fname, url)
     try:
-      subprocess.check_output(cmd,stderr=subprocess.STDOUT,shell=True)
+      subprocess.check_output(cmd, stderr=subprocess.STDOUT, shell=True)
       success = True
     except subprocess.CalledProcessError as e:
       print("Calling wget failed with: %s" % e.output.decode('UTF-8'))
 
   if not success:
-    error("Failed to download source code with 'curl' or 'wget'")
+    raise Exception("Failed to download source code with 'curl' or 'wget' from " + url)
   return
 
-def checkmd5sum(md5sum,fname):
+def checkmd5sum(md5sum, fname):
-    with open(fname,'rb') as fh:
+    with open(fname, 'rb') as fh:
         m = hashlib.md5()
         while True:
             data = fh.read(81920)
@@ -66,3 +69,6 @@ def checkmd5sum(md5sum,fname):
     fh.close()
     return m.hexdigest() == md5sum
 
+def getfallback(lib, url):
+  archive = re.sub(r'^https://.*/([^/]+gz)', r'-\1', url)
+  return 'https://download.lammps.org/thirdparty/' + lib + archive

lib/latte/Install.py

@@ -10,7 +10,7 @@ import sys, os, subprocess, shutil, tarfile
 from argparse import ArgumentParser
 
 sys.path.append('..')
-from install_helpers import fullpath, geturl, checkmd5sum
+from install_helpers import fullpath, geturl, checkmd5sum, getfallback
 
 parser = ArgumentParser(prog='Install.py',
                         description="LAMMPS library build wrapper script")
@@ -86,17 +86,25 @@ if buildflag:
   url = "https://github.com/lanl/LATTE/archive/v%s.tar.gz" % version
   lattepath = fullpath(homepath)
   lattedir = os.path.join(lattepath, homedir)
+  fallback = getfallback('latte', url)
+  filename = 'LATTE.tar.gz'
 
 # download and unpack LATTE tarball
 
 if buildflag:
   print("Downloading LATTE ...")
-  geturl(url, "LATTE.tar.gz")
+  try:
+    geturl(url, filename)
+  except:
+    geturl(fallback, filename)
 
   # verify downloaded archive integrity via md5 checksum, if known.
   if version in checksums:
-    if not checkmd5sum(checksums[version], 'LATTE.tar.gz'):
+    if not checkmd5sum(checksums[version], filename):
-      sys.exit("Checksum for LATTE library does not match")
+      print("Checksum did not match. Trying fallback URL", fallback)
+      geturl(fallback, filename)
+      if not checkmd5sum(checksums[version], filename):
+        sys.exit("Checksum for LATTE library does not match for fallback, too.")
 
   print("Unpacking LATTE ...")
   if os.path.exists(lattedir):

lib/mdi/Install.py

@@ -9,7 +9,7 @@ import sys,os,subprocess
 import glob
 
 sys.path.append('..')
-from install_helpers import checkmd5sum
+from install_helpers import fullpath, geturl, checkmd5sum, getfallback
 
 # help message
 
@@ -51,49 +51,6 @@ def error(str=None):
   # expand to full path name
 # process leading '~' or relative path
 
-def fullpath(path):
-  return os.path.abspath(os.path.expanduser(path))
-
-def which(program):
-  def is_exe(fpath):
-    return os.path.isfile(fpath) and os.access(fpath, os.X_OK)
-
-  fpath, fname = os.path.split(program)
-  if fpath:
-    if is_exe(program):
-      return program
-  else:
-    for path in os.environ["PATH"].split(os.pathsep):
-      path = path.strip('"')
-      exe_file = os.path.join(path, program)
-      if is_exe(exe_file):
-        return exe_file
-
-  return None
-
-def geturl(url,fname):
-  success = False
-
-  if which('curl') != None:
-    cmd = 'curl -L -o "%s" %s' % (fname,url)
-    try:
-      subprocess.check_output(cmd,stderr=subprocess.STDOUT,shell=True)
-      success = True
-    except subprocess.CalledProcessError as e:
-      print("Calling curl failed with: %s" % e.output.decode('UTF-8'))
-
-  if not success and which('wget') != None:
-    cmd = 'wget -O "%s" %s' % (fname,url)
-    try:
-      subprocess.check_output(cmd,stderr=subprocess.STDOUT,shell=True)
-      success = True
-    except subprocess.CalledProcessError as e:
-      print("Calling wget failed with: %s" % e.output.decode('UTF-8'))
-
-  if not success:
-    error("Failed to download source code with 'curl' or 'wget'")
-  return
-
 # parse args
 
 args = sys.argv[1:]
@@ -123,17 +80,24 @@ lib = os.path.basename(cwd)
 
 # download and unpack MDI_Library tarball
 
-homepath = "."
+homepath = fullpath('.')
 homedir = "%s/MDI_Library" % homepath
 
 print("Downloading MDI_Library ...")
-mditar = "%s/v%s.tar.gz" % (homepath,version)
+mditar = "%s/v%s.tar.gz" % (homepath, version)
-geturl(url, mditar)
+fallback = getfallback('mdi', url)
+try:
+  geturl(url, mditar)
+except:
+  geturl(fallback, mditar)
 
 # verify downloaded archive integrity via md5 checksum, if known.
 if version in checksums:
   if not checkmd5sum(checksums[version], mditar):
-    sys.exit("Checksum for MDI library does not match")
+    print("Checksum did not match. Trying fallback URL", fallback)
+    geturl(fallback, mditar)
+    if not checkmd5sum(checksums[version], mditar):
+      sys.exit("Checksum for MDI library does not match")
 
 print("Unpacking MDI_Library tarball ...")
 if os.path.exists("%s/v%s" % (homepath,version)):
@@ -199,7 +163,6 @@ makefile_lammps = open(str(dir_path) + "/Makefile.lammps", "a")
 makefile_lammps.write(str(rpath_option) + "\n")
 makefile_lammps.close()
 
-
 shared_files = glob.glob( os.path.join( homepath, "liblink", "lib%s.a" % lib) )
 if len(shared_files) > 0:
   print("Build was successful")

lib/mscg/Install.py

@@ -10,7 +10,7 @@ import sys, os, subprocess, shutil, tarfile
 from argparse import ArgumentParser
 
 sys.path.append('..')
-from install_helpers import fullpath, geturl
+from install_helpers import fullpath, geturl, checkmd5sum, getfallback
 
 parser = ArgumentParser(prog='Install.py',
                         description="LAMMPS library build wrapper script")
@@ -84,7 +84,19 @@ if pathflag:
 if buildflag:
   print("Downloading MS-CG ...")
   tarname = os.path.join(homepath, tarname)
-  geturl(url, tarname)
+  fallback = getfallback('mscg', url)
+  try:
+    geturl(url, tarname)
+  except:
+    geturl(fallback, tarname)
+
+  # verify downloaded archive integrity via md5 checksum, if known.
+  if mscgver in checksums:
+    if not checkmd5sum(checksums[mscgver], tarname):
+      print("Checksum did not match. Trying fallback URL", fallback)
+      geturl(fallback, tarname)
+      if not checkmd5sum(checksums[mscgver], tarname):
+        sys.exit("Checksum for LATTE library does not match for fallback, too.")
 
   print("Unpacking MS-CG tarfile ...")
 

lib/pace/Install.py

@@ -13,7 +13,7 @@ import sys
 from argparse import ArgumentParser
 
 sys.path.append('..')
-from install_helpers import fullpath, geturl, checkmd5sum
+from install_helpers import fullpath, geturl, checkmd5sum, getfallback
 
 # settings
 
@@ -77,14 +77,21 @@ if buildflag:
         print("Downloading pace tarball ...")
         archive_filename = "%s.%s" % (version, archive_extension)
         download_filename = "%s/%s" % (thisdir, archive_filename)
+        fallback = getfallback('pacelib', url)
         print("Downloading from ", url, " to ", download_filename, end=" ")
-        geturl(url, download_filename)
+        try:
+            geturl(url, download_filename)
+        except:
+            geturl(fallback, download_filename)
         print(" done")
 
         # verify downloaded archive integrity via md5 checksum, if known.
         if version in checksums:
-            if not checkmd5sum(checksums[version], archive_filename):
+            if not checkmd5sum(checksums[version], download_filename):
-                sys.exit("Checksum for pace library does not match")
+                print("Checksum did not match. Trying fallback URL", fallback)
+                geturl(fallback, download_filename)
+                if not checkmd5sum(checksums[version], download_filename):
+                    sys.exit("Checksum for pace library does not match for fallback, too.")
 
         print("Unpacking pace tarball ...")
         src_folder = thisdir + "/src"

lib/plumed/Install.py

@@ -10,7 +10,7 @@ import sys, os, platform, subprocess, shutil
 from argparse import ArgumentParser
 
 sys.path.append('..')
-from install_helpers import get_cpus, fullpath, geturl, checkmd5sum
+from install_helpers import get_cpus, fullpath, geturl, checkmd5sum, getfallback
 
 parser = ArgumentParser(prog='Install.py',
                         description="LAMMPS library build wrapper script")
@@ -86,6 +86,7 @@ buildflag = args.build
 pathflag = args.path is not None
 plumedpath = args.path
 mode = args.mode
+version = args.version
 
 homepath = fullpath('.')
 homedir = "%s/plumed2" % (homepath)
@@ -101,14 +102,21 @@ if pathflag:
 
 if buildflag:
   url = "https://github.com/plumed/plumed2/releases/download/v%s/plumed-src-%s.tgz" % (version, version)
-  filename = "plumed-src-%s.tar.gz" %version
+  filename = "plumed-src-%s.tar.gz" % version
+  fallback = getfallback('plumed', url)
   print("Downloading plumed  ...")
-  geturl(url, filename)
+  try:
+    geturl(url, filename)
+  except:
+    geturl(fallback, filename)
 
   # verify downloaded archive integrity via md5 checksum, if known.
   if version in checksums:
     if not checkmd5sum(checksums[version], filename):
-      sys.exit("Checksum for plumed2 library does not match")
+      print("Checksum did not match. Trying fallback URL", fallback)
+      geturl(fallback, filename)
+      if not checkmd5sum(checksums[version], filename):
+        sys.exit("Checksum for plumed2 library does not match for fallback, too.")
 
   print("Unpacking plumed2 source tarball ...")
   if os.path.exists("%s/plumed-%s" % (homepath, version)):

lib/scafacos/Install.py

@@ -10,15 +10,13 @@ import sys, os, subprocess, shutil, tarfile
 from argparse import ArgumentParser
 
 sys.path.append('..')
-from install_helpers import fullpath, geturl, get_cpus, checkmd5sum
+from install_helpers import fullpath, geturl, get_cpus, checkmd5sum, getfallback
 
-parser = ArgumentParser(prog='Install.py',
+parser = ArgumentParser(prog='Install.py', description="LAMMPS library build wrapper script")
-                        description="LAMMPS library build wrapper script")
 
 # settings
 
 version = "1.0.1"
-url = "https://github.com/scafacos/scafacos/releases/download/v%s/scafacos-%s.tar.gz" % (version, version)
 
 # known checksums for different ScaFaCoS versions. used to validate the download.
 checksums = { \
@@ -59,6 +57,7 @@ if not args.build and not args.path:
 buildflag = args.build
 pathflag = args.path is not None
 version = args.version
+url = "https://github.com/scafacos/scafacos/releases/download/v%s/scafacos-%s.tar.gz" % (version, version)
 
 homepath = fullpath(".")
 scafacospath = os.path.join(homepath, "scafacos-%s" % version)
@@ -76,12 +75,20 @@ if pathflag:
 
 if buildflag:
   print("Downloading ScaFaCoS ...")
-  geturl(url, "%s/scafacos-%s.tar.gz" % (homepath, version))
+  filename = "%s/scafacos-%s.tar.gz" % (homepath, version)
+  fallback = getfallback('scafacos', url)
+  try:
+    geturl(url, filename)
+  except:
+    geturl(fallback, filename)
 
   # verify downloaded archive integrity via md5 checksum, if known.
   if version in checksums:
-    if not checkmd5sum(checksums[version], '%s/scafacos-%s.tar.gz' % (homepath, version)):
+    if not checkmd5sum(checksums[version], filename):
-      sys.exit("Checksum for ScaFaCoS library does not match")
+      print("Checksum did not match. Trying fallback URL", fallback)
+      geturl(fallback, filename)
+      if not checkmd5sum(checksums[version], filename):
+        sys.exit("Checksum for ScaFaCoS library does not match for fallback, too.")
 
   print("Unpacking ScaFaCoS tarball ...")
   if os.path.exists(scafacospath):