From 891d3c8296fd9158209c9d6b37ad800718ed3c77 Mon Sep 17 00:00:00 2001 From: Axel Kohlmeyer Date: Thu, 24 Feb 2022 17:04:16 -0500 Subject: [PATCH 01/18] update setup.py to be compatible with building "pure" and "binary" wheels --- python/pyproject.toml | 3 +++ python/setup.py | 34 +++++++++++++++++++++++++++++++++- python/wheel_requirements.txt | 4 ++++ 3 files changed, 40 insertions(+), 1 deletion(-) create mode 100644 python/pyproject.toml create mode 100644 python/wheel_requirements.txt diff --git a/python/pyproject.toml b/python/pyproject.toml new file mode 100644 index 0000000000..b5c9a51ece --- /dev/null +++ b/python/pyproject.toml @@ -0,0 +1,3 @@ +[build-system] +requires = [ "setuptools>=42", "wheel" ] +build-backend = "setuptools.build_meta" diff --git a/python/setup.py b/python/setup.py index 5a6a54258a..6a271a1d26 100644 --- a/python/setup.py +++ b/python/setup.py @@ -1,6 +1,7 @@ # this only installs the LAMMPS python package # it assumes the LAMMPS shared library is already installed from setuptools import setup +from setuptools.dist import Distribution from sys import version_info import os,time LAMMPS_PYTHON_DIR = os.path.dirname(os.path.realpath(__file__)) @@ -21,18 +22,49 @@ def get_lammps_version(): t = time.strptime("".join(line[start_pos:end_pos].split()), "%d%b%Y") return "{}.{}.{}".format(t.tm_year,t.tm_mon,t.tm_mday) +class BinaryDistribution(Distribution): + """Wrapper to enforce creating a binary package""" + def has_ext_modules(foo): + return True + +libpath = os.environ.get("LAMMPS_SHARED_LIB") + if version_info.major >= 3: pkgs = ['lammps', 'lammps.mliap'] else: pkgs = ['lammps'] +with open("README", "r", encoding="utf-8") as fh: + long_description = fh.read() + +if libpath: + pkgdata = {'lammps': [ libpath ]} + bdist = BinaryDistribution +else: + pkgdata = {} + bdist = Distribution + setup( name = "lammps", version = get_lammps_version(), author = "Steve Plimpton", author_email = "sjplimp@sandia.gov", url = "https://www.lammps.org", + project_urls = { + "Bug Tracker": "https://github.com/lammps/lammps/issues", + }, description = "LAMMPS Molecular Dynamics Python package", + long_description = long_description, + long_description_content_type = "text/plain", + classifiers = [ + "Programming Language :: Python :: 3", + "Development Status :: 5 - Production/Stable", + "Environment :: Console", + "License :: OSI Approved :: GNU General Public License v2 (GPLv2)", + "Operating System :: OS Independent", + ], license = "GPL", - packages=pkgs, + packages = pkgs, + package_data = pkgdata, + distclass = bdist, ) diff --git a/python/wheel_requirements.txt b/python/wheel_requirements.txt new file mode 100644 index 0000000000..dafedeee23 --- /dev/null +++ b/python/wheel_requirements.txt @@ -0,0 +1,4 @@ +pip +build +wheel +setuptools From 4b7731e831aa6667445e58d9a966c41d628417a8 Mon Sep 17 00:00:00 2001 From: Axel Kohlmeyer Date: Thu, 24 Feb 2022 21:47:37 -0500 Subject: [PATCH 02/18] update install.py script to build a wheel and install that --- cmake/CMakeLists.txt | 4 +- python/install.py | 111 +++++++++---------------------------------- python/makewheel.py | 15 ++++++ python/setup.py | 9 ++-- src/Makefile | 3 +- 5 files changed, 44 insertions(+), 98 deletions(-) create mode 100644 python/makewheel.py diff --git a/cmake/CMakeLists.txt b/cmake/CMakeLists.txt index 3f409c3820..6b49bea1c4 100644 --- a/cmake/CMakeLists.txt +++ b/cmake/CMakeLists.txt @@ -751,10 +751,8 @@ if(BUILD_SHARED_LIBS) if(Python_EXECUTABLE) add_custom_target( install-python ${CMAKE_COMMAND} -E remove_directory build - COMMAND ${Python_EXECUTABLE} install.py -v ${LAMMPS_SOURCE_DIR}/version.h - -p ${LAMMPS_PYTHON_DIR}/lammps + COMMAND ${Python_EXECUTABLE} ${LAMMPS_PYTHON_DIR}/install.py -p ${LAMMPS_PYTHON_DIR}/lammps -l ${CMAKE_BINARY_DIR}/liblammps${LAMMPS_MACHINE}${CMAKE_SHARED_LIBRARY_SUFFIX} - WORKING_DIRECTORY ${LAMMPS_PYTHON_DIR} COMMENT "Installing LAMMPS Python module") else() add_custom_target( diff --git a/python/install.py b/python/install.py index a3668754d9..32b39fb421 100644 --- a/python/install.py +++ b/python/install.py @@ -10,7 +10,7 @@ build target in the conventional and CMake based build systems # copy LAMMPS shared library and lammps package to system dirs from __future__ import print_function -import sys,os,shutil,time +import sys,os,shutil,time,glob from argparse import ArgumentParser parser = ArgumentParser(prog='install.py', @@ -20,11 +20,6 @@ parser.add_argument("-p", "--package", required=True, help="path to the LAMMPS Python package") parser.add_argument("-l", "--lib", required=True, help="path to the compiled LAMMPS shared library") -parser.add_argument("-v", "--version", required=True, - help="path to the LAMMPS version.h header file") - -parser.add_argument("-d","--dir", - help="Legacy custom installation folder selection for package and library") args = parser.parse_args() @@ -46,91 +41,31 @@ if args.lib: else: args.lib = os.path.abspath(args.lib) -if args.version: - if not os.path.exists(args.version): - print( "ERROR: LAMMPS version header file %s does not exist" % args.version) - parser.print_help() - sys.exit(1) - else: - args.version = os.path.abspath(args.version) - -if args.dir: - if not os.path.isdir(args.dir): - print( "ERROR: Installation folder %s does not exist" % args.dir) - parser.print_help() - sys.exit(1) - else: - args.dir = os.path.abspath(args.dir) - -# if a custom directory is given, we copy the files directly -# without any special processing or additional steps to that folder - -if args.dir: - print("Copying LAMMPS Python package to custom folder %s" % args.dir) - try: - shutil.copytree(args.package, os.path.join(args.dir,'lammps')) - except shutil.Error: - pass # fail silently - - print("Copying LAMMPS shared library to custom folder %s" % args.dir) - try: - shutil.copyfile(args.lib, os.path.join(args.dir,os.path.basename(args.lib))) - except shutil.Error: - pass # fail silently - - sys.exit() - -# extract LAMMPS version string from header -# and convert to python packaging compatible version -def get_lammps_version(header): - with open(header, 'r') as f: - line = f.readline() - start_pos = line.find('"')+1 - end_pos = line.find('"', start_pos) - t = time.strptime("".join(line[start_pos:end_pos].split()), "%d%b%Y") - return "{}.{}.{}".format(t.tm_year,t.tm_mon,t.tm_mday) - -verstr = get_lammps_version(args.version) - -print("Installing LAMMPS Python package version %s into site-packages folder" % verstr) - # we need to switch to the folder of the python package +olddir = os.path.abspath('.') os.chdir(os.path.dirname(args.package)) -from distutils.core import setup -from distutils.sysconfig import get_python_lib -import site -from sys import version_info +print("Purging existing wheels...") +for wheel in glob.glob('lammps-*.whl'): + print("deleting " + wheel) + os.remove(wheel) -if version_info.major >= 3: - pkgs = ['lammps', 'lammps.mliap'] -else: - pkgs = ['lammps'] +# create virtual environment for building the wheel +shutil.rmtree('buildwheel',True) +os.putenv('LAMMPS_SHARED_LIB',args.lib) +#os.environ['LAMMPS_SHARED_LIB'] = args.lib +shutil.copy(args.lib,'lammps') +os.system(sys.executable + ' -m virtualenv buildwheel') +os.system(sys.executable + ' makewheel.py') -#Arguments common to global or user install -- everything but data_files -setup_kwargs= dict(name="lammps", - version=verstr, - author="Steve Plimpton", - author_email="sjplimp@sandia.gov", - url="https://www.lammps.org", - description="LAMMPS Molecular Dynamics Python package", - license="GPL", - packages=pkgs, - ) +# remove temporary folders and files +shutil.rmtree('buildwheel',True) +shutil.rmtree('build',True) +shutil.rmtree('lammps.egg-info',True) +os.remove(os.path.join('lammps',os.path.basename(args.lib))) -tryuser=False -try: - sys.argv = ["setup.py","install"] # as if had run "python setup.py install" - setup_kwargs['data_files']=[(os.path.join(get_python_lib(), 'lammps'), [args.lib])] - setup(**setup_kwargs) -except: # lgtm [py/catch-base-exception] - tryuser=True - print ("Installation into global site-packages folder failed.\nTrying user folder %s now." % site.USER_SITE) - -if tryuser: - try: - sys.argv = ["setup.py","install","--user"] # as if had run "python setup.py install --user" - setup_kwargs['data_files']=[(os.path.join(site.USER_SITE, 'lammps'), [args.lib])] - setup(**setup_kwargs) - except: # lgtm [py/catch-base-exception] - print("Installation into user site package folder failed.") +print("Installing wheel") +for wheel in glob.glob('lammps-*.whl'): + os.system(sys.executable + " -m pip install --force-reinstall " + wheel) + shutil.copy(wheel, olddir) + os.remove(wheel) diff --git a/python/makewheel.py b/python/makewheel.py new file mode 100644 index 0000000000..99e0ffdab6 --- /dev/null +++ b/python/makewheel.py @@ -0,0 +1,15 @@ +#!/usr/bin/env python + +import sys,os,shutil + +if sys.platform == 'win32': + virtenv=os.path.join('buildwheel','Scripts','activate_this.py') +else: + virtenv=os.path.join('buildwheel','bin','activate_this.py') + +exec(open(virtenv).read(), {'__file__': virtenv}) + +os.system('python -m pip install --upgrade pip') +os.system('python -m pip install --upgrade -r wheel_requirements.txt') +print("Building new binary wheel") +os.system('python -m build --wheel -o .') diff --git a/python/setup.py b/python/setup.py index 6a271a1d26..8071af2a7f 100644 --- a/python/setup.py +++ b/python/setup.py @@ -3,7 +3,7 @@ from setuptools import setup from setuptools.dist import Distribution from sys import version_info -import os,time +import os,time,shutil LAMMPS_PYTHON_DIR = os.path.dirname(os.path.realpath(__file__)) LAMMPS_DIR = os.path.dirname(LAMMPS_PYTHON_DIR) LAMMPS_SOURCE_DIR = os.path.join(LAMMPS_DIR, 'src') @@ -27,8 +27,7 @@ class BinaryDistribution(Distribution): def has_ext_modules(foo): return True -libpath = os.environ.get("LAMMPS_SHARED_LIB") - +libname = os.path.basename(os.environ.get("LAMMPS_SHARED_LIB")) if version_info.major >= 3: pkgs = ['lammps', 'lammps.mliap'] else: @@ -37,8 +36,8 @@ else: with open("README", "r", encoding="utf-8") as fh: long_description = fh.read() -if libpath: - pkgdata = {'lammps': [ libpath ]} +if libname: + pkgdata = {'lammps': [ libname ]} bdist = BinaryDistribution else: pkgdata = {} diff --git a/src/Makefile b/src/Makefile index 44cebcb20c..6dfa0357ce 100644 --- a/src/Makefile +++ b/src/Makefile @@ -458,8 +458,7 @@ mpi-stubs: sinclude ../lib/python/Makefile.lammps install-python: @rm -rf ../python/build - @$(PYTHON) ../python/install.py -v ../src/version.h \ - -p ../python/lammps -l ../src/liblammps.so + @$(PYTHON) ../python/install.py -p ../python/lammps -l ../src/liblammps.so # Create a tarball of src dir and packages From e2092a578897d75fa8522d247a361159787f8267 Mon Sep 17 00:00:00 2001 From: Axel Kohlmeyer Date: Fri, 25 Feb 2022 00:52:13 -0500 Subject: [PATCH 03/18] first attempt at supporting multi-config builders --- cmake/CMakeLists.txt | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/cmake/CMakeLists.txt b/cmake/CMakeLists.txt index 6b49bea1c4..0b270cc68f 100644 --- a/cmake/CMakeLists.txt +++ b/cmake/CMakeLists.txt @@ -16,6 +16,7 @@ endif() project(lammps CXX) set(SOVERSION 0) +get_property(BUILD_IS_MULTI_CONFIG GLOBAL PROPERTY GENERATOR_IS_MULTI_CONFIG) get_filename_component(LAMMPS_DIR ${CMAKE_CURRENT_SOURCE_DIR}/.. ABSOLUTE) get_filename_component(LAMMPS_LIB_BINARY_DIR ${CMAKE_BINARY_DIR}/lib ABSOLUTE) @@ -748,11 +749,15 @@ if(BUILD_SHARED_LIBS) else() find_package(Python COMPONENTS Interpreter) endif() + if(BUILD_IS_MULTI_CONFIG) + set(LIBLAMMPS_SHARED_BINARY ${CMAKE_BINARY_DIR}/$/liblammps${LAMMPS_MACHINE}${CMAKE_SHARED_LIBRARY_SUFFIX}) + else() + set(LIBLAMMPS_SHARED_BINARY ${CMAKE_BINARY_DIR}/liblammps${LAMMPS_MACHINE}${CMAKE_SHARED_LIBRARY_SUFFIX}) + endif() if(Python_EXECUTABLE) add_custom_target( install-python ${CMAKE_COMMAND} -E remove_directory build - COMMAND ${Python_EXECUTABLE} ${LAMMPS_PYTHON_DIR}/install.py -p ${LAMMPS_PYTHON_DIR}/lammps - -l ${CMAKE_BINARY_DIR}/liblammps${LAMMPS_MACHINE}${CMAKE_SHARED_LIBRARY_SUFFIX} + COMMAND ${Python_EXECUTABLE} ${LAMMPS_PYTHON_DIR}/install.py -p ${LAMMPS_PYTHON_DIR}/lammps -l ${LIBLAMMPS_SHARED_BINARY} COMMENT "Installing LAMMPS Python module") else() add_custom_target( @@ -797,7 +802,6 @@ if(ClangFormat_FOUND) endif() get_target_property(DEFINES lammps COMPILE_DEFINITIONS) -get_property(BUILD_IS_MULTI_CONFIG GLOBAL PROPERTY GENERATOR_IS_MULTI_CONFIG) if(BUILD_IS_MULTI_CONFIG) set(LAMMPS_BUILD_TYPE "Multi-Config") else() From 4630a5ffb8e4c3c955edf5deb9f1c7c2d7989ecc Mon Sep 17 00:00:00 2001 From: Axel Kohlmeyer Date: Fri, 25 Feb 2022 08:43:38 -0500 Subject: [PATCH 04/18] must create virtualenv with same python we selected to run install.py --- python/install.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/install.py b/python/install.py index 32b39fb421..3b01a834b5 100644 --- a/python/install.py +++ b/python/install.py @@ -55,7 +55,7 @@ shutil.rmtree('buildwheel',True) os.putenv('LAMMPS_SHARED_LIB',args.lib) #os.environ['LAMMPS_SHARED_LIB'] = args.lib shutil.copy(args.lib,'lammps') -os.system(sys.executable + ' -m virtualenv buildwheel') +os.system(sys.executable + ' -m virtualenv buildwheel -p ' + sys.executable) os.system(sys.executable + ' makewheel.py') # remove temporary folders and files From ff4cd2a5e95311590b8ce9f2de9f89f48de68041 Mon Sep 17 00:00:00 2001 From: Axel Kohlmeyer Date: Fri, 25 Feb 2022 08:43:51 -0500 Subject: [PATCH 05/18] python2.7 compatibility --- python/setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/setup.py b/python/setup.py index 8071af2a7f..aeb5b2d881 100644 --- a/python/setup.py +++ b/python/setup.py @@ -33,7 +33,7 @@ if version_info.major >= 3: else: pkgs = ['lammps'] -with open("README", "r", encoding="utf-8") as fh: +with open("README", "r") as fh: long_description = fh.read() if libname: From 59e2b819e0b3f5dd9f5c3d87829dcc220b17d307 Mon Sep 17 00:00:00 2001 From: Axel Kohlmeyer Date: Fri, 25 Feb 2022 08:44:15 -0500 Subject: [PATCH 06/18] need python3-venv package for updated make install-python --- tools/singularity/ubuntu18.04.def | 1 + tools/singularity/ubuntu18.04_amd_rocm.def | 1 + tools/singularity/ubuntu18.04_gpu.def | 1 + tools/singularity/ubuntu18.04_intel_opencl.def | 1 + tools/singularity/ubuntu18.04_nvidia.def | 1 + 5 files changed, 5 insertions(+) diff --git a/tools/singularity/ubuntu18.04.def b/tools/singularity/ubuntu18.04.def index 1364b8cc4b..ebea335d6b 100644 --- a/tools/singularity/ubuntu18.04.def +++ b/tools/singularity/ubuntu18.04.def @@ -56,6 +56,7 @@ From: ubuntu:18.04 python3-pkg-resources \ python3-setuptools \ python3-virtualenv \ + python3-venv \ rsync \ ssh \ texlive \ diff --git a/tools/singularity/ubuntu18.04_amd_rocm.def b/tools/singularity/ubuntu18.04_amd_rocm.def index a99f4931ef..f1d327052d 100644 --- a/tools/singularity/ubuntu18.04_amd_rocm.def +++ b/tools/singularity/ubuntu18.04_amd_rocm.def @@ -95,6 +95,7 @@ From: ubuntu:18.04 python3-pkg-resources \ python3-setuptools \ python3-virtualenv \ + python3-venv \ rsync \ ssh \ vim-nox \ diff --git a/tools/singularity/ubuntu18.04_gpu.def b/tools/singularity/ubuntu18.04_gpu.def index cc02280d8a..452a3eaee8 100644 --- a/tools/singularity/ubuntu18.04_gpu.def +++ b/tools/singularity/ubuntu18.04_gpu.def @@ -100,6 +100,7 @@ From: ubuntu:18.04 python3-pkg-resources \ python3-setuptools \ python3-virtualenv \ + python3-venv \ rsync \ ssh \ vim-nox \ diff --git a/tools/singularity/ubuntu18.04_intel_opencl.def b/tools/singularity/ubuntu18.04_intel_opencl.def index 3bb8b990d1..fd7363dbae 100644 --- a/tools/singularity/ubuntu18.04_intel_opencl.def +++ b/tools/singularity/ubuntu18.04_intel_opencl.def @@ -59,6 +59,7 @@ From: ubuntu:18.04 python3-pkg-resources \ python3-setuptools \ python3-virtualenv \ + python3-venv \ rsync \ ssh \ vim-nox \ diff --git a/tools/singularity/ubuntu18.04_nvidia.def b/tools/singularity/ubuntu18.04_nvidia.def index a97e4a52ec..f834e1a91e 100644 --- a/tools/singularity/ubuntu18.04_nvidia.def +++ b/tools/singularity/ubuntu18.04_nvidia.def @@ -59,6 +59,7 @@ From: nvidia/cuda:11.4.2-devel-ubuntu18.04 python3-pkg-resources \ python3-setuptools \ python3-virtualenv \ + python3-venv \ rsync \ ssh \ vim-nox \ From 6ab247e7f8495a1678bbb5eb0cb5bf78b2faaf4f Mon Sep 17 00:00:00 2001 From: Axel Kohlmeyer Date: Fri, 25 Feb 2022 09:12:01 -0500 Subject: [PATCH 07/18] no need to create a virtualenv inside a temporal virtualenv --- python/makewheel.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/makewheel.py b/python/makewheel.py index 99e0ffdab6..cd7389e606 100644 --- a/python/makewheel.py +++ b/python/makewheel.py @@ -12,4 +12,4 @@ exec(open(virtenv).read(), {'__file__': virtenv}) os.system('python -m pip install --upgrade pip') os.system('python -m pip install --upgrade -r wheel_requirements.txt') print("Building new binary wheel") -os.system('python -m build --wheel -o .') +os.system('python -m build -n --wheel -o .') From 257805d325c625f92b42d3abf28215818dd41c63 Mon Sep 17 00:00:00 2001 From: Axel Kohlmeyer Date: Fri, 25 Feb 2022 09:32:41 -0500 Subject: [PATCH 08/18] make prompt name consistent --- tools/singularity/rocky8.def | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/singularity/rocky8.def b/tools/singularity/rocky8.def index ea74a50339..098fd2b868 100644 --- a/tools/singularity/rocky8.def +++ b/tools/singularity/rocky8.def @@ -74,7 +74,7 @@ EOF CUSTOM_PROMPT_ENV=/.singularity.d/env/99-zz_custom_prompt.sh cat >$CUSTOM_PROMPT_ENV < Date: Fri, 25 Feb 2022 09:33:26 -0500 Subject: [PATCH 09/18] restore exception handling with enforce --user install for older pip versions that don't do that automatically --- python/install.py | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/python/install.py b/python/install.py index 3b01a834b5..4a6d4d3772 100644 --- a/python/install.py +++ b/python/install.py @@ -10,7 +10,7 @@ build target in the conventional and CMake based build systems # copy LAMMPS shared library and lammps package to system dirs from __future__ import print_function -import sys,os,shutil,time,glob +import sys,os,shutil,time,glob,subprocess from argparse import ArgumentParser parser = ArgumentParser(prog='install.py', @@ -47,8 +47,8 @@ os.chdir(os.path.dirname(args.package)) print("Purging existing wheels...") for wheel in glob.glob('lammps-*.whl'): - print("deleting " + wheel) - os.remove(wheel) + print("deleting " + wheel) + os.remove(wheel) # create virtual environment for building the wheel shutil.rmtree('buildwheel',True) @@ -66,6 +66,17 @@ os.remove(os.path.join('lammps',os.path.basename(args.lib))) print("Installing wheel") for wheel in glob.glob('lammps-*.whl'): - os.system(sys.executable + " -m pip install --force-reinstall " + wheel) + try: + txt = subprocess.check_output([sys.executable, '-m', 'pip', 'install', '--force-reinstall', wheel], stderr=subprocess.STDOUT, shell=False) + print(txt.decode('UTF-8')) + continue + except: + pass + try: + print('Installing wheel into standard site-packages folder failed. Trying user folder now') + txt = subprocess.check_output([sys.executable, '-m', 'pip', 'install', '--user', '--force-reinstall', wheel], stderr=subprocess.STDOUT, shell=False) + print(txt.decode('UTF-8')) + except: + sys.exit('Failed to install wheel ' + wheel) shutil.copy(wheel, olddir) os.remove(wheel) From 033045a534fea4a84aa18b04c743bac72e7bda9a Mon Sep 17 00:00:00 2001 From: Axel Kohlmeyer Date: Fri, 25 Feb 2022 13:20:34 -0500 Subject: [PATCH 10/18] need to check if virtualenv creation failed and exit with a suitable error message --- python/install.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/python/install.py b/python/install.py index 4a6d4d3772..28ab758a07 100644 --- a/python/install.py +++ b/python/install.py @@ -55,7 +55,12 @@ shutil.rmtree('buildwheel',True) os.putenv('LAMMPS_SHARED_LIB',args.lib) #os.environ['LAMMPS_SHARED_LIB'] = args.lib shutil.copy(args.lib,'lammps') -os.system(sys.executable + ' -m virtualenv buildwheel -p ' + sys.executable) +try: + txt = subprocess.check_output([sys.executable, '-m', 'virtualenv', 'buildwheel', '-p', sys.executable], stderr=subprocess.STDOUT, shell=False) + print(txt.decode('UTF-8')) +except subprocess.CalledProcessError as err: + sys.exit("Failed to create a virtualenv: {0}".format(err.output.decode('UTF-8'))) + os.system(sys.executable + ' makewheel.py') # remove temporary folders and files From 511678017e9bcab7865c3dbea3881fc2f32220d0 Mon Sep 17 00:00:00 2001 From: Axel Kohlmeyer Date: Fri, 25 Feb 2022 13:44:47 -0500 Subject: [PATCH 11/18] update inline docs and add -n/--noinstall flag to skipp installation --- python/install.py | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/python/install.py b/python/install.py index 28ab758a07..7cce42d2ba 100644 --- a/python/install.py +++ b/python/install.py @@ -1,10 +1,13 @@ #!/usr/bin/env python """ -Installer script to install the LAMMPS python package and the corresponding -shared library into either the system-wide site-packages tree, or - failing -that - into the corresponding user tree. Called from the 'install-python' -build target in the conventional and CMake based build systems +Script to build a "binary wheel" for the 'pip' Python package manager for +the LAMMPS python module which includes the shared library file. After a +successful build the script attempts to install the wheel into a system +specific site-packages folder or - failing that - into the corresponding +user site-packages folder. Called from the 'install-python' build target +in the GNU make and CMake based build systems. Can also be called +independently and used to build the wheel without installing it. """ # copy LAMMPS shared library and lammps package to system dirs @@ -20,6 +23,8 @@ parser.add_argument("-p", "--package", required=True, help="path to the LAMMPS Python package") parser.add_argument("-l", "--lib", required=True, help="path to the compiled LAMMPS shared library") +parser.add_argument("-n", "--noinstall", action="store_true", default=False, + help="only build a binary wheel. Don't attempt to install it") args = parser.parse_args() @@ -69,6 +74,9 @@ shutil.rmtree('build',True) shutil.rmtree('lammps.egg-info',True) os.remove(os.path.join('lammps',os.path.basename(args.lib))) +if args.noinstall: + exit(0) + print("Installing wheel") for wheel in glob.glob('lammps-*.whl'): try: From bb3a5b40573707ab48405b0dd36b6dd5b9295ea5 Mon Sep 17 00:00:00 2001 From: Axel Kohlmeyer Date: Fri, 25 Feb 2022 14:11:22 -0500 Subject: [PATCH 12/18] update docs --- doc/src/Developer_comm_ops.rst | 2 +- doc/src/Python_install.rst | 101 +++++++++++++++------------------ 2 files changed, 46 insertions(+), 57 deletions(-) diff --git a/doc/src/Developer_comm_ops.rst b/doc/src/Developer_comm_ops.rst index c234793946..d343fc073a 100644 --- a/doc/src/Developer_comm_ops.rst +++ b/doc/src/Developer_comm_ops.rst @@ -90,7 +90,7 @@ For *Fix* classes there is an optional second argument to the fix performs multiple modes of communication, with different numbers of values per atom. The fix should set the *comm_forward* and *comm_reverse* variables to the maximum value, but can invoke the -communication for a particulard mode with a smaller value. For this +communication for a particular mode with a smaller value. For this to work, the *pack_forward_comm()*, etc methods typically use a class member variable to choose which values to pack/unpack into/from the buffer. diff --git a/doc/src/Python_install.rst b/doc/src/Python_install.rst index abf96accbf..79cbca6242 100644 --- a/doc/src/Python_install.rst +++ b/doc/src/Python_install.rst @@ -25,11 +25,10 @@ Installing the LAMMPS Python Module and Shared Library ====================================================== Making LAMMPS usable within Python and vice versa requires putting the -LAMMPS Python package (``lammps``) into a location where the -Python interpreter can find it and installing the LAMMPS shared library -into a folder that the dynamic loader searches or inside of the installed -``lammps`` package folder. There are multiple ways to achieve -this. +LAMMPS Python package (``lammps``) into a location where the Python +interpreter can find it and installing the LAMMPS shared library into a +folder that the dynamic loader searches or inside of the installed +``lammps`` package folder. There are multiple ways to achieve this. #. Do a full LAMMPS installation of libraries, executables, selected headers, documentation (if enabled), and supporting files (only @@ -159,33 +158,30 @@ this. make install-python - This will try to install (only) the shared library and the Python - package into a system folder and if that fails (due to missing - write permissions) will instead do the installation to a user - folder under ``$HOME/.local``. For a system-wide installation you + This will try to build a so-called (binary) 'wheel', a compressed + binary python package and then install it with the python package + manager 'pip'. Installation will be attempted into a system-wide + ``site-packages`` folder and if that fails into the corresponding + folder in the user's home directory. For a system-wide installation you would have to gain superuser privilege, e.g. though ``sudo`` - +------------------------+-----------------------------------------------------------------+-------------------------------------------------------------+ - | File | Location | Notes | - +========================+=================================================================+=============================================================+ - | LAMMPS Python package | * ``$HOME/.local/lib/pythonX.Y/site-packages/lammps`` (32bit) | ``X.Y`` depends on the installed Python version | - | | * ``$HOME/.local/lib64/pythonX.Y/site-packages/lammps`` (64bit) | | - +------------------------+-----------------------------------------------------------------+-------------------------------------------------------------+ - | LAMMPS shared library | * ``$HOME/.local/lib/pythonX.Y/site-packages/lammps`` (32bit) | ``X.Y`` depends on the installed Python version | - | | * ``$HOME/.local/lib64/pythonX.Y/site-packages/lammps`` (64bit) | | - +------------------------+-----------------------------------------------------------------+-------------------------------------------------------------+ + +------------------------+----------------------------------------------------------+-------------------------------------------------------------+ + | File | Location | Notes | + +========================+==========================================================+=============================================================+ + | LAMMPS Python package | * ``$HOME/.local/lib/pythonX.Y/site-packages/lammps`` | ``X.Y`` depends on the installed Python version | + +------------------------+----------------------------------------------------------+-------------------------------------------------------------+ + | LAMMPS shared library | * ``$HOME/.local/lib/pythonX.Y/site-packages/lammps`` | ``X.Y`` depends on the installed Python version | + +------------------------+----------------------------------------------------------+-------------------------------------------------------------+ For a system-wide installation those folders would then become. - +------------------------+---------------------------------------------------------+-------------------------------------------------------------+ - | File | Location | Notes | - +========================+=========================================================+=============================================================+ - | LAMMPS Python package | * ``/usr/lib/pythonX.Y/site-packages/lammps`` (32bit) | ``X.Y`` depends on the installed Python version | - | | * ``/usr/lib64/pythonX.Y/site-packages/lammps`` (64bit) | | - +------------------------+---------------------------------------------------------+-------------------------------------------------------------+ - | LAMMPS shared library | * ``/usr/lib/pythonX.Y/site-packages/lammps`` (32bit) | ``X.Y`` depends on the installed Python version | - | | * ``/usr/lib64/pythonX.Y/site-packages/lammps`` (64bit) | | - +------------------------+---------------------------------------------------------+-------------------------------------------------------------+ + +------------------------+-------------------------------------------------+-------------------------------------------------------------+ + | File | Location | Notes | + +========================+=================================================+=============================================================+ + | LAMMPS Python package | * ``/usr/lib/pythonX.Y/site-packages/lammps`` | ``X.Y`` depends on the installed Python version | + +------------------------+-------------------------------------------------+-------------------------------------------------------------+ + | LAMMPS shared library | * ``/usr/lib/pythonX.Y/site-packages/lammps`` | ``X.Y`` depends on the installed Python version | + +------------------------+-------------------------------------------------+-------------------------------------------------------------+ No environment variables need to be set for those, as those folders are searched by default by Python or the LAMMPS Python @@ -201,16 +197,12 @@ this. .. code-block:: bash - $ python install.py -p -l -v [-d ] + $ python install.py -p -l [-n] * The ``-p`` flag points to the ``lammps`` Python package folder to be installed, * the ``-l`` flag points to the LAMMPS shared library file to be installed, - * the ``-v`` flag points to the ``version.h`` file in the LAMMPS source - * and the optional ``-d`` flag to a custom (legacy) installation folder - - If you use a legacy installation folder, you will need to set your - ``PYTHONPATH`` and ``LD_LIBRARY_PATH`` (and/or ``DYLD_LIBRARY_PATH``) environment - variables accordingly as explained in the description for "In place use". + * and the optional ``-n`` instructs the script to only build a wheel file + but not attempt to install it. .. tab:: Virtual environment @@ -257,32 +249,29 @@ this. package and the shared library file are installed into the following locations: - +------------------------+-----------------------------------------------------------------+-------------------------------------------------------------+ - | File | Location | Notes | - +========================+=================================================================+=============================================================+ - | LAMMPS Python Module | * ``$VIRTUAL_ENV/lib/pythonX.Y/site-packages/lammps`` (32bit) | ``X.Y`` depends on the installed Python version | - | | * ``$VIRTUAL_ENV/lib64/pythonX.Y/site-packages/lammps`` (64bit) | | - +------------------------+-----------------------------------------------------------------+-------------------------------------------------------------+ - | LAMMPS shared library | * ``$VIRTUAL_ENV/lib/pythonX.Y/site-packages/lammps`` (32bit) | ``X.Y`` depends on the installed Python version | - | | * ``$VIRTUAL_ENV/lib64/pythonX.Y/site-packages/lammps`` (64bit) | | - +------------------------+-----------------------------------------------------------------+-------------------------------------------------------------+ + +------------------------+--------------------------------------------------------+-------------------------------------------------------------+ + | File | Location | Notes | + +========================+========================================================+=============================================================+ + | LAMMPS Python Module | * ``$VIRTUAL_ENV/lib/pythonX.Y/site-packages/lammps`` | ``X.Y`` depends on the installed Python version | + +------------------------+--------------------------------------------------------+-------------------------------------------------------------+ + | LAMMPS shared library | * ``$VIRTUAL_ENV/lib/pythonX.Y/site-packages/lammps`` | ``X.Y`` depends on the installed Python version | + +------------------------+--------------------------------------------------------+-------------------------------------------------------------+ If you do a full installation (CMake only) with "install", this leads to the following installation locations: - +------------------------+-----------------------------------------------------------------+-------------------------------------------------------------+ - | File | Location | Notes | - +========================+=================================================================+=============================================================+ - | LAMMPS Python Module | * ``$VIRTUAL_ENV/lib/pythonX.Y/site-packages/lammps`` (32bit) | ``X.Y`` depends on the installed Python version | - | | * ``$VIRTUAL_ENV/lib64/pythonX.Y/site-packages/lammps`` (64bit) | | - +------------------------+-----------------------------------------------------------------+-------------------------------------------------------------+ - | LAMMPS shared library | * ``$VIRTUAL_ENV/lib/`` (32bit) | Set shared loader environment variable to this path | - | | * ``$VIRTUAL_ENV/lib64/`` (64bit) | (see below for more info on this) | - +------------------------+-----------------------------------------------------------------+-------------------------------------------------------------+ - | LAMMPS executable | * ``$VIRTUAL_ENV/bin/`` | | - +------------------------+-----------------------------------------------------------------+-------------------------------------------------------------+ - | LAMMPS potential files | * ``$VIRTUAL_ENV/share/lammps/potentials/`` | Set ``LAMMPS_POTENTIALS`` environment variable to this path | - +------------------------+-----------------------------------------------------------------+-------------------------------------------------------------+ + +------------------------+--------------------------------------------------------+-------------------------------------------------------------+ + | File | Location | Notes | + +========================+========================================================+=============================================================+ + | LAMMPS Python Module | * ``$VIRTUAL_ENV/lib/pythonX.Y/site-packages/lammps`` | ``X.Y`` depends on the installed Python version | + +------------------------+--------------------------------------------------------+-------------------------------------------------------------+ + | LAMMPS shared library | * ``$VIRTUAL_ENV/lib/`` (32bit) | Set shared loader environment variable to this path | + | | * ``$VIRTUAL_ENV/lib64/`` (64bit) | (see below for more info on this) | + +------------------------+--------------------------------------------------------+-------------------------------------------------------------+ + | LAMMPS executable | * ``$VIRTUAL_ENV/bin/`` | | + +------------------------+--------------------------------------------------------+-------------------------------------------------------------+ + | LAMMPS potential files | * ``$VIRTUAL_ENV/share/lammps/potentials/`` | Set ``LAMMPS_POTENTIALS`` environment variable to this path | + +------------------------+--------------------------------------------------------+-------------------------------------------------------------+ In that case you need to modify the ``$HOME/myenv/bin/activate`` script in a similar fashion you need to update your From 578a7cc54c56653b1d92c2e3afb76f7454802ab6 Mon Sep 17 00:00:00 2001 From: Axel Kohlmeyer Date: Fri, 25 Feb 2022 16:10:46 -0500 Subject: [PATCH 13/18] document and handle the case of a previous distutils installation. in this case 'make install-python' should not continue but instead it now aborts and asks the user to do a manual uninstall --- doc/src/Python_install.rst | 17 +++++++++++++++++ python/install.py | 6 ++++-- 2 files changed, 21 insertions(+), 2 deletions(-) diff --git a/doc/src/Python_install.rst b/doc/src/Python_install.rst index 79cbca6242..aaeeec18f1 100644 --- a/doc/src/Python_install.rst +++ b/doc/src/Python_install.rst @@ -187,6 +187,23 @@ folder that the dynamic loader searches or inside of the installed folders are searched by default by Python or the LAMMPS Python package. + .. versionchanged:: TBD + + .. note:: + + If there is an existing installation of the LAMMPS python + module, ``make install-python`` will try to update it. + However, that will fail if the older version of the module + was installed by LAMMPS versions until 17Feb2022. Those + were using the distutils package, which does not create a + "manifest" that allows a clean uninstall. The ``make + install-python`` command will always produce a + lammps-----.whl file (the + 'wheel'). And this file can be later installed directly with + ``python -m pip install .whl`` without having to + type ``make install-python`` again and repeating the build + step, too. + For the traditional make process you can override the python version to version x.y when calling ``make`` with ``PYTHON=pythonX.Y``. For a CMake based compilation this choice diff --git a/python/install.py b/python/install.py index 7cce42d2ba..e4a8b6af7b 100644 --- a/python/install.py +++ b/python/install.py @@ -83,8 +83,10 @@ for wheel in glob.glob('lammps-*.whl'): txt = subprocess.check_output([sys.executable, '-m', 'pip', 'install', '--force-reinstall', wheel], stderr=subprocess.STDOUT, shell=False) print(txt.decode('UTF-8')) continue - except: - pass + except subprocess.CalledProcessError as err: + errmsg = err.output.decode('UTF-8') + if errmsg.find("distutils installed"): + sys.exit(errmsg + "You need to uninstall the LAMMPS python module manually first.\n") try: print('Installing wheel into standard site-packages folder failed. Trying user folder now') txt = subprocess.check_output([sys.executable, '-m', 'pip', 'install', '--user', '--force-reinstall', wheel], stderr=subprocess.STDOUT, shell=False) From 1dbff92eeb9fbcb57756795f10bcae3abf1e41fa Mon Sep 17 00:00:00 2001 From: Axel Kohlmeyer Date: Fri, 25 Feb 2022 16:23:48 -0500 Subject: [PATCH 14/18] spelling --- doc/utils/sphinx-config/false_positives.txt | 3 +++ 1 file changed, 3 insertions(+) diff --git a/doc/utils/sphinx-config/false_positives.txt b/doc/utils/sphinx-config/false_positives.txt index 18ae834078..31e03140f1 100644 --- a/doc/utils/sphinx-config/false_positives.txt +++ b/doc/utils/sphinx-config/false_positives.txt @@ -3,6 +3,7 @@ aat abc abf ABI +abi abo Abramyan absTol @@ -2450,6 +2451,7 @@ ortho orthonormal orthorhombic Ortner +os oso Otype Ouadfel @@ -3616,6 +3618,7 @@ wget Whelan whitesmoke whitespace +whl Wi Wicaksono widom From 7fd41bea6a9a55f8e7f55f3308dd6e53f09fc189 Mon Sep 17 00:00:00 2001 From: Axel Kohlmeyer Date: Fri, 25 Feb 2022 20:10:13 -0500 Subject: [PATCH 15/18] add some comments to the refactored scripts to explain technical details --- python/install.py | 26 +++++++++++++++++++++----- python/makewheel.py | 3 +++ 2 files changed, 24 insertions(+), 5 deletions(-) diff --git a/python/install.py b/python/install.py index e4a8b6af7b..efb056df6b 100644 --- a/python/install.py +++ b/python/install.py @@ -10,8 +10,6 @@ in the GNU make and CMake based build systems. Can also be called independently and used to build the wheel without installing it. """ -# copy LAMMPS shared library and lammps package to system dirs - from __future__ import print_function import sys,os,shutil,time,glob,subprocess from argparse import ArgumentParser @@ -50,22 +48,28 @@ if args.lib: olddir = os.path.abspath('.') os.chdir(os.path.dirname(args.package)) +# remove any wheel files left over from previous calls print("Purging existing wheels...") for wheel in glob.glob('lammps-*.whl'): print("deleting " + wheel) os.remove(wheel) -# create virtual environment for building the wheel -shutil.rmtree('buildwheel',True) +# copy shared object to the current folder so that +# it will show up in the installation at the expected location os.putenv('LAMMPS_SHARED_LIB',args.lib) -#os.environ['LAMMPS_SHARED_LIB'] = args.lib shutil.copy(args.lib,'lammps') + +# create a virtual environment for building the wheel +shutil.rmtree('buildwheel',True) try: txt = subprocess.check_output([sys.executable, '-m', 'virtualenv', 'buildwheel', '-p', sys.executable], stderr=subprocess.STDOUT, shell=False) print(txt.decode('UTF-8')) except subprocess.CalledProcessError as err: sys.exit("Failed to create a virtualenv: {0}".format(err.output.decode('UTF-8'))) +# now run the commands to build the wheel. those must be in a separate script +# and run in subprocess, since this will use the virtual environment and +# there is no simple way to return from that in python. os.system(sys.executable + ' makewheel.py') # remove temporary folders and files @@ -74,9 +78,21 @@ shutil.rmtree('build',True) shutil.rmtree('lammps.egg-info',True) os.remove(os.path.join('lammps',os.path.basename(args.lib))) +# stop here if we were asked not to install the wheel we created if args.noinstall: exit(0) +# install the wheel with pip. first try to install in the default environment. +# that will be a virtual environment, if active, or the system folder. +# recent versions of pip will automatically drop to use the user folder +# in case the system folder is not writable. + +# we use a subprocess so we can catch an exception on failure. +# we need to check whether pip refused to install because of a +# version of the module previously installed with distutils. those +# must be uninstalled manually. We must not ignore this and drop +# back to install into a (forced) user folder. + print("Installing wheel") for wheel in glob.glob('lammps-*.whl'): try: diff --git a/python/makewheel.py b/python/makewheel.py index cd7389e606..64ecbe2464 100644 --- a/python/makewheel.py +++ b/python/makewheel.py @@ -2,6 +2,7 @@ import sys,os,shutil +# find python script to activate the virtual environment and source it if sys.platform == 'win32': virtenv=os.path.join('buildwheel','Scripts','activate_this.py') else: @@ -9,7 +10,9 @@ else: exec(open(virtenv).read(), {'__file__': virtenv}) +# update pip and install all requirements to build the wheel os.system('python -m pip install --upgrade pip') os.system('python -m pip install --upgrade -r wheel_requirements.txt') + print("Building new binary wheel") os.system('python -m build -n --wheel -o .') From 7c11d1675d415639aedff05ca42b1b127507b081 Mon Sep 17 00:00:00 2001 From: Axel Kohlmeyer Date: Sun, 27 Feb 2022 06:33:22 -0500 Subject: [PATCH 16/18] fix bug with `make install` --- python/setup.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/python/setup.py b/python/setup.py index aeb5b2d881..b3c53819aa 100644 --- a/python/setup.py +++ b/python/setup.py @@ -27,7 +27,6 @@ class BinaryDistribution(Distribution): def has_ext_modules(foo): return True -libname = os.path.basename(os.environ.get("LAMMPS_SHARED_LIB")) if version_info.major >= 3: pkgs = ['lammps', 'lammps.mliap'] else: @@ -36,8 +35,9 @@ else: with open("README", "r") as fh: long_description = fh.read() +libname = os.environ.get("LAMMPS_SHARED_LIB") if libname: - pkgdata = {'lammps': [ libname ]} + pkgdata = {'lammps': [ os.path.basename(libname) ]} bdist = BinaryDistribution else: pkgdata = {} From 0be14d4ed83f3db5281dbbd9823a7074f0be9722 Mon Sep 17 00:00:00 2001 From: Axel Kohlmeyer Date: Sun, 27 Feb 2022 07:02:39 -0500 Subject: [PATCH 17/18] move basename operation from setup.py to install.py --- python/install.py | 2 +- python/setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/python/install.py b/python/install.py index efb056df6b..a71a601c42 100644 --- a/python/install.py +++ b/python/install.py @@ -56,7 +56,7 @@ for wheel in glob.glob('lammps-*.whl'): # copy shared object to the current folder so that # it will show up in the installation at the expected location -os.putenv('LAMMPS_SHARED_LIB',args.lib) +os.putenv('LAMMPS_SHARED_LIB',os.path.basename(args.lib)) shutil.copy(args.lib,'lammps') # create a virtual environment for building the wheel diff --git a/python/setup.py b/python/setup.py index b3c53819aa..0097e3d596 100644 --- a/python/setup.py +++ b/python/setup.py @@ -37,7 +37,7 @@ with open("README", "r") as fh: libname = os.environ.get("LAMMPS_SHARED_LIB") if libname: - pkgdata = {'lammps': [ os.path.basename(libname) ]} + pkgdata = {'lammps': [ libname ]} bdist = BinaryDistribution else: pkgdata = {} From 06f4ae0f1f012b7d7da5c288936b23badb8d78a7 Mon Sep 17 00:00:00 2001 From: Axel Kohlmeyer Date: Sun, 27 Feb 2022 07:03:02 -0500 Subject: [PATCH 18/18] ignore wheel files --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 3fb3af0d13..bd2d0ea705 100644 --- a/.gitignore +++ b/.gitignore @@ -12,6 +12,7 @@ *.sif *.dll *.pyc +*.whl a.out __pycache__