Merge pull request #3306 from akohlmey/makewheel-fix

Fix issues with building a wheel file during 'make install-python'

cmake/CMakeLists.txt

@@ -786,14 +786,16 @@ if(BUILD_SHARED_LIBS)
     find_package(Python COMPONENTS Interpreter)
   endif()
   if(BUILD_IS_MULTI_CONFIG)
-    set(LIBLAMMPS_SHARED_BINARY ${CMAKE_BINARY_DIR}/$<CONFIG>/liblammps${LAMMPS_MACHINE}${CMAKE_SHARED_LIBRARY_SUFFIX})
+    set(MY_BUILD_DIR ${CMAKE_BINARY_DIR}/$<CONFIG>)
   else()
-    set(LIBLAMMPS_SHARED_BINARY ${CMAKE_BINARY_DIR}/liblammps${LAMMPS_MACHINE}${CMAKE_SHARED_LIBRARY_SUFFIX})
+    set(MY_BUILD_DIR ${CMAKE_BINARY_DIR})
   endif()
+  set(LIBLAMMPS_SHARED_BINARY ${MY_BUILD_DIR}/liblammps${LAMMPS_MACHINE}${CMAKE_SHARED_LIBRARY_SUFFIX})
   if(Python_EXECUTABLE)
     add_custom_target(
       install-python ${CMAKE_COMMAND} -E remove_directory build
-      COMMAND ${Python_EXECUTABLE} ${LAMMPS_PYTHON_DIR}/install.py -p ${LAMMPS_PYTHON_DIR}/lammps -l ${LIBLAMMPS_SHARED_BINARY}
+      COMMAND ${Python_EXECUTABLE} ${LAMMPS_PYTHON_DIR}/install.py -p ${LAMMPS_PYTHON_DIR}/lammps
+              -l ${LIBLAMMPS_SHARED_BINARY} -w ${MY_BUILD_DIR}
       COMMENT "Installing LAMMPS Python module")
   else()
     add_custom_target(

python/install.py

@@ -23,6 +23,8 @@ parser.add_argument("-l", "--lib", required=True,
                     help="path to the compiled LAMMPS shared library")
 parser.add_argument("-n", "--noinstall", action="store_true", default=False,
                     help="only build a binary wheel. Don't attempt to install it")
+parser.add_argument("-w", "--wheeldir", required=False,
+                    help="path to a directory where the created wheel will be stored")
 
 args = parser.parse_args()
 
@@ -30,7 +32,7 @@ args = parser.parse_args()
 
 if args.package:
   if not os.path.exists(args.package):
-    print( "ERROR: LAMMPS package %s does not exist" % args.package)
+    print("ERROR: LAMMPS package %s does not exist" % args.package)
     parser.print_help()
     sys.exit(1)
   else:
@@ -38,12 +40,20 @@ if args.package:
 
 if args.lib:
   if not os.path.exists(args.lib):
-    print( "ERROR: LAMMPS shared library %s does not exist" % args.lib)
+    print("ERROR: LAMMPS shared library %s does not exist" % args.lib)
     parser.print_help()
     sys.exit(1)
   else:
     args.lib = os.path.abspath(args.lib)
 
+if args.wheeldir:
+  if not os.path.exists(args.wheeldir):
+    print("ERROR: directory %s to store the wheel does not exist" % args.wheeldir)
+    parser.print_help()
+    sys.exit(1)
+  else:
+    args.wheeldir = os.path.abspath(args.wheeldir)
+
 # we need to switch to the folder of the python package
 olddir = os.path.abspath('.')
 os.chdir(os.path.dirname(args.package))
@@ -80,10 +90,18 @@ os.remove(os.path.join('lammps',os.path.basename(args.lib)))
 
 # stop here if we were asked not to install the wheel we created
 if args.noinstall:
-    exit(0)
+  if args.wheeldir:
+    for wheel in glob.glob('lammps-*.whl'):
+      shutil.copy(wheel, args.wheeldir)
+      os.remove(wheel)
+  exit(0)
 
 # install the wheel with pip. first try to install in the default environment.
 # that will be a virtual environment, if active, or the system folder.
+# if in a virtual environment, we must not use the python executable
+# that is running this script (configured by cmake), but use "python"
+# from the regular system path. The user may have changed to the virtual
+# environment *after* running cmake.
 # recent versions of pip will automatically drop to use the user folder
 # in case the system folder is not writable.
 
@@ -93,21 +111,35 @@ if args.noinstall:
 # must be uninstalled manually. We must not ignore this and drop
 # back to install into a (forced) user folder.
 
-print("Installing wheel")
+if "VIRTUAL_ENV" in os.environ:
+  print("Installing wheel into virtual environment")
+  py_exe = 'python'
+else:
+  print("Installing wheel into system site-packages folder")
+  py_exe = sys.executable
+
 for wheel in glob.glob('lammps-*.whl'):
   try:
-    txt = subprocess.check_output([sys.executable, '-m', 'pip', 'install', '--force-reinstall', wheel], stderr=subprocess.STDOUT, shell=False)
+    txt = subprocess.check_output([py_exe, '-m', 'pip', 'install', '--force-reinstall', wheel], stderr=subprocess.STDOUT, shell=False)
     print(txt.decode('UTF-8'))
+    if args.wheeldir:
+      shutil.copy(wheel, args.wheeldir)
+    else:
+      shutil.copy(wheel, olddir)
+    os.remove(wheel)
     continue
   except subprocess.CalledProcessError as err:
     errmsg = err.output.decode('UTF-8')
     if errmsg.find("distutils installed"):
       sys.exit(errmsg + "You need to uninstall the LAMMPS python module manually first.\n")
   try:
-    print('Installing wheel into standard site-packages folder failed. Trying user folder now')
+    print('Installing wheel into system site-packages folder failed. Trying user folder now')
     txt = subprocess.check_output([sys.executable, '-m', 'pip', 'install', '--user', '--force-reinstall', wheel], stderr=subprocess.STDOUT, shell=False)
     print(txt.decode('UTF-8'))
+    if args.wheeldir:
+      shutil.copy(wheel, args.wheeldir)
+    else:
+      shutil.copy(wheel, olddir)
+    os.remove(wheel)
   except:
     sys.exit('Failed to install wheel ' + wheel)
-  shutil.copy(wheel, olddir)
-  os.remove(wheel)

python/makewheel.py

@@ -1,14 +1,26 @@
 #!/usr/bin/env python
 
-import sys,os
+import sys,os,site
 
-# find python script to activate the virtual environment and source it
+base = os.path.abspath('buildwheel')
 if sys.platform == 'win32':
-  virtenv=os.path.join('buildwheel','Scripts','activate_this.py')
+  bin_dir=os.path.join(base,'Scripts')
 else:
-  virtenv=os.path.join('buildwheel','bin','activate_this.py')
+  bin_dir=os.path.join(base,'bin')
 
-exec(open(virtenv).read(), {'__file__': virtenv})
+# prepend bin to PATH, set venv path
+os.environ["PATH"] = os.pathsep.join([bin_dir] + os.environ.get("PATH", "").split(os.pathsep))
+os.environ["VIRTUAL_ENV"] = base
+
+# add the virtual environments libraries to the host python import mechanism
+prev_length = len(sys.path)
+for lib in "__LIB_FOLDERS__".split(os.pathsep):
+    path = os.path.realpath(os.path.join(bin_dir, lib))
+    site.addsitedir(path)
+sys.path[:] = sys.path[prev_length:] + sys.path[0:prev_length]
+
+sys.real_prefix = sys.prefix
+sys.prefix = base
 
 # update pip and install all requirements to build the wheel
 os.system('python -m pip install --upgrade pip')

src/Makefile

@@ -461,7 +461,7 @@ mpi-stubs:
 sinclude ../lib/python/Makefile.lammps
 install-python:
 	@rm -rf ../python/build
-	@$(PYTHON) ../python/install.py -p ../python/lammps -l ../src/liblammps.so
+	@$(PYTHON) ../python/install.py -p ../python/lammps -l ../src/liblammps.so -w $(PWD)
 
 # Create a tarball of src dir and packages