diff --git a/doc/src/Run_options.txt b/doc/src/Run_options.txt index 87b272a866..df1a57421a 100644 --- a/doc/src/Run_options.txt +++ b/doc/src/Run_options.txt @@ -126,9 +126,10 @@ are intended for computational work like running LAMMPS. By default Ng = 1 and Ns is not set. Depending on which flavor of MPI you are running, LAMMPS will look for -one of these 3 environment variables +one of these 4 environment variables SLURM_LOCALID (various MPI variants compiled with SLURM support) +MPT_LRANK (HPE MPI) MV2_COMM_WORLD_LOCAL_RANK (Mvapich) OMPI_COMM_WORLD_LOCAL_RANK (OpenMPI) :pre diff --git a/src/KOKKOS/kokkos.cpp b/src/KOKKOS/kokkos.cpp index 32550e8285..18dff991b2 100644 --- a/src/KOKKOS/kokkos.cpp +++ b/src/KOKKOS/kokkos.cpp @@ -113,23 +113,37 @@ KokkosLMP::KokkosLMP(LAMMPS *lmp, int narg, char **arg) : Pointers(lmp) } iarg += 2; + int set_flag = 0; char *str; if ((str = getenv("SLURM_LOCALID"))) { int local_rank = atoi(str); device = local_rank % ngpus; if (device >= skip_gpu) device++; + set_flag = 1; + } + if ((str = getenv("MPT_LRANK"))) { + int local_rank = atoi(str); + device = local_rank % ngpus; + if (device >= skip_gpu) device++; + set_flag = 1; } if ((str = getenv("MV2_COMM_WORLD_LOCAL_RANK"))) { int local_rank = atoi(str); device = local_rank % ngpus; if (device >= skip_gpu) device++; + set_flag = 1; } if ((str = getenv("OMPI_COMM_WORLD_LOCAL_RANK"))) { int local_rank = atoi(str); device = local_rank % ngpus; if (device >= skip_gpu) device++; + set_flag = 1; } + if (ngpus > 1 && !set_flag) + error->all(FLERR,"Could not determine local MPI rank for multiple " + "GPUs with Kokkos CUDA because MPI library not recognized"); + } else if (strcmp(arg[iarg],"t") == 0 || strcmp(arg[iarg],"threads") == 0) { nthreads = atoi(arg[iarg+1]); diff --git a/src/KOKKOS/kokkos.h b/src/KOKKOS/kokkos.h index b9f1e66c68..7b605bee1e 100644 --- a/src/KOKKOS/kokkos.h +++ b/src/KOKKOS/kokkos.h @@ -74,6 +74,11 @@ E: Invalid Kokkos command-line args Self-explanatory. See Section 2.7 of the manual for details. +E: Could not determine local MPI rank for multiple GPUs with Kokkos CUDA +because MPI library not recognized + +The local MPI rank was not found in one of four supported environment variables. + E: GPUs are requested but Kokkos has not been compiled for CUDA Recompile Kokkos with CUDA support to use GPUs.