From 5526cc2c53662cfb1fc63e0df8098285b1631a8c Mon Sep 17 00:00:00 2001
From: Stan Moore <stamoor@sandia.gov>
Date: Fri, 6 Nov 2020 12:01:45 -0700
Subject: [PATCH 01/11] Fix issue with Kokkos::finalize and library interface

---
 src/KOKKOS/kokkos.cpp    | 2 --
 src/accelerator_kokkos.h | 5 +++++
 src/error.cpp            | 7 ++++---
 src/library.cpp          | 6 ++++++
 src/library.h            | 1 +
 src/main.cpp             | 4 ++++
 6 files changed, 20 insertions(+), 5 deletions(-)

diff --git a/src/KOKKOS/kokkos.cpp b/src/KOKKOS/kokkos.cpp
index fe72981d50..c1b099829f 100644
--- a/src/KOKKOS/kokkos.cpp
+++ b/src/KOKKOS/kokkos.cpp
@@ -291,9 +291,7 @@ KokkosLMP::KokkosLMP(LAMMPS *lmp, int narg, char **arg) : Pointers(lmp)
 
 KokkosLMP::~KokkosLMP()
 {
-  // finalize Kokkos
 
-  Kokkos::finalize();
 }
 
 /* ----------------------------------------------------------------------
diff --git a/src/accelerator_kokkos.h b/src/accelerator_kokkos.h
index 9fb9cf3690..2f7b387755 100644
--- a/src/accelerator_kokkos.h
+++ b/src/accelerator_kokkos.h
@@ -123,5 +123,10 @@ class DAT {
 
 }
 
+namespace Kokkos {
+  static int is_initialized() {return false;}
+  static void finalize() {}
+}
+
 #endif
 #endif
diff --git a/src/error.cpp b/src/error.cpp
index 1ce9a0f3fe..9f3fb98022 100644
--- a/src/error.cpp
+++ b/src/error.cpp
@@ -75,7 +75,7 @@ void Error::universe_all(const std::string &file, int line, const std::string &s
 
   throw LAMMPSException(mesg);
 #else
-  if (lmp->kokkos) Kokkos::finalize();
+  if (Kokkos::is_initialized()) Kokkos::finalize();
   MPI_Finalize();
   exit(1);
 #endif
@@ -161,7 +161,7 @@ void Error::all(const std::string &file, int line, const std::string &str)
   if (logfile) fclose(logfile);
 
   if (universe->nworlds > 1) MPI_Abort(universe->uworld,1);
-  if (lmp->kokkos) Kokkos::finalize();
+  if (Kokkos::is_initialized()) Kokkos::finalize();
   MPI_Finalize();
   exit(1);
 #endif
@@ -200,6 +200,7 @@ void Error::one(const std::string &file, int line, const std::string &str)
 #else
   if (screen) fflush(screen);
   if (logfile) fflush(logfile);
+  if (Kokkos::is_initialized()) Kokkos::finalize();
   MPI_Abort(world,1);
   exit(1); // to trick "smart" compilers into believing this does not return
 #endif
@@ -246,7 +247,7 @@ void Error::done(int status)
   if (screen && screen != stdout) fclose(screen);
   if (logfile) fclose(logfile);
 
-  if (lmp->kokkos) Kokkos::finalize();
+  if (Kokkos::is_initialized()) Kokkos::finalize();
   MPI_Finalize();
   exit(status);
 }
diff --git a/src/library.cpp b/src/library.cpp
index 2fd1486bc2..fd21d7357c 100644
--- a/src/library.cpp
+++ b/src/library.cpp
@@ -18,6 +18,7 @@
 #include "library.h"
 #include <mpi.h>
 
+#include "accelerator_kokkos.h"
 #include "atom.h"
 #include "atom_vec.h"
 #include "comm.h"
@@ -348,6 +349,11 @@ void lammps_mpi_finalize()
   }
 }
 
+void lammps_kokkos_finalize()
+{
+  if (Kokkos::is_initialized()) Kokkos::finalize();
+}
+
 // ----------------------------------------------------------------------
 // Library functions to process commands
 // ----------------------------------------------------------------------
diff --git a/src/library.h b/src/library.h
index a38ea2dc82..243e6afa3e 100644
--- a/src/library.h
+++ b/src/library.h
@@ -96,6 +96,7 @@ void  lammps_close(void *handle);
 
 void  lammps_mpi_init();
 void  lammps_mpi_finalize();
+void  lammps_kokkos_finalize();
 
 /* ----------------------------------------------------------------------
  * Library functions to process commands
diff --git a/src/main.cpp b/src/main.cpp
index 582de6999c..fc5c84e3ec 100644
--- a/src/main.cpp
+++ b/src/main.cpp
@@ -13,6 +13,7 @@
 
 #include "lammps.h"
 #include "input.h"
+#include "accelerator_kokkos.h"
 
 #include <mpi.h>
 #include <cstdlib>
@@ -53,8 +54,10 @@ int main(int argc, char **argv)
     lammps->input->file();
     delete lammps;
   } catch(LAMMPSAbortException &ae) {
+    if (Kokkos::is_initialized()) Kokkos::finalize();
     MPI_Abort(ae.universe, 1);
   } catch(LAMMPSException &e) {
+    if (Kokkos::is_initialized()) Kokkos::finalize();
     MPI_Barrier(MPI_COMM_WORLD);
     MPI_Finalize();
     exit(1);
@@ -64,6 +67,7 @@ int main(int argc, char **argv)
   lammps->input->file();
   delete lammps;
 #endif
+  if (Kokkos::is_initialized()) Kokkos::finalize();
   MPI_Barrier(MPI_COMM_WORLD);
   MPI_Finalize();
 }

From 825facf6e09583a5e8068d277f0738d8aa31189c Mon Sep 17 00:00:00 2001
From: Stan Moore <stamoor@sandia.gov>
Date: Fri, 6 Nov 2020 12:33:31 -0700
Subject: [PATCH 02/11] Fix compile issue with accelerator_kokkos.h

---
 src/accelerator_kokkos.h | 6 +-----
 1 file changed, 1 insertion(+), 5 deletions(-)

diff --git a/src/accelerator_kokkos.h b/src/accelerator_kokkos.h
index 2f7b387755..d8a63bd426 100644
--- a/src/accelerator_kokkos.h
+++ b/src/accelerator_kokkos.h
@@ -63,6 +63,7 @@ class KokkosLMP {
 
 class Kokkos {
  public:
+  static int is_initialized() {return false;}
   static void finalize() {}
 };
 
@@ -123,10 +124,5 @@ class DAT {
 
 }
 
-namespace Kokkos {
-  static int is_initialized() {return false;}
-  static void finalize() {}
-}
-
 #endif
 #endif

From 2d5e8f050acf092e461a277f665e5c94d5e44a5a Mon Sep 17 00:00:00 2001
From: Stan Gerald Moore <stamoor@sandia.gov>
Date: Thu, 24 Jun 2021 13:25:35 -0600
Subject: [PATCH 03/11] Make Kokkos init/finalize calls safer

---
 src/KOKKOS/kokkos.cpp    | 21 ++++++++++++++++++++-
 src/KOKKOS/kokkos.h      |  2 ++
 src/accelerator_kokkos.h | 12 ++++++------
 src/error.cpp            |  9 +++++----
 src/library.cpp          |  2 +-
 src/main.cpp             |  8 +++++---
 6 files changed, 39 insertions(+), 15 deletions(-)

diff --git a/src/KOKKOS/kokkos.cpp b/src/KOKKOS/kokkos.cpp
index 14a689e72a..53fda7684c 100644
--- a/src/KOKKOS/kokkos.cpp
+++ b/src/KOKKOS/kokkos.cpp
@@ -189,7 +189,7 @@ KokkosLMP::KokkosLMP(LAMMPS *lmp, int narg, char **arg) : Pointers(lmp)
   args.num_numa = numa;
   args.device_id = device;
 
-  Kokkos::initialize(args);
+  initialize(args);
 
   // default settings for package kokkos command
 
@@ -302,6 +302,25 @@ KokkosLMP::~KokkosLMP()
 
 }
 
+/* ---------------------------------------------------------------------- */
+
+void KokkosLMP::initialize(Kokkos::InitArguments args)
+{
+  if (!Kokkos::is_initialized())
+    Kokkos::initialize(args);
+}
+
+/* ---------------------------------------------------------------------- */
+
+void KokkosLMP::finalize()
+{
+  static int is_finalized = 0;
+
+  if (Kokkos::is_initialized() && !is_finalized)
+    Kokkos::finalize();
+  is_finalized = 1;
+}
+
 /* ----------------------------------------------------------------------
    invoked by package kokkos command
 ------------------------------------------------------------------------- */
diff --git a/src/KOKKOS/kokkos.h b/src/KOKKOS/kokkos.h
index 22060ecc09..7782e4e00c 100644
--- a/src/KOKKOS/kokkos.h
+++ b/src/KOKKOS/kokkos.h
@@ -51,6 +51,8 @@ class KokkosLMP : protected Pointers {
 
   KokkosLMP(class LAMMPS *, int, char **);
   ~KokkosLMP();
+  static void initialize(Kokkos::InitArguments);
+  static void finalize();
   void accelerator(int, char **);
   int neigh_count(int);
 
diff --git a/src/accelerator_kokkos.h b/src/accelerator_kokkos.h
index 2ee15e946a..118ac3b9c8 100644
--- a/src/accelerator_kokkos.h
+++ b/src/accelerator_kokkos.h
@@ -43,6 +43,10 @@
 #include "modify.h"
 #include "neighbor.h"
 
+namespace Kokkos {
+  typedef int InitArguements;
+};
+
 #define LAMMPS_INLINE inline
 
 namespace LAMMPS_NS {
@@ -56,17 +60,13 @@ class KokkosLMP {
 
   KokkosLMP(class LAMMPS *, int, char **) { kokkos_exists = 0; }
   ~KokkosLMP() {}
+  static void initialize(Kokkos::InitArguements args) {}
+  static void finalize() {}
   void accelerator(int, char **) {}
   int neigh_list_kokkos(int) { return 0; }
   int neigh_count(int) { return 0; }
 };
 
-class Kokkos {
- public:
-  static int is_initialized() {return false;}
-  static void finalize() {}
-};
-
 class AtomKokkos : public Atom {
  public:
   tagint **k_special;
diff --git a/src/error.cpp b/src/error.cpp
index fd12cbe80e..9811a1d3eb 100644
--- a/src/error.cpp
+++ b/src/error.cpp
@@ -81,7 +81,7 @@ void Error::universe_all(const std::string &file, int line, const std::string &s
 
   throw LAMMPSException(mesg);
 #else
-  if (Kokkos::is_initialized()) Kokkos::finalize();
+  KokkosLMP::finalize();
   MPI_Finalize();
   exit(1);
 #endif
@@ -107,6 +107,7 @@ void Error::universe_one(const std::string &file, int line, const std::string &s
 
   throw LAMMPSAbortException(mesg, universe->uworld);
 #else
+  KokkosLMP::finalize();
   MPI_Abort(universe->uworld,1);
   exit(1); // to trick "smart" compilers into believing this does not return
 #endif
@@ -173,8 +174,8 @@ void Error::all(const std::string &file, int line, const std::string &str)
   if (screen && screen != stdout) fclose(screen);
   if (logfile) fclose(logfile);
 
+  KokkosLMP::finalize();
   if (universe->nworlds > 1) MPI_Abort(universe->uworld,1);
-  if (Kokkos::is_initialized()) Kokkos::finalize();
   MPI_Finalize();
   exit(1);
 #endif
@@ -213,7 +214,7 @@ void Error::one(const std::string &file, int line, const std::string &str)
 #else
   if (screen) fflush(screen);
   if (logfile) fflush(logfile);
-  if (Kokkos::is_initialized()) Kokkos::finalize();
+  KokkosLMP::finalize();
   MPI_Abort(world,1);
   exit(1); // to trick "smart" compilers into believing this does not return
 #endif
@@ -316,7 +317,7 @@ void Error::done(int status)
   if (screen && screen != stdout) fclose(screen);
   if (logfile) fclose(logfile);
 
-  if (Kokkos::is_initialized()) Kokkos::finalize();
+  KokkosLMP::finalize();
   MPI_Finalize();
   exit(status);
 }
diff --git a/src/library.cpp b/src/library.cpp
index 4fcf382247..178364d777 100644
--- a/src/library.cpp
+++ b/src/library.cpp
@@ -356,7 +356,7 @@ void lammps_mpi_finalize()
 
 void lammps_kokkos_finalize()
 {
-  if (Kokkos::is_initialized()) Kokkos::finalize();
+  KokkosLMP::finalize();
 }
 
 // ----------------------------------------------------------------------
diff --git a/src/main.cpp b/src/main.cpp
index ff1b3f9a09..76ae4fde09 100644
--- a/src/main.cpp
+++ b/src/main.cpp
@@ -78,15 +78,16 @@ int main(int argc, char **argv)
     lammps->input->file();
     delete lammps;
   } catch (LAMMPSAbortException &ae) {
-    if (Kokkos::is_initialized()) Kokkos::finalize();
+    KokkosLMP::finalize();
     MPI_Abort(ae.universe, 1);
   } catch (LAMMPSException &e) {
-    if (Kokkos::is_initialized()) Kokkos::finalize();
+    KokkosLMP::finalize();
     MPI_Barrier(lammps_comm);
     MPI_Finalize();
     exit(1);
   } catch (fmt::format_error &fe) {
     fprintf(stderr, "fmt::format_error: %s\n", fe.what());
+    KokkosLMP::finalize();
     MPI_Abort(MPI_COMM_WORLD, 1);
     exit(1);
   }
@@ -97,11 +98,12 @@ int main(int argc, char **argv)
     delete lammps;
   } catch (fmt::format_error &fe) {
     fprintf(stderr, "fmt::format_error: %s\n", fe.what());
+    KokkosLMP::finalize();
     MPI_Abort(MPI_COMM_WORLD, 1);
     exit(1);
   }
 #endif
-  if (Kokkos::is_initialized()) Kokkos::finalize();
+  KokkosLMP::finalize();
   MPI_Barrier(lammps_comm);
   MPI_Finalize();
 }

From 30ac0107c866d4124282a39c220b738eb90213e7 Mon Sep 17 00:00:00 2001
From: Stan Gerald Moore <stamoor@sandia.gov>
Date: Fri, 25 Jun 2021 10:32:54 -0600
Subject: [PATCH 04/11] Remove unused function

---
 src/accelerator_kokkos.h | 5 -----
 1 file changed, 5 deletions(-)

diff --git a/src/accelerator_kokkos.h b/src/accelerator_kokkos.h
index 118ac3b9c8..0cea855f1e 100644
--- a/src/accelerator_kokkos.h
+++ b/src/accelerator_kokkos.h
@@ -43,10 +43,6 @@
 #include "modify.h"
 #include "neighbor.h"
 
-namespace Kokkos {
-  typedef int InitArguements;
-};
-
 #define LAMMPS_INLINE inline
 
 namespace LAMMPS_NS {
@@ -60,7 +56,6 @@ class KokkosLMP {
 
   KokkosLMP(class LAMMPS *, int, char **) { kokkos_exists = 0; }
   ~KokkosLMP() {}
-  static void initialize(Kokkos::InitArguements args) {}
   static void finalize() {}
   void accelerator(int, char **) {}
   int neigh_list_kokkos(int) { return 0; }

From a74a718b6f25ac6819196263885b3ec73871d345 Mon Sep 17 00:00:00 2001
From: Stan Gerald Moore <stamoor@sandia.gov>
Date: Fri, 25 Jun 2021 10:47:55 -0600
Subject: [PATCH 05/11] Add additional error check

---
 src/KOKKOS/kokkos.cpp | 7 ++++---
 src/KOKKOS/kokkos.h   | 2 ++
 2 files changed, 6 insertions(+), 3 deletions(-)

diff --git a/src/KOKKOS/kokkos.cpp b/src/KOKKOS/kokkos.cpp
index 53fda7684c..7c53ce0388 100644
--- a/src/KOKKOS/kokkos.cpp
+++ b/src/KOKKOS/kokkos.cpp
@@ -306,16 +306,17 @@ KokkosLMP::~KokkosLMP()
 
 void KokkosLMP::initialize(Kokkos::InitArguments args)
 {
-  if (!Kokkos::is_initialized())
+  if (!Kokkos::is_initialized()) {
+    if (is_finalized)
+      error->all(FLERR,"Kokkos package already finalized, cannot re-initialize");
     Kokkos::initialize(args);
+  }
 }
 
 /* ---------------------------------------------------------------------- */
 
 void KokkosLMP::finalize()
 {
-  static int is_finalized = 0;
-
   if (Kokkos::is_initialized() && !is_finalized)
     Kokkos::finalize();
   is_finalized = 1;
diff --git a/src/KOKKOS/kokkos.h b/src/KOKKOS/kokkos.h
index 7782e4e00c..7d05512bd7 100644
--- a/src/KOKKOS/kokkos.h
+++ b/src/KOKKOS/kokkos.h
@@ -49,6 +49,8 @@ class KokkosLMP : protected Pointers {
   int newtonflag;
   double binsize;
 
+  static int is_finalized;
+
   KokkosLMP(class LAMMPS *, int, char **);
   ~KokkosLMP();
   static void initialize(Kokkos::InitArguments);

From ef54c7290e40864d486a654fc0e5efa675db0156 Mon Sep 17 00:00:00 2001
From: Axel Kohlmeyer <akohlmey@gmail.com>
Date: Sun, 27 Jun 2021 15:40:38 -0400
Subject: [PATCH 06/11] must pass pointer to Error class as argument since
 static functions don't have access to "this"

---
 src/KOKKOS/kokkos.cpp | 4 ++--
 src/KOKKOS/kokkos.h   | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/KOKKOS/kokkos.cpp b/src/KOKKOS/kokkos.cpp
index 7c53ce0388..9f0cb14683 100644
--- a/src/KOKKOS/kokkos.cpp
+++ b/src/KOKKOS/kokkos.cpp
@@ -189,7 +189,7 @@ KokkosLMP::KokkosLMP(LAMMPS *lmp, int narg, char **arg) : Pointers(lmp)
   args.num_numa = numa;
   args.device_id = device;
 
-  initialize(args);
+  KokkosLMP::initialize(args,error);
 
   // default settings for package kokkos command
 
@@ -304,7 +304,7 @@ KokkosLMP::~KokkosLMP()
 
 /* ---------------------------------------------------------------------- */
 
-void KokkosLMP::initialize(Kokkos::InitArguments args)
+void KokkosLMP::initialize(Kokkos::InitArguments args, Error *error)
 {
   if (!Kokkos::is_initialized()) {
     if (is_finalized)
diff --git a/src/KOKKOS/kokkos.h b/src/KOKKOS/kokkos.h
index 7d05512bd7..a62666ba2a 100644
--- a/src/KOKKOS/kokkos.h
+++ b/src/KOKKOS/kokkos.h
@@ -53,7 +53,7 @@ class KokkosLMP : protected Pointers {
 
   KokkosLMP(class LAMMPS *, int, char **);
   ~KokkosLMP();
-  static void initialize(Kokkos::InitArguments);
+  static void initialize(Kokkos::InitArguments, Error *);
   static void finalize();
   void accelerator(int, char **);
   int neigh_count(int);

From 8ec3d90f5c57e9a49df632a1ba80dc2b640b8c3f Mon Sep 17 00:00:00 2001
From: Axel Kohlmeyer <akohlmey@gmail.com>
Date: Sun, 27 Jun 2021 15:41:59 -0400
Subject: [PATCH 07/11] simplify

---
 src/KOKKOS/kokkos.cpp | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/src/KOKKOS/kokkos.cpp b/src/KOKKOS/kokkos.cpp
index 9f0cb14683..ffef36cd68 100644
--- a/src/KOKKOS/kokkos.cpp
+++ b/src/KOKKOS/kokkos.cpp
@@ -167,10 +167,8 @@ KokkosLMP::KokkosLMP(LAMMPS *lmp, int narg, char **arg) : Pointers(lmp)
 
   // initialize Kokkos
 
-  if (me == 0) {
-    if (screen) fprintf(screen,"  will use up to %d GPU(s) per node\n",ngpus);
-    if (logfile) fprintf(logfile,"  will use up to %d GPU(s) per node\n",ngpus);
-  }
+  if (me == 0)
+    utils::logmesg(lmp, "  will use up to {} GPU(s) per node\n",ngpus);
 
 #ifdef LMP_KOKKOS_GPU
   if (ngpus <= 0)

From 2ef47fce69e8a4624fce87d28763afb4a9c8e27f Mon Sep 17 00:00:00 2001
From: Axel Kohlmeyer <akohlmey@gmail.com>
Date: Sun, 27 Jun 2021 16:08:22 -0400
Subject: [PATCH 08/11] add missing initializer for KokkosLMP::is_finalized

---
 src/KOKKOS/kokkos.cpp | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/src/KOKKOS/kokkos.cpp b/src/KOKKOS/kokkos.cpp
index ffef36cd68..2a83b40bba 100644
--- a/src/KOKKOS/kokkos.cpp
+++ b/src/KOKKOS/kokkos.cpp
@@ -69,6 +69,8 @@ GPU_AWARE_UNKNOWN
 
 using namespace LAMMPS_NS;
 
+int KokkosLMP::is_finalized = 0;
+
 /* ---------------------------------------------------------------------- */
 
 KokkosLMP::KokkosLMP(LAMMPS *lmp, int narg, char **arg) : Pointers(lmp)

From 9a740a4a60c3e769abcbfa6d4f4d214e12bb9c46 Mon Sep 17 00:00:00 2001
From: Axel Kohlmeyer <akohlmey@gmail.com>
Date: Sun, 27 Jun 2021 17:07:42 -0400
Subject: [PATCH 09/11] add lammps_kokos_finalize() support to various LAMMPS
 wrappers and document

---
 doc/src/Library_create.rst |  1 +
 doc/src/Python_create.rst  |  5 ++++-
 fortran/lammps.f90         | 15 +++++++--------
 python/lammps/core.py      | 10 ++++++++--
 src/library.cpp            | 21 +++++++++++++++++++--
 tools/swig/lammps.i        |  2 ++
 6 files changed, 41 insertions(+), 13 deletions(-)

diff --git a/doc/src/Library_create.rst b/doc/src/Library_create.rst
index 350569e54e..b3fea4b89e 100644
--- a/doc/src/Library_create.rst
+++ b/doc/src/Library_create.rst
@@ -9,6 +9,7 @@ This section documents the following functions:
 - :cpp:func:`lammps_close`
 - :cpp:func:`lammps_mpi_init`
 - :cpp:func:`lammps_mpi_finalize`
+- :cpp:func:`lammps_kokkos_finalize`
 
 --------------------
 
diff --git a/doc/src/Python_create.rst b/doc/src/Python_create.rst
index 00b1c08814..ec4241f36a 100644
--- a/doc/src/Python_create.rst
+++ b/doc/src/Python_create.rst
@@ -134,7 +134,10 @@ compiled with.
 The :py:func:`lmp.close() <lammps.lammps.close()>` call is
 optional since the LAMMPS class instance will also be deleted
 automatically during the :py:class:`lammps <lammps.lammps>` class
-destructor.
+destructor.  Instead of :py:func:`lmp.close() <lammps.lammps.close()>`
+it is also possible to call :py:func:`lmp.finalize() <lammps.lammps.finalize()>`;
+this will destruct the LAMMPS instance, but also finalized and release
+the MPI and/or Kokkos environment if enabled and active.
 
 Note that you can create multiple LAMMPS objects in your Python
 script, and coordinate and run multiple simulations, e.g.
diff --git a/fortran/lammps.f90 b/fortran/lammps.f90
index 21909e1288..2e78cdd00b 100644
--- a/fortran/lammps.f90
+++ b/fortran/lammps.f90
@@ -76,17 +76,15 @@ MODULE LIBLAMMPS
         TYPE(c_ptr), VALUE :: handle
       END SUBROUTINE lammps_close
 
-      SUBROUTINE lammps_mpi_init(handle) BIND(C, name='lammps_mpi_init')
-        IMPORT :: c_ptr
-        TYPE(c_ptr), VALUE :: handle
+      SUBROUTINE lammps_mpi_init() BIND(C, name='lammps_mpi_init')
       END SUBROUTINE lammps_mpi_init
 
-      SUBROUTINE lammps_mpi_finalize(handle) &
-          BIND(C, name='lammps_mpi_finalize')
-        IMPORT :: c_ptr
-        TYPE(c_ptr), VALUE :: handle
+      SUBROUTINE lammps_mpi_finalize() BIND(C, name='lammps_mpi_finalize')
       END SUBROUTINE lammps_mpi_finalize
 
+      SUBROUTINE lammps_kokkos_finalize() BIND(C, name='lammps_kokkos_finalize')
+      END SUBROUTINE lammps_kokkos_finalize
+
       SUBROUTINE lammps_file(handle,filename) BIND(C, name='lammps_file')
         IMPORT :: c_ptr
         TYPE(c_ptr), VALUE :: handle
@@ -188,7 +186,8 @@ CONTAINS
 
     IF (PRESENT(finalize)) THEN
         IF (finalize) THEN
-            CALL lammps_mpi_finalize(self%handle)
+            CALL lammps_kokkos_finalize()
+            CALL lammps_mpi_finalize()
         END IF
     END IF
   END SUBROUTINE lmp_close
diff --git a/python/lammps/core.py b/python/lammps/core.py
index 88eba735b0..c7b51a6b54 100644
--- a/python/lammps/core.py
+++ b/python/lammps/core.py
@@ -460,10 +460,16 @@ class lammps(object):
   # -------------------------------------------------------------------------
 
   def finalize(self):
-    """Shut down the MPI communication through the library interface by calling :cpp:func:`lammps_finalize`.
+    """Shut down the MPI communication and Kokkos environment (if active) through the
+       library interface by  calling :cpp:func:`lammps_mpi_finalize` and
+       :cpp:func:`lammps_kokkos_finalize`.
+
+       You cannot create or use any LAMMPS instances after this function is called
+       unless LAMMPS was compiled without MPI and without Kokkos support.
     """
     self.close()
-    self.lib.lammps_finalize()
+    self.lib.lammps_kokkos_finalize()
+    self.lib.lammps_mpi_finalize()
 
   # -------------------------------------------------------------------------
 
diff --git a/src/library.cpp b/src/library.cpp
index 178364d777..111430ced0 100644
--- a/src/library.cpp
+++ b/src/library.cpp
@@ -334,8 +334,8 @@ The MPI standard requires that any MPI application calls
 do any MPI calls, MPI is still initialized internally to avoid errors
 accessing any MPI functions.  This function should then be called right
 before exiting the program to wait until all (parallel) tasks are
-completed and then MPI is cleanly shut down.  After this function no
-more MPI calls may be made.
+completed and then MPI is cleanly shut down.  After calling this
+function no more MPI calls may be made.
 
 .. versionadded:: 18Sep2020
 
@@ -354,6 +354,23 @@ void lammps_mpi_finalize()
   }
 }
 
+/* ---------------------------------------------------------------------- */
+
+/** Shut down the Kokkos library environment.
+ *
+\verbatim embed:rst
+
+The Kokkos library may only be initialized once during the execution of
+a process.  This is done automatically the first time Kokkos
+functionality is used.  This requires that the Kokkos environment
+must be explicitly shut down after any LAMMPS instance using it is
+closed (to release associated resources).
+After calling this function no Kokkos functionality may be used.
+
+.. versionadded:: TBD
+
+\endverbatim */
+
 void lammps_kokkos_finalize()
 {
   KokkosLMP::finalize();
diff --git a/tools/swig/lammps.i b/tools/swig/lammps.i
index ec37120c07..56547dda53 100644
--- a/tools/swig/lammps.i
+++ b/tools/swig/lammps.i
@@ -63,6 +63,7 @@ extern void  *lammps_open_fortran(int argc, char **argv, int f_comm);
 extern void   lammps_close(void *handle);
 extern void   lammps_mpi_init();
 extern void   lammps_mpi_finalize();
+extern void   lammps_kokkos_finalize();
 extern void   lammps_file(void *handle, const char *file);
 extern char  *lammps_command(void *handle, const char *cmd);
 extern void   lammps_commands_list(void *handle, int ncmd, const char **cmds);
@@ -185,6 +186,7 @@ extern void  *lammps_open_fortran(int argc, char **argv, int f_comm);
 extern void   lammps_close(void *handle);
 extern void   lammps_mpi_init();
 extern void   lammps_mpi_finalize();
+extern void   lammps_kokkos_finalize();
 extern void   lammps_file(void *handle, const char *file);
 extern char  *lammps_command(void *handle, const char *cmd);
 extern void   lammps_commands_list(void *handle, int ncmd, const char **cmds);

From d7a24dee91ba9a9dc05ec49f83d9e80e921df4bf Mon Sep 17 00:00:00 2001
From: Stan Gerald Moore <stamoor@sandia.gov>
Date: Wed, 30 Jun 2021 12:08:33 -0600
Subject: [PATCH 10/11] Add warning

---
 src/KOKKOS/kokkos.cpp | 13 ++++++++++++-
 src/KOKKOS/kokkos.h   | 13 +++++++++++--
 2 files changed, 23 insertions(+), 3 deletions(-)

diff --git a/src/KOKKOS/kokkos.cpp b/src/KOKKOS/kokkos.cpp
index 2a83b40bba..49bb6369d1 100644
--- a/src/KOKKOS/kokkos.cpp
+++ b/src/KOKKOS/kokkos.cpp
@@ -69,6 +69,7 @@ GPU_AWARE_UNKNOWN
 
 using namespace LAMMPS_NS;
 
+Kokkos::InitArguments KokkosLMP::args{-1, -1, -1, false};
 int KokkosLMP::is_finalized = 0;
 
 /* ---------------------------------------------------------------------- */
@@ -157,6 +158,10 @@ KokkosLMP::KokkosLMP(LAMMPS *lmp, int narg, char **arg) : Pointers(lmp)
     } else if (strcmp(arg[iarg],"t") == 0 ||
                strcmp(arg[iarg],"threads") == 0) {
       nthreads = atoi(arg[iarg+1]);
+
+      if (nthreads <= 0)
+        error->all(FLERR,"Invalid number of threads requested for Kokkos: must be 1 or greater");
+
       iarg += 2;
 
     } else if (strcmp(arg[iarg],"n") == 0 ||
@@ -184,7 +189,13 @@ KokkosLMP::KokkosLMP(LAMMPS *lmp, int narg, char **arg) : Pointers(lmp)
                          "than the OpenMP backend");
 #endif
 
-  Kokkos::InitArguments args;
+  // cannot change Kokkos library parameters after first initalization
+
+  if (args.num_threads != -1)
+    if (args.num_threads != nthreads || args.num_numa != numa || args.device_id != device)
+      if (me == 0)
+        error->warning(FLERR,"Kokkos package already initalized, cannot reinitialize with different parameters");
+
   args.num_threads = nthreads;
   args.num_numa = numa;
   args.device_id = device;
diff --git a/src/KOKKOS/kokkos.h b/src/KOKKOS/kokkos.h
index a62666ba2a..fbea17388d 100644
--- a/src/KOKKOS/kokkos.h
+++ b/src/KOKKOS/kokkos.h
@@ -50,6 +50,7 @@ class KokkosLMP : protected Pointers {
   double binsize;
 
   static int is_finalized;
+  static Kokkos::InitArguments args;
 
   KokkosLMP(class LAMMPS *, int, char **);
   ~KokkosLMP();
@@ -88,13 +89,21 @@ because MPI library not recognized
 
 The local MPI rank was not found in one of four supported environment variables.
 
+E: Invalid number of threads requested for Kokkos: must be 1 or greater
+
+Self-explanatory.
+
 E: GPUs are requested but Kokkos has not been compiled for CUDA
 
 Recompile Kokkos with CUDA support to use GPUs.
 
-E: Kokkos has been compiled for CUDA but no GPUs are requested
+E: Kokkos has been compiled for CUDA, HIP, or SYCL but no GPUs are requested
 
-One or more GPUs must be used when Kokkos is compiled for CUDA.
+One or more GPUs must be used when Kokkos is compiled for CUDA/HIP/SYCL.
+
+W: Kokkos package already initalized, cannot reinitialize with different parameters
+
+Self-explanatory.
 
 E: Illegal ... command
 

From b1d16508feac4140aa93857ac0fb4473f029b180 Mon Sep 17 00:00:00 2001
From: Axel Kohlmeyer <akohlmey@gmail.com>
Date: Wed, 30 Jun 2021 17:40:42 -0400
Subject: [PATCH 11/11] small tweak to recover first initialization settings
 for consistent output

---
 src/KOKKOS/kokkos.cpp                  | 30 +++++++++++++++-----------
 src/KOKKOS/kokkos.h                    |  1 +
 unittest/python/python-capabilities.py |  7 ++++++
 3 files changed, 26 insertions(+), 12 deletions(-)

diff --git a/src/KOKKOS/kokkos.cpp b/src/KOKKOS/kokkos.cpp
index 49bb6369d1..2588bf5d49 100644
--- a/src/KOKKOS/kokkos.cpp
+++ b/src/KOKKOS/kokkos.cpp
@@ -71,6 +71,7 @@ using namespace LAMMPS_NS;
 
 Kokkos::InitArguments KokkosLMP::args{-1, -1, -1, false};
 int KokkosLMP::is_finalized = 0;
+int KokkosLMP::init_ngpus = 0;
 
 /* ---------------------------------------------------------------------- */
 
@@ -172,7 +173,23 @@ KokkosLMP::KokkosLMP(LAMMPS *lmp, int narg, char **arg) : Pointers(lmp)
     } else error->all(FLERR,"Invalid Kokkos command-line args");
   }
 
-  // initialize Kokkos
+  // Initialize Kokkos. However, we cannot change any
+  // Kokkos library parameters after the first initalization
+
+  if (args.num_threads != -1) {
+    if (args.num_threads != nthreads || args.num_numa != numa || args.device_id != device)
+      if (me == 0)
+        error->warning(FLERR,"Kokkos package already initalized, cannot reinitialize with different parameters");
+    nthreads = args.num_threads;
+    numa = args.num_numa;
+    device = args.device_id;
+    ngpus = init_ngpus;
+  } else {
+    args.num_threads = nthreads;
+    args.num_numa = numa;
+    args.device_id = device;
+    init_ngpus = ngpus;
+  }
 
   if (me == 0)
     utils::logmesg(lmp, "  will use up to {} GPU(s) per node\n",ngpus);
@@ -189,17 +206,6 @@ KokkosLMP::KokkosLMP(LAMMPS *lmp, int narg, char **arg) : Pointers(lmp)
                          "than the OpenMP backend");
 #endif
 
-  // cannot change Kokkos library parameters after first initalization
-
-  if (args.num_threads != -1)
-    if (args.num_threads != nthreads || args.num_numa != numa || args.device_id != device)
-      if (me == 0)
-        error->warning(FLERR,"Kokkos package already initalized, cannot reinitialize with different parameters");
-
-  args.num_threads = nthreads;
-  args.num_numa = numa;
-  args.device_id = device;
-
   KokkosLMP::initialize(args,error);
 
   // default settings for package kokkos command
diff --git a/src/KOKKOS/kokkos.h b/src/KOKKOS/kokkos.h
index fbea17388d..65990544ad 100644
--- a/src/KOKKOS/kokkos.h
+++ b/src/KOKKOS/kokkos.h
@@ -51,6 +51,7 @@ class KokkosLMP : protected Pointers {
 
   static int is_finalized;
   static Kokkos::InitArguments args;
+  static int init_ngpus;
 
   KokkosLMP(class LAMMPS *, int, char **);
   ~KokkosLMP();
diff --git a/unittest/python/python-capabilities.py b/unittest/python/python-capabilities.py
index 2ace093a7e..8a3fbbd665 100644
--- a/unittest/python/python-capabilities.py
+++ b/unittest/python/python-capabilities.py
@@ -165,6 +165,13 @@ class PythonCapabilities(unittest.TestCase):
             if self.cmake_cache['GPU_PREC'].lower() == 'single':
                  self.assertIn('single',settings['GPU']['precision'])
 
+        if self.cmake_cache['PKG_KOKKOS']:
+            if self.cmake_cache['Kokkos_ENABLE_OPENMP']:
+                self.assertIn('openmp',settings['KOKKOS']['api'])
+            if self.cmake_cache['Kokkos_ENABLE_SERIAL']:
+                self.assertIn('serial',settings['KOKKOS']['api'])
+            self.assertIn('double',settings['KOKKOS']['precision'])
+
     def test_gpu_device(self):
 
         info = self.lmp.get_gpu_device_info()