From 21b2bf0253c359c9024ac6e33c0ff50a8ddeebda Mon Sep 17 00:00:00 2001 From: Axel Kohlmeyer Date: Mon, 6 Mar 2023 22:17:12 -0500 Subject: [PATCH 1/5] avoid including accelerator_kokkos.h in main.cpp --- cmake/Modules/Packages/KOKKOS.cmake | 3 --- src/KOKKOS/kokkos.cpp | 9 +++++++++ src/accelerator_kokkos.h | 5 +++++ src/main.cpp | 15 +++++++++------ 4 files changed, 23 insertions(+), 9 deletions(-) diff --git a/cmake/Modules/Packages/KOKKOS.cmake b/cmake/Modules/Packages/KOKKOS.cmake index b4eb227d61..00486e73db 100644 --- a/cmake/Modules/Packages/KOKKOS.cmake +++ b/cmake/Modules/Packages/KOKKOS.cmake @@ -72,13 +72,11 @@ if(DOWNLOAD_KOKKOS) set_target_properties(LAMMPS::KOKKOSCONTAINERS PROPERTIES IMPORTED_LOCATION "${INSTALL_DIR}/lib/libkokkoscontainers.a") target_link_libraries(lammps PRIVATE LAMMPS::KOKKOSCORE LAMMPS::KOKKOSCONTAINERS) - target_link_libraries(lmp PRIVATE LAMMPS::KOKKOSCORE LAMMPS::KOKKOSCONTAINERS) add_dependencies(LAMMPS::KOKKOSCORE kokkos_build) add_dependencies(LAMMPS::KOKKOSCONTAINERS kokkos_build) elseif(EXTERNAL_KOKKOS) find_package(Kokkos 3.7.01 REQUIRED CONFIG) target_link_libraries(lammps PRIVATE Kokkos::kokkos) - target_link_libraries(lmp PRIVATE Kokkos::kokkos) else() set(LAMMPS_LIB_KOKKOS_SRC_DIR ${LAMMPS_LIB_SOURCE_DIR}/kokkos) set(LAMMPS_LIB_KOKKOS_BIN_DIR ${LAMMPS_LIB_BINARY_DIR}/kokkos) @@ -98,7 +96,6 @@ else() ${LAMMPS_LIB_KOKKOS_BIN_DIR}) target_include_directories(lammps PRIVATE ${Kokkos_INCLUDE_DIRS}) target_link_libraries(lammps PRIVATE kokkos) - target_link_libraries(lmp PRIVATE kokkos) if(BUILD_SHARED_LIBS_WAS_ON) set(BUILD_SHARED_LIBS ON) endif() diff --git a/src/KOKKOS/kokkos.cpp b/src/KOKKOS/kokkos.cpp index 8b45c786e5..4e96e3218c 100644 --- a/src/KOKKOS/kokkos.cpp +++ b/src/KOKKOS/kokkos.cpp @@ -33,6 +33,10 @@ #include // for getpid() #endif +namespace LAMMPS_NS { + void kokkos_lmp_finalize(); +} + #ifdef LMP_KOKKOS_GPU // for detecting GPU-aware MPI support: @@ -80,6 +84,11 @@ using namespace LAMMPS_NS; int KokkosLMP::is_finalized = 0; int KokkosLMP::init_ngpus = 0; +void LAMMPS_NS::kokkos_lmp_finalize() +{ + KokkosLMP::finalize(); +} + /* ---------------------------------------------------------------------- */ KokkosLMP::KokkosLMP(LAMMPS *lmp, int narg, char **arg) : Pointers(lmp) diff --git a/src/accelerator_kokkos.h b/src/accelerator_kokkos.h index 36a376bff8..37d86a89e3 100644 --- a/src/accelerator_kokkos.h +++ b/src/accelerator_kokkos.h @@ -61,6 +61,11 @@ class KokkosLMP { int neigh_count(int) { return 0; } }; +void kokkos_lmp_finalize() +{ + KokkosLMP::finalize(); +} + class AtomKokkos : public Atom { public: tagint **k_special; diff --git a/src/main.cpp b/src/main.cpp index e263938e57..606f31274d 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -13,7 +13,6 @@ #include "lammps.h" -#include "accelerator_kokkos.h" #include "input.h" #include "lmppython.h" @@ -33,6 +32,10 @@ #include #endif +namespace LAMMPS_NS { +extern void kokkos_lmp_finalize(); +} + using namespace LAMMPS_NS; /* ---------------------------------------------------------------------- @@ -76,18 +79,18 @@ int main(int argc, char **argv) lammps->input->file(); delete lammps; } catch (LAMMPSAbortException &ae) { - KokkosLMP::finalize(); + kokkos_lmp_finalize(); Python::finalize(); MPI_Abort(ae.universe, 1); } catch (LAMMPSException &) { - KokkosLMP::finalize(); + kokkos_lmp_finalize(); Python::finalize(); MPI_Barrier(lammps_comm); MPI_Finalize(); exit(1); } catch (fmt::format_error &fe) { fprintf(stderr, "fmt::format_error: %s\n", fe.what()); - KokkosLMP::finalize(); + kokkos_lmp_finalize(); Python::finalize(); MPI_Abort(MPI_COMM_WORLD, 1); exit(1); @@ -99,13 +102,13 @@ int main(int argc, char **argv) delete lammps; } catch (fmt::format_error &fe) { fprintf(stderr, "fmt::format_error: %s\n", fe.what()); - KokkosLMP::finalize(); + kokkos_lmp_finalize(); Python::finalize(); MPI_Abort(MPI_COMM_WORLD, 1); exit(1); } #endif - KokkosLMP::finalize(); + kokkos_lmp_finalize(); Python::finalize(); MPI_Barrier(lammps_comm); MPI_Finalize(); From 40790f6c4593e9dbdf8d785dea7ec7930825caeb Mon Sep 17 00:00:00 2001 From: Axel Kohlmeyer Date: Mon, 6 Mar 2023 22:49:20 -0500 Subject: [PATCH 2/5] fix up non-KOKKOS case --- src/accelerator_kokkos.h | 5 ----- src/main.cpp | 10 ++++++++++ 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/src/accelerator_kokkos.h b/src/accelerator_kokkos.h index 37d86a89e3..36a376bff8 100644 --- a/src/accelerator_kokkos.h +++ b/src/accelerator_kokkos.h @@ -61,11 +61,6 @@ class KokkosLMP { int neigh_count(int) { return 0; } }; -void kokkos_lmp_finalize() -{ - KokkosLMP::finalize(); -} - class AtomKokkos : public Atom { public: tagint **k_special; diff --git a/src/main.cpp b/src/main.cpp index 606f31274d..543833498b 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -32,9 +32,19 @@ #include #endif +#if defined(LMP_KOKKOS) namespace LAMMPS_NS { extern void kokkos_lmp_finalize(); } +#else +#include "accelerator_kokkos.h" +namespace LAMMPS_NS { +void kokkos_lmp_finalize() +{ + KokkosLMP::finalize(); +} +} +#endif using namespace LAMMPS_NS; From 6c1ed6a9b52ffb36fc4159306aa8c8cdee8b929b Mon Sep 17 00:00:00 2001 From: Axel Kohlmeyer Date: Tue, 7 Mar 2023 08:23:36 -0500 Subject: [PATCH 3/5] use finalize wrappers from the LAMMPS library interface --- src/KOKKOS/kokkos.cpp | 9 --------- src/main.cpp | 36 +++++++++++------------------------- 2 files changed, 11 insertions(+), 34 deletions(-) diff --git a/src/KOKKOS/kokkos.cpp b/src/KOKKOS/kokkos.cpp index 4e96e3218c..8b45c786e5 100644 --- a/src/KOKKOS/kokkos.cpp +++ b/src/KOKKOS/kokkos.cpp @@ -33,10 +33,6 @@ #include // for getpid() #endif -namespace LAMMPS_NS { - void kokkos_lmp_finalize(); -} - #ifdef LMP_KOKKOS_GPU // for detecting GPU-aware MPI support: @@ -84,11 +80,6 @@ using namespace LAMMPS_NS; int KokkosLMP::is_finalized = 0; int KokkosLMP::init_ngpus = 0; -void LAMMPS_NS::kokkos_lmp_finalize() -{ - KokkosLMP::finalize(); -} - /* ---------------------------------------------------------------------- */ KokkosLMP::KokkosLMP(LAMMPS *lmp, int narg, char **arg) : Pointers(lmp) diff --git a/src/main.cpp b/src/main.cpp index 543833498b..746ccc41e9 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -14,7 +14,7 @@ #include "lammps.h" #include "input.h" -#include "lmppython.h" +#include "library.h" #if defined(LAMMPS_EXCEPTIONS) #include "exceptions.h" @@ -32,20 +32,6 @@ #include #endif -#if defined(LMP_KOKKOS) -namespace LAMMPS_NS { -extern void kokkos_lmp_finalize(); -} -#else -#include "accelerator_kokkos.h" -namespace LAMMPS_NS { -void kokkos_lmp_finalize() -{ - KokkosLMP::finalize(); -} -} -#endif - using namespace LAMMPS_NS; /* ---------------------------------------------------------------------- @@ -89,19 +75,19 @@ int main(int argc, char **argv) lammps->input->file(); delete lammps; } catch (LAMMPSAbortException &ae) { - kokkos_lmp_finalize(); - Python::finalize(); + lammps_kokkos_finalize(); + lammps_python_finalize(); MPI_Abort(ae.universe, 1); } catch (LAMMPSException &) { - kokkos_lmp_finalize(); - Python::finalize(); + lammps_kokkos_finalize(); + lammps_python_finalize(); MPI_Barrier(lammps_comm); MPI_Finalize(); exit(1); } catch (fmt::format_error &fe) { fprintf(stderr, "fmt::format_error: %s\n", fe.what()); - kokkos_lmp_finalize(); - Python::finalize(); + lammps_kokkos_finalize(); + lammps_python_finalize(); MPI_Abort(MPI_COMM_WORLD, 1); exit(1); } @@ -112,14 +98,14 @@ int main(int argc, char **argv) delete lammps; } catch (fmt::format_error &fe) { fprintf(stderr, "fmt::format_error: %s\n", fe.what()); - kokkos_lmp_finalize(); - Python::finalize(); + lammps_kokkos_finalize(); + lammps_python_finalize(); MPI_Abort(MPI_COMM_WORLD, 1); exit(1); } #endif - kokkos_lmp_finalize(); - Python::finalize(); + lammps_kokkos_finalize(); + lammps_python_finalize(); MPI_Barrier(lammps_comm); MPI_Finalize(); } From 2aa0e76ad97409759cace54a604fb8450579e760 Mon Sep 17 00:00:00 2001 From: Axel Kohlmeyer Date: Tue, 7 Mar 2023 13:43:27 -0500 Subject: [PATCH 4/5] avoid linking errors due to nvcc wrapper warning silencing. --- cmake/CMakeLists.txt | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/cmake/CMakeLists.txt b/cmake/CMakeLists.txt index a6956f5f5d..41229e9cd6 100644 --- a/cmake/CMakeLists.txt +++ b/cmake/CMakeLists.txt @@ -538,7 +538,10 @@ set(CMAKE_TUNE_FLAGS "${CMAKE_TUNE_DEFAULT}" CACHE STRING "Compiler and machine separate_arguments(CMAKE_TUNE_FLAGS) foreach(_FLAG ${CMAKE_TUNE_FLAGS}) target_compile_options(lammps PRIVATE ${_FLAG}) - target_compile_options(lmp PRIVATE ${_FLAG}) + # skip these flags when linking the main executable + if(NOT (("${_FLAG}" STREQUAL "-Xcudafe") OR (("${_FLAG}" STREQUAL "--diag_suppress=unrecognized_pragma")))) + target_compile_options(lmp PRIVATE ${_FLAG}) + endif() endforeach() ######################################################################## # Basic system tests (standard libraries, headers, functions, types) # From b21d915a7ce5e1b55008383f3d38ec174c0abc0d Mon Sep 17 00:00:00 2001 From: Axel Kohlmeyer Date: Tue, 7 Mar 2023 13:47:39 -0500 Subject: [PATCH 5/5] simplify --- src/main.cpp | 28 +++++++++++++++++----------- 1 file changed, 17 insertions(+), 11 deletions(-) diff --git a/src/main.cpp b/src/main.cpp index 746ccc41e9..e3e304a0d4 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -34,6 +34,13 @@ using namespace LAMMPS_NS; +// for convenience +static void finalize() +{ + lammps_kokkos_finalize(); + lammps_python_finalize(); +} + /* ---------------------------------------------------------------------- main program to drive LAMMPS ------------------------------------------------------------------------- */ @@ -41,7 +48,6 @@ using namespace LAMMPS_NS; int main(int argc, char **argv) { MPI_Init(&argc, &argv); - MPI_Comm lammps_comm = MPI_COMM_WORLD; #if defined(LMP_MDI) @@ -75,19 +81,21 @@ int main(int argc, char **argv) lammps->input->file(); delete lammps; } catch (LAMMPSAbortException &ae) { - lammps_kokkos_finalize(); - lammps_python_finalize(); + finalize(); MPI_Abort(ae.universe, 1); } catch (LAMMPSException &) { - lammps_kokkos_finalize(); - lammps_python_finalize(); + finalize(); MPI_Barrier(lammps_comm); MPI_Finalize(); exit(1); } catch (fmt::format_error &fe) { fprintf(stderr, "fmt::format_error: %s\n", fe.what()); - lammps_kokkos_finalize(); - lammps_python_finalize(); + finalize(); + MPI_Abort(MPI_COMM_WORLD, 1); + exit(1); + } catch (std::exception &e) { + fprintf(stderr, "Exception: %s\n", e.what()); + finalize(); MPI_Abort(MPI_COMM_WORLD, 1); exit(1); } @@ -98,14 +106,12 @@ int main(int argc, char **argv) delete lammps; } catch (fmt::format_error &fe) { fprintf(stderr, "fmt::format_error: %s\n", fe.what()); - lammps_kokkos_finalize(); - lammps_python_finalize(); + finalize(); MPI_Abort(MPI_COMM_WORLD, 1); exit(1); } #endif - lammps_kokkos_finalize(); - lammps_python_finalize(); + finalize(); MPI_Barrier(lammps_comm); MPI_Finalize(); }