diff --git a/.github/workflows/coverity.yml b/.github/workflows/coverity.yml index c0c3e3f89a..2691b9e895 100644 --- a/.github/workflows/coverity.yml +++ b/.github/workflows/coverity.yml @@ -67,7 +67,6 @@ jobs: -D PKG_MANIFOLD=on \ -D PKG_MDI=on \ -D PKG_MGPT=on \ - -D PKG_ML-PACE=on \ -D PKG_ML-RANN=on \ -D PKG_MOLFILE=on \ -D PKG_NETCDF=on \ diff --git a/.github/workflows/lammps-gui-flatpak.yml b/.github/workflows/lammps-gui-flatpak.yml new file mode 100644 index 0000000000..d7dc602476 --- /dev/null +++ b/.github/workflows/lammps-gui-flatpak.yml @@ -0,0 +1,53 @@ +# GitHub action to build LAMMPS-GUI as a flatpak bundle +name: "Build LAMMPS-GUI as flatpak bundle" + +on: + push: + branches: + - develop + + workflow_dispatch: + +concurrency: + group: ${{ github.event_name }}-${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: ${{github.event_name == 'pull_request'}} + +jobs: + build: + name: LAMMPS-GUI flatpak build + if: ${{ github.repository == 'lammps/lammps' }} + runs-on: ubuntu-latest + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + fetch-depth: 2 + + - name: Install extra packages + run: | + sudo apt-get update + sudo apt-get install -y ccache \ + libeigen3-dev \ + libcurl4-openssl-dev \ + mold \ + ninja-build \ + python3-dev \ + flatpak \ + flatpak-builder + + - name: Set up access to flatpak repo + run: flatpak --user remote-add --if-not-exists flathub https://dl.flathub.org/repo/flathub.flatpakrepo + + - name: Build flatpak + run: | + mkdir flatpack-state + sed -i -e 's/branch:.*/branch: develop/' tools/lammps-gui/org.lammps.lammps-gui.yml + flatpak-builder --force-clean --verbose --repo=flatpak-repo \ + --install-deps-from=flathub --state-dir=flatpak-state \ + --user --ccache --default-branch=${{ github.ref_name }} \ + flatpak-build tools/lammps-gui/org.lammps.lammps-gui.yml + flatpak build-bundle --runtime-repo=https://flathub.org/repo/flathub.flatpakrepo \ + --verbose flatpak-repo LAMMPS-Linux-x86_64-GUI.flatpak \ + org.lammps.lammps-gui ${{ github.ref_name }} + flatpak install -y -v --user LAMMPS-Linux-x86_64-GUI.flatpak diff --git a/.github/workflows/style-check.yml b/.github/workflows/style-check.yml index 7be2c4fc46..e3567140fb 100644 --- a/.github/workflows/style-check.yml +++ b/.github/workflows/style-check.yml @@ -35,3 +35,4 @@ jobs: make check-permissions make check-homepage make check-errordocs + make check-fmtlib diff --git a/.github/workflows/unittest-arm64.yml b/.github/workflows/unittest-arm64.yml new file mode 100644 index 0000000000..094c5fb0c1 --- /dev/null +++ b/.github/workflows/unittest-arm64.yml @@ -0,0 +1,81 @@ +# GitHub action to build LAMMPS on Linux with ARM64 and run standard unit tests +name: "Unittest for Linux on ARM64" + +on: + push: + branches: [develop] + + workflow_dispatch: + +concurrency: + group: ${{ github.event_name }}-${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: ${{github.event_name == 'pull_request'}} + +jobs: + build: + name: Linux ARM64 Unit Test + if: ${{ github.repository == 'lammps/lammps' }} + runs-on: ubuntu-22.04-arm + env: + CCACHE_DIR: ${{ github.workspace }}/.ccache + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + fetch-depth: 2 + + - name: Install extra packages + run: | + sudo apt-get update + sudo apt-get install -y ccache \ + libeigen3-dev \ + libcurl4-openssl-dev \ + mold \ + ninja-build \ + python3-dev + + - name: Create Build Environment + run: mkdir build + + - name: Set up ccache + uses: actions/cache@v4 + with: + path: ${{ env.CCACHE_DIR }} + key: linux-unit-ccache-${{ github.sha }} + restore-keys: linux-unit-ccache- + + - name: Building LAMMPS via CMake + shell: bash + run: | + ccache -z + python3 -m venv linuxenv + source linuxenv/bin/activate + python3 -m pip install numpy + python3 -m pip install pyyaml + cmake -S cmake -B build \ + -C cmake/presets/gcc.cmake \ + -C cmake/presets/most.cmake \ + -D CMAKE_CXX_COMPILER_LAUNCHER=ccache \ + -D CMAKE_C_COMPILER_LAUNCHER=ccache \ + -D BUILD_SHARED_LIBS=on \ + -D DOWNLOAD_POTENTIALS=off \ + -D ENABLE_TESTING=on \ + -D MLIAP_ENABLE_ACE=on \ + -D MLIAP_ENABLE_PYTHON=off \ + -D PKG_MANIFOLD=on \ + -D PKG_ML-PACE=on \ + -D PKG_ML-RANN=on \ + -D PKG_RHEO=on \ + -D PKG_PTM=on \ + -D PKG_PYTHON=on \ + -D PKG_QTB=on \ + -D PKG_SMTBQ=on \ + -G Ninja + cmake --build build + ccache -s + + - name: Run Tests + working-directory: build + shell: bash + run: ctest -V -LE unstable diff --git a/cmake/CMakeLists.txt b/cmake/CMakeLists.txt index cf10e8b544..ff0d69e316 100644 --- a/cmake/CMakeLists.txt +++ b/cmake/CMakeLists.txt @@ -3,6 +3,9 @@ # CMake build system # This file is part of LAMMPS cmake_minimum_required(VERSION 3.16) +if(CMAKE_VERSION VERSION_LESS 3.20) + message(WARNING "LAMMPS is planning to require at least CMake version 3.20 by Summer 2025. Please upgrade!") +endif() ######################################## # set policy to silence warnings about ignoring _ROOT but use it if(POLICY CMP0074) @@ -144,16 +147,28 @@ if((PKG_KOKKOS) AND (Kokkos_ENABLE_CUDA) AND NOT (CMAKE_CXX_COMPILER_ID STREQUAL set(CMAKE_TUNE_DEFAULT "${CMAKE_TUNE_DEFAULT}" "-Xcudafe --diag_suppress=unrecognized_pragma,--diag_suppress=128") endif() -# we require C++11 without extensions. Kokkos requires at least C++17 (currently) +# we *require* C++11 without extensions but prefer C++17. +# Kokkos requires at least C++17 (currently) if(NOT CMAKE_CXX_STANDARD) - set(CMAKE_CXX_STANDARD 11) + if(cxx_std_17 IN_LIST CMAKE_CXX_COMPILE_FEATURES) + set(CMAKE_CXX_STANDARD 17) + else() + set(CMAKE_CXX_STANDARD 11) + endif() endif() if(CMAKE_CXX_STANDARD LESS 11) message(FATAL_ERROR "C++ standard must be set to at least 11") endif() +if(CMAKE_CXX_STANDARD LESS 17) + message(WARNING "Selecting C++17 standard is preferred over C++${CMAKE_CXX_STANDARD}") +endif() if(PKG_KOKKOS AND (CMAKE_CXX_STANDARD LESS 17)) set(CMAKE_CXX_STANDARD 17) endif() +# turn off C++17 check in lmptype.h +if(LAMMPS_CXX11) + add_compile_definitions(LAMMPS_CXX11) +endif() set(CMAKE_CXX_STANDARD_REQUIRED ON) set(CMAKE_CXX_EXTENSIONS OFF CACHE BOOL "Use compiler extensions") # ugly hacks for MSVC which by default always reports an old C++ standard in the __cplusplus macro @@ -347,6 +362,17 @@ foreach(PKG ${STANDARD_PACKAGES} ${SUFFIX_PACKAGES}) option(PKG_${PKG} "Build ${PKG} Package" OFF) endforeach() +set(DEPRECATED_PACKAGES AWPMD ATC POEMS) +foreach(PKG ${DEPRECATED_PACKAGES}) + if(PKG_${PKG}) + message(WARNING + "The ${PKG} package will be removed from LAMMPS in Summer 2025 due to lack of " + "maintenance and use of code constructs that conflict with modern C++ compilers " + "and standards. Please contact developers@lammps.org if you have any concerns " + "about this step.") + endif() +endforeach() + ###################################################### # packages with special compiler needs or external libs ###################################################### @@ -579,6 +605,16 @@ foreach(PKG_WITH_INCL KSPACE PYTHON ML-IAP VORONOI COLVARS ML-HDNNP MDI MOLFILE endif() endforeach() +# settings for misc packages and styles +if(PKG_MISC) + option(LAMMPS_ASYNC_IMD "Asynchronous IMD processing" OFF) + mark_as_advanced(LAMMPS_ASYNC_IMD) + if(LAMMPS_ASYNC_IMD) + target_compile_definitions(lammps PRIVATE -DLAMMPS_ASYNC_IMD) + message(STATUS "Using IMD in asynchronous mode") + endif() +endif() + # optionally enable building script wrappers using swig option(WITH_SWIG "Build scripting language wrappers with SWIG" OFF) if(WITH_SWIG) @@ -1078,12 +1114,15 @@ if(BUILD_TOOLS) message(STATUS "<<< Building Tools >>>") endif() if(BUILD_LAMMPS_GUI) - message(STATUS "<<< Building LAMMPS GUI >>>") + message(STATUS "<<< Building LAMMPS-GUI >>>") if(LAMMPS_GUI_USE_PLUGIN) message(STATUS "Loading LAMMPS library as plugin at run time") else() message(STATUS "Linking LAMMPS library at compile time") endif() + if(BUILD_WHAM) + message(STATUS "<<< Building WHAM >>>") + endif() endif() if(ENABLE_TESTING) message(STATUS "<<< Building Unit Tests >>>") diff --git a/cmake/Modules/Packages/INTEL.cmake b/cmake/Modules/Packages/INTEL.cmake index e6755bf23b..6fb1c57e8a 100644 --- a/cmake/Modules/Packages/INTEL.cmake +++ b/cmake/Modules/Packages/INTEL.cmake @@ -72,6 +72,10 @@ if(INTEL_ARCH STREQUAL "KNL") if(NOT CMAKE_CXX_COMPILER_ID STREQUAL "Intel") message(FATAL_ERROR "Must use Intel compiler with INTEL for KNL architecture") endif() + message(WARNING, "Support for Intel Xeon Phi accelerators and Knight's Landing CPUs " + "will be removed from LAMMPS in Summer 2025 due to lack of available machines " + "in labs and HPC centers and removed support in recent compilers " + "Please contact developers@lammps.org if you have any concerns about this step.") set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -xHost -qopenmp -qoffload") set(MIC_OPTIONS "-qoffload-option,mic,compiler,\"-fp-model fast=2 -mGLOB_default_function_attrs=\\\"gather_scatter_loop_unroll=4\\\"\"") target_compile_options(lammps PRIVATE -xMIC-AVX512 -qoffload -fno-alias -ansi-alias -restrict -qoverride-limits ${MIC_OPTIONS}) diff --git a/cmake/Modules/Packages/KOKKOS.cmake b/cmake/Modules/Packages/KOKKOS.cmake index ddd2daefcd..2731b0df14 100644 --- a/cmake/Modules/Packages/KOKKOS.cmake +++ b/cmake/Modules/Packages/KOKKOS.cmake @@ -7,26 +7,13 @@ endif() ######################################################################## # consistency checks and Kokkos options/settings required by LAMMPS -if(Kokkos_ENABLE_CUDA) - option(Kokkos_ENABLE_IMPL_CUDA_MALLOC_ASYNC "CUDA asynchronous malloc support" OFF) - mark_as_advanced(Kokkos_ENABLE_IMPL_CUDA_MALLOC_ASYNC) - if(Kokkos_ENABLE_IMPL_CUDA_MALLOC_ASYNC) - message(STATUS "KOKKOS: CUDA malloc async support enabled") - else() - message(STATUS "KOKKOS: CUDA malloc async support disabled") - endif() -endif() if(Kokkos_ENABLE_HIP) option(Kokkos_ENABLE_HIP_MULTIPLE_KERNEL_INSTANTIATIONS "Enable multiple kernel instantiations with HIP" ON) mark_as_advanced(Kokkos_ENABLE_HIP_MULTIPLE_KERNEL_INSTANTIATIONS) option(Kokkos_ENABLE_ROCTHRUST "Use RoCThrust library" ON) mark_as_advanced(Kokkos_ENABLE_ROCTHRUST) - - if(Kokkos_ARCH_AMD_GFX942 OR Kokkos_ARCH_AMD_GFX940) - option(Kokkos_ENABLE_IMPL_HIP_UNIFIED_MEMORY "Enable unified memory with HIP" ON) - mark_as_advanced(Kokkos_ENABLE_IMPL_HIP_UNIFIED_MEMORY) - endif() endif() + # Adding OpenMP compiler flags without the checks done for # BUILD_OMP can result in compile failures. Enforce consistency. if(Kokkos_ENABLE_OPENMP) @@ -70,8 +57,8 @@ if(DOWNLOAD_KOKKOS) list(APPEND KOKKOS_LIB_BUILD_ARGS "-DCMAKE_CXX_EXTENSIONS=${CMAKE_CXX_EXTENSIONS}") list(APPEND KOKKOS_LIB_BUILD_ARGS "-DCMAKE_TOOLCHAIN_FILE=${CMAKE_TOOLCHAIN_FILE}") include(ExternalProject) - set(KOKKOS_URL "https://github.com/kokkos/kokkos/archive/4.4.01.tar.gz" CACHE STRING "URL for KOKKOS tarball") - set(KOKKOS_MD5 "de6ee80d00b6212b02bfb7f1e71a8392" CACHE STRING "MD5 checksum of KOKKOS tarball") + set(KOKKOS_URL "https://github.com/kokkos/kokkos/archive/4.5.01.tar.gz" CACHE STRING "URL for KOKKOS tarball") + set(KOKKOS_MD5 "4d832aa0284169d9e3fbae3165286bc6" CACHE STRING "MD5 checksum of KOKKOS tarball") mark_as_advanced(KOKKOS_URL) mark_as_advanced(KOKKOS_MD5) GetFallbackURL(KOKKOS_URL KOKKOS_FALLBACK) @@ -96,7 +83,7 @@ if(DOWNLOAD_KOKKOS) add_dependencies(LAMMPS::KOKKOSCORE kokkos_build) add_dependencies(LAMMPS::KOKKOSCONTAINERS kokkos_build) elseif(EXTERNAL_KOKKOS) - find_package(Kokkos 4.4.01 REQUIRED CONFIG) + find_package(Kokkos 4.5.01 REQUIRED CONFIG) target_link_libraries(lammps PRIVATE Kokkos::kokkos) else() set(LAMMPS_LIB_KOKKOS_SRC_DIR ${LAMMPS_LIB_SOURCE_DIR}/kokkos) @@ -130,7 +117,6 @@ set(KOKKOS_PKG_SOURCES ${KOKKOS_PKG_SOURCES_DIR}/kokkos.cpp ${KOKKOS_PKG_SOURCES_DIR}/atom_vec_kokkos.cpp ${KOKKOS_PKG_SOURCES_DIR}/comm_kokkos.cpp ${KOKKOS_PKG_SOURCES_DIR}/comm_tiled_kokkos.cpp - ${KOKKOS_PKG_SOURCES_DIR}/group_kokkos.cpp ${KOKKOS_PKG_SOURCES_DIR}/min_kokkos.cpp ${KOKKOS_PKG_SOURCES_DIR}/min_linesearch_kokkos.cpp ${KOKKOS_PKG_SOURCES_DIR}/neighbor_kokkos.cpp diff --git a/cmake/Modules/Packages/ML-PACE.cmake b/cmake/Modules/Packages/ML-PACE.cmake index 8660898138..b30c61b8e4 100644 --- a/cmake/Modules/Packages/ML-PACE.cmake +++ b/cmake/Modules/Packages/ML-PACE.cmake @@ -1,50 +1,62 @@ # PACE library support for ML-PACE package +find_package(pace QUIET) -# set policy to silence warnings about timestamps of downloaded files. review occasionally if it may be set to NEW -if(POLICY CMP0135) - cmake_policy(SET CMP0135 OLD) -endif() - -set(PACELIB_URL "https://github.com/ICAMS/lammps-user-pace/archive/refs/tags/v.2023.11.25.fix.tar.gz" CACHE STRING "URL for PACE evaluator library sources") -set(PACELIB_MD5 "b45de9a633f42ed65422567e3ce56f9f" CACHE STRING "MD5 checksum of PACE evaluator library tarball") -mark_as_advanced(PACELIB_URL) -mark_as_advanced(PACELIB_MD5) -GetFallbackURL(PACELIB_URL PACELIB_FALLBACK) - -# LOCAL_ML-PACE points to top-level dir with local lammps-user-pace repo, -# to make it easier to check local build without going through the public github releases -if(LOCAL_ML-PACE) - set(lib-pace "${LOCAL_ML-PACE}") +if(pace_FOUND) + find_package(pace) + target_link_libraries(lammps PRIVATE pace::pace) else() - # download library sources to build folder - if(EXISTS ${CMAKE_BINARY_DIR}/libpace.tar.gz) - file(MD5 ${CMAKE_BINARY_DIR}/libpace.tar.gz DL_MD5) - endif() - if(NOT "${DL_MD5}" STREQUAL "${PACELIB_MD5}") - message(STATUS "Downloading ${PACELIB_URL}") - file(DOWNLOAD ${PACELIB_URL} ${CMAKE_BINARY_DIR}/libpace.tar.gz STATUS DL_STATUS SHOW_PROGRESS) - file(MD5 ${CMAKE_BINARY_DIR}/libpace.tar.gz DL_MD5) - if((NOT DL_STATUS EQUAL 0) OR (NOT "${DL_MD5}" STREQUAL "${PACELIB_MD5}")) - message(WARNING "Download from primary URL ${PACELIB_URL} failed\nTrying fallback URL ${PACELIB_FALLBACK}") - file(DOWNLOAD ${PACELIB_FALLBACK} ${CMAKE_BINARY_DIR}/libpace.tar.gz EXPECTED_HASH MD5=${PACELIB_MD5} SHOW_PROGRESS) + # set policy to silence warnings about timestamps of downloaded files. review occasionally if it may be set to NEW + if(POLICY CMP0135) + cmake_policy(SET CMP0135 OLD) endif() - else() - message(STATUS "Using already downloaded archive ${CMAKE_BINARY_DIR}/libpace.tar.gz") - endif() + + set(PACELIB_URL "https://github.com/ICAMS/lammps-user-pace/archive/refs/tags/v.2023.11.25.fix2.tar.gz" CACHE STRING "URL for PACE evaluator library sources") + set(PACELIB_MD5 "a53bd87cfee8b07d9f44bc17aad69c3f" CACHE STRING "MD5 checksum of PACE evaluator library tarball") + mark_as_advanced(PACELIB_URL) + mark_as_advanced(PACELIB_MD5) + GetFallbackURL(PACELIB_URL PACELIB_FALLBACK) + + # LOCAL_ML-PACE points to top-level dir with local lammps-user-pace repo, + # to make it easier to check local build without going through the public github releases + if(LOCAL_ML-PACE) + set(lib-pace "${LOCAL_ML-PACE}") + else() + # download library sources to build folder + if(EXISTS ${CMAKE_BINARY_DIR}/libpace.tar.gz) + file(MD5 ${CMAKE_BINARY_DIR}/libpace.tar.gz DL_MD5) + endif() + if(NOT "${DL_MD5}" STREQUAL "${PACELIB_MD5}") + message(STATUS "Downloading ${PACELIB_URL}") + file(DOWNLOAD ${PACELIB_URL} ${CMAKE_BINARY_DIR}/libpace.tar.gz STATUS DL_STATUS SHOW_PROGRESS) + file(MD5 ${CMAKE_BINARY_DIR}/libpace.tar.gz DL_MD5) + if((NOT DL_STATUS EQUAL 0) OR (NOT "${DL_MD5}" STREQUAL "${PACELIB_MD5}")) + message(WARNING "Download from primary URL ${PACELIB_URL} failed\nTrying fallback URL ${PACELIB_FALLBACK}") + file(DOWNLOAD ${PACELIB_FALLBACK} ${CMAKE_BINARY_DIR}/libpace.tar.gz EXPECTED_HASH MD5=${PACELIB_MD5} SHOW_PROGRESS) + endif() + else() + message(STATUS "Using already downloaded archive ${CMAKE_BINARY_DIR}/libpace.tar.gz") + endif() - # uncompress downloaded sources - execute_process( - COMMAND ${CMAKE_COMMAND} -E remove_directory lammps-user-pace* - COMMAND ${CMAKE_COMMAND} -E tar xzf libpace.tar.gz - WORKING_DIRECTORY ${CMAKE_BINARY_DIR} - ) - get_newest_file(${CMAKE_BINARY_DIR}/lammps-user-pace-* lib-pace) -endif() - -add_subdirectory(${lib-pace} build-pace) -set_target_properties(pace PROPERTIES CXX_EXTENSIONS ON OUTPUT_NAME lammps_pace${LAMMPS_MACHINE}) - -if(CMAKE_PROJECT_NAME STREQUAL "lammps") - target_link_libraries(lammps PRIVATE pace) + # uncompress downloaded sources + execute_process( + COMMAND ${CMAKE_COMMAND} -E remove_directory lammps-user-pace* + COMMAND ${CMAKE_COMMAND} -E tar xzf libpace.tar.gz + WORKING_DIRECTORY ${CMAKE_BINARY_DIR} + ) + get_newest_file(${CMAKE_BINARY_DIR}/lammps-user-pace-* lib-pace) + endif() + + # some preinstalled yaml-cpp versions don't provide a namespaced target + find_package(yaml-cpp QUIET) + if(TARGET yaml-cpp AND NOT TARGET yaml-cpp::yaml-cpp) + add_library(yaml-cpp::yaml-cpp ALIAS yaml-cpp) + endif() + + add_subdirectory(${lib-pace} build-pace) + set_target_properties(pace PROPERTIES CXX_EXTENSIONS ON OUTPUT_NAME lammps_pace${LAMMPS_MACHINE}) + + if(CMAKE_PROJECT_NAME STREQUAL "lammps") + target_link_libraries(lammps PRIVATE pace) + endif() endif() diff --git a/cmake/Modules/Packages/PLUMED.cmake b/cmake/Modules/Packages/PLUMED.cmake index 8dab157a24..5e5d110626 100644 --- a/cmake/Modules/Packages/PLUMED.cmake +++ b/cmake/Modules/Packages/PLUMED.cmake @@ -32,9 +32,9 @@ endif() # Note: must also adjust check for supported API versions in # fix_plumed.cpp when version changes from v2.n.x to v2.n+1.y -set(PLUMED_URL "https://github.com/plumed/plumed2/releases/download/v2.9.2/plumed-src-2.9.2.tgz" +set(PLUMED_URL "https://github.com/plumed/plumed2/releases/download/v2.9.3/plumed-src-2.9.3.tgz" CACHE STRING "URL for PLUMED tarball") -set(PLUMED_MD5 "04862602a372c1013bdfee2d6d03bace" CACHE STRING "MD5 checksum of PLUMED tarball") +set(PLUMED_MD5 "ee1249805fe94bccee17d10610d3f6f1" CACHE STRING "MD5 checksum of PLUMED tarball") mark_as_advanced(PLUMED_URL) mark_as_advanced(PLUMED_MD5) diff --git a/cmake/Modules/Packages/VTK.cmake b/cmake/Modules/Packages/VTK.cmake index a0de1e0ff4..48b1cdba11 100644 --- a/cmake/Modules/Packages/VTK.cmake +++ b/cmake/Modules/Packages/VTK.cmake @@ -1,3 +1,5 @@ +# FindVTK requires that C support is enabled when looking for MPI support +enable_language(C) find_package(VTK REQUIRED NO_MODULE) target_compile_definitions(lammps PRIVATE -DLAMMPS_VTK) if (VTK_MAJOR_VERSION VERSION_LESS 9.0) diff --git a/doc/doxygen/Doxyfile.in b/doc/doxygen/Doxyfile.in index e905d5a64d..e3a5415b20 100644 --- a/doc/doxygen/Doxyfile.in +++ b/doc/doxygen/Doxyfile.in @@ -2,7 +2,7 @@ DOXYFILE_ENCODING = UTF-8 PROJECT_NAME = "LAMMPS Programmer's Guide" -PROJECT_NUMBER = "4 May 2022" +PROJECT_NUMBER = "19 November 2024" PROJECT_BRIEF = "Documentation of the LAMMPS library interface and Python wrapper" PROJECT_LOGO = lammps-logo.png CREATE_SUBDIRS = NO diff --git a/doc/src/Build.rst b/doc/src/Build.rst index 7cf2a01992..7ca8cd428e 100644 --- a/doc/src/Build.rst +++ b/doc/src/Build.rst @@ -1,10 +1,14 @@ Build LAMMPS ============ -LAMMPS is built as a library and an executable from source code using -either traditional makefiles for use with GNU make (which may require -manual editing), or using a build environment generated by CMake (Unix -Makefiles, Ninja, Xcode, Visual Studio, KDevelop, CodeBlocks and more). +LAMMPS is built as a library and an executable from source code using a +build environment generated by CMake (Unix Makefiles, Ninja, Xcode, +Visual Studio, KDevelop, CodeBlocks and more depending on the platform). +Using CMake is the preferred way to build LAMMPS. In addition, LAMMPS +can be compiled using the legacy build system based on traditional +makefiles for use with GNU make (which may require manual editing). +Support for the legacy build system is slowly being phased out and may +not be available for all optional features. As an alternative, you can download a package with pre-built executables or automated build trees, as described in the :doc:`Install ` diff --git a/doc/src/Build_basics.rst b/doc/src/Build_basics.rst index e82832cf57..be4b312578 100644 --- a/doc/src/Build_basics.rst +++ b/doc/src/Build_basics.rst @@ -160,7 +160,7 @@ with the OpenMP 3.1 semantics used in LAMMPS for maximal compatibility with compiler versions in use. If compilation with OpenMP enabled fails because of your compiler requiring strict OpenMP 4.0 semantics, you can change the behavior by adding ``-D LAMMPS_OMP_COMPAT=4`` to the -``LMP_INC`` variable in your makefile, or add it to the command line +``LMP_INC`` variable in your makefile, or add it to the command-line flags while configuring with CMake. LAMMPS will auto-detect a suitable setting for most GNU, Clang, and Intel compilers. @@ -196,13 +196,18 @@ LAMMPS. .. tab:: CMake build - By default CMake will use the compiler it finds according to + By default CMake will use the compiler it finds according to its internal preferences, and it will add optimization flags appropriate to that compiler and any :doc:`accelerator packages ` you have included in the build. CMake will check if the detected or selected compiler is compatible with the C++ support requirements of LAMMPS and stop with an error, if this - is not the case. + is not the case. A C++11 compatible compiler is currently + required, but a transition to require C++17 is in progress and + planned to be completed in Summer 2025. Currently, setting + ``-DLAMMPS_CXX11=yes`` is required when configuring with CMake while + using a C++11 compatible compiler that does not support C++17, + otherwise setting ``-DCMAKE_CXX_STANDARD=17`` is preferred. You can tell CMake to look for a specific compiler with setting CMake variables (listed below) during configuration. For a few @@ -223,6 +228,8 @@ LAMMPS. -D CMAKE_C_COMPILER=name # name of C compiler -D CMAKE_Fortran_COMPILER=name # name of Fortran compiler + -D CMAKE_CXX_STANDARD=17 # put compiler in C++17 mode + -D LAMMPS_CXX11=yes # enforce compilation in C++11 mode -D CMAKE_CXX_FLAGS=string # flags to use with C++ compiler -D CMAKE_C_FLAGS=string # flags to use with C compiler -D CMAKE_Fortran_FLAGS=string # flags to use with Fortran compiler @@ -321,15 +328,23 @@ LAMMPS. you would have to install a newer compiler that supports C++11; either as a binary package or through compiling from source. - If you build LAMMPS with any :doc:`Speed_packages` included, - there may be specific compiler or linker flags that are either - required or recommended to enable required features and to - achieve optimal performance. You need to include these in the - ``CCFLAGS`` and ``LINKFLAGS`` settings above. For details, see the - documentation for the individual packages listed on the - :doc:`Speed_packages` page. Or examine these files in the - ``src/MAKE/OPTIONS`` directory. They correspond to each of the 5 - accelerator packages and their hardware variants: + While a C++11 compatible compiler is currently sufficient to compile + LAMMPS, a transition to require C++17 is in progress and planned to + be completed in Summer 2025. Currently, setting ``-DLAMMPS_CXX11`` + in the ``LMP_INC =`` line in the machine makefile is required when + using a C++11 compatible compiler that does not support C++17. + Otherwise, to enable C++17 support (if not enabled by default) using + a compiler flag like ``-std=c++17`` in CCFLAGS may needed. + + If you build LAMMPS with any :doc:`Speed_packages` included, + there may be specific compiler or linker flags that are either + required or recommended to enable required features and to + achieve optimal performance. You need to include these in the + ``CCFLAGS`` and ``LINKFLAGS`` settings above. For details, see the + documentation for the individual packages listed on the + :doc:`Speed_packages` page. Or examine these files in the + ``src/MAKE/OPTIONS`` directory. They correspond to each of the 5 + accelerator packages and their hardware variants: .. code-block:: bash @@ -502,6 +517,8 @@ using CMake or Make. # chain.x, micelle2d.x, msi2lmp, phana, # stl_bin2txt -D BUILD_LAMMPS_GUI=value # yes or no (default). Build LAMMPS-GUI + -D BUILD_WHAM=value # yes (default). Download and build WHAM; + # only available for BUILD_LAMMPS_GUI=yes The generated binaries will also become part of the LAMMPS installation (see below). diff --git a/doc/src/Build_cmake.rst b/doc/src/Build_cmake.rst index 1b2bef936e..56b8e450f3 100644 --- a/doc/src/Build_cmake.rst +++ b/doc/src/Build_cmake.rst @@ -8,7 +8,7 @@ packages. Links to those pages on the :doc:`Build overview ` page. The following text assumes some familiarity with CMake and focuses on -using the command line tool ``cmake`` and what settings are supported +using the command-line tool ``cmake`` and what settings are supported for building LAMMPS. A more detailed tutorial on how to use CMake itself, the text mode or graphical user interface, to change the generated output files for different build tools and development @@ -16,7 +16,7 @@ environments is on a :doc:`separate page `. .. note:: - LAMMPS currently requires that CMake version 3.16 or later is available. + LAMMPS currently requires that CMake version 3.20 or later is available. .. warning:: @@ -32,22 +32,22 @@ environments is on a :doc:`separate page `. Advantages of using CMake ^^^^^^^^^^^^^^^^^^^^^^^^^ -CMake is an alternative to compiling LAMMPS in the traditional way -through :doc:`(manually customized) makefiles `. Using -CMake has multiple advantages that are specifically helpful for -people with limited experience in compiling software or for people -that want to modify or extend LAMMPS. +CMake is the preferred way of compiling LAMMPS in contrast to the legacy +build system based on GNU make and through :doc:`(manually customized) +makefiles `. Using CMake has multiple advantages that are +specifically helpful for people with limited experience in compiling +software or for people that want to modify or extend LAMMPS. - CMake can detect available hardware, tools, features, and libraries and adapt the LAMMPS default build configuration accordingly. - CMake can generate files for different build tools and integrated development environments (IDE). -- CMake supports customization of settings with a command line, text +- CMake supports customization of settings with a command-line, text mode, or graphical user interface. No manual editing of files, - knowledge of file formats or complex command line syntax is required. + knowledge of file formats or complex command-line syntax is required. - All enabled components are compiled in a single build operation. - Automated dependency tracking for all files and configuration options. -- Support for true out-of-source compilation. Multiple configurations +- Support for true out-of-source compilation. Multiple configurations and settings with different choices of LAMMPS packages, settings, or compilers can be configured and built concurrently from the same source tree. @@ -68,7 +68,7 @@ that purpose you can use either the command-line utility ``cmake`` (or graphical utility ``cmake-gui``, or use them interchangeably. The second step is then the compilation and linking of all objects, libraries, and executables using the selected build tool. Here is a -minimal example using the command line version of CMake to build LAMMPS +minimal example using the command-line version of CMake to build LAMMPS with no add-on packages enabled and no customization: .. code-block:: bash @@ -131,7 +131,7 @@ file called ``CMakeLists.txt`` (for LAMMPS it is located in the configuration step. The cache file contains all current CMake settings. To modify settings, enable or disable features, you need to set -*variables* with either the ``-D`` command line flag (``-D +*variables* with either the ``-D`` command-line flag (``-D VARIABLE1_NAME=value``) or change them in the text mode of the graphical user interface. The ``-D`` flag can be used several times in one command. @@ -141,11 +141,11 @@ a different compiler tool chain. Those are loaded with the ``-C`` flag (``-C ../cmake/presets/basic.cmake``). This step would only be needed once, as the settings from the preset files are stored in the ``CMakeCache.txt`` file. It is also possible to customize the build -by adding one or more ``-D`` flags to the CMake command line. +by adding one or more ``-D`` flags to the CMake command. Generating files for alternate build tools (e.g. Ninja) and project files for IDEs like Eclipse, CodeBlocks, or Kate can be selected using the ``-G`` -command line flag. A list of available generator settings for your +command-line flag. A list of available generator settings for your specific CMake version is given when running ``cmake --help``. .. _cmake_multiconfig: diff --git a/doc/src/Build_development.rst b/doc/src/Build_development.rst index 3adec76abb..5c6475c7fa 100644 --- a/doc/src/Build_development.rst +++ b/doc/src/Build_development.rst @@ -263,9 +263,9 @@ will be skipped if prerequisite features are not available in LAMMPS. time. Preference is given to parts of the code base that are easy to test or commonly used. -Tests as shown by the ``ctest`` program are command lines defined in the +Tests as shown by the ``ctest`` program are commands defined in the ``CMakeLists.txt`` files in the ``unittest`` directory tree. A few -tests simply execute LAMMPS with specific command line flags and check +tests simply execute LAMMPS with specific command-line flags and check the output to the screen for expected content. A large number of unit tests are special tests programs using the `GoogleTest framework `_ and linked to the LAMMPS @@ -420,7 +420,7 @@ during MD timestepping and manipulate per-atom properties like positions, velocities, and forces. For those fix styles, testing can be done in a very similar fashion as for force fields and thus there is a test program `test_fix_timestep` that shares a lot of code, properties, -and command line flags with the force field style testers described in +and command-line flags with the force field style testers described in the previous section. This tester will set up a small molecular system run with verlet run @@ -642,10 +642,10 @@ The following target are available for both, GNU make and CMake: .. _gh-cli: -GitHub command line interface +GitHub command-line interface ----------------------------- -GitHub has developed a `command line tool `_ +GitHub has developed a `command-line tool `_ to interact with the GitHub website via a command called ``gh``. This is extremely convenient when working with a Git repository hosted on GitHub (like LAMMPS). It is thus highly recommended to install it diff --git a/doc/src/Build_extras.rst b/doc/src/Build_extras.rst index 8465bea829..dab2267ee8 100644 --- a/doc/src/Build_extras.rst +++ b/doc/src/Build_extras.rst @@ -48,6 +48,7 @@ This is the list of packages that may require additional steps. * :ref:`LEPTON ` * :ref:`MACHDYN ` * :ref:`MDI ` + * :ref:`MISC ` * :ref:`ML-HDNNP ` * :ref:`ML-IAP ` * :ref:`ML-PACE ` @@ -209,7 +210,7 @@ necessary for ``hipcc`` and the linker to work correctly. Using the CHIP-SPV implementation of HIP is supported. It allows one to run HIP code on Intel GPUs via the OpenCL or Level Zero back ends. To use CHIP-SPV, you must set ``-DHIP_USE_DEVICE_SORT=OFF`` in your CMake -command line as CHIP-SPV does not yet support hipCUB. As of Summer 2022, +command-line as CHIP-SPV does not yet support hipCUB. As of Summer 2022, the use of HIP for Intel GPUs is experimental. You should only use this option in preparations to run on Aurora system at Argonne. @@ -232,7 +233,7 @@ option in preparations to run on Aurora system at Argonne. .. code:: bash - # CUDA target (not recommended, use GPU_ARCH=cuda) + # CUDA target (not recommended, use GPU_API=cuda) # !!! DO NOT set CMAKE_CXX_COMPILER !!! export HIP_PLATFORM=nvcc export HIP_PATH=/path/to/HIP/install @@ -421,9 +422,10 @@ minutes to hours) to build. Of course you only need to do that once.) cmake build system. The ``lib/kim/Install.py`` script supports a ``CMAKE`` environment variable if the cmake executable is named other than ``cmake`` on your system. Additional environment variables may be - provided on the command line for use by cmake. For example, to use the - ``cmake3`` executable and tell it to use the gnu version 11 compilers - to build KIM, one could use the following command line. + set with the ``make`` command for use by cmake. For example, to use the + ``cmake3`` executable and tell it to use the GNU version 11 compilers + called ``g++-11``, ``gcc-11`` and ``gfortran-11`` to build KIM, one + could use the following command. .. code-block:: bash @@ -546,16 +548,7 @@ They must be specified in uppercase. - Local machine * - AMDAVX - HOST - - AMD 64-bit x86 CPU (AVX 1) - * - ZEN - - HOST - - AMD Zen class CPU (AVX 2) - * - ZEN2 - - HOST - - AMD Zen2 class CPU (AVX 2) - * - ZEN3 - - HOST - - AMD Zen3 class CPU (AVX 2) + - AMD chip * - ARMV80 - HOST - ARMv8.0 Compatible CPU @@ -571,105 +564,126 @@ They must be specified in uppercase. * - A64FX - HOST - ARMv8.2 with SVE Support + * - ARMV9_GRACE + - HOST + - ARMv9 NVIDIA Grace CPU * - SNB - HOST - - Intel Sandy/Ivy Bridge CPU (AVX 1) + - Intel Sandy/Ivy Bridge CPUs * - HSW - HOST - - Intel Haswell CPU (AVX 2) + - Intel Haswell CPUs * - BDW - HOST - - Intel Broadwell Xeon E-class CPU (AVX 2 + transactional mem) - * - SKL - - HOST - - Intel Skylake Client CPU - * - SKX - - HOST - - Intel Skylake Xeon Server CPU (AVX512) + - Intel Broadwell Xeon E-class CPUs * - ICL - HOST - - Intel Ice Lake Client CPU (AVX512) + - Intel Ice Lake Client CPUs (AVX512) * - ICX - HOST - - Intel Ice Lake Xeon Server CPU (AVX512) - * - SPR + - Intel Ice Lake Xeon Server CPUs (AVX512) + * - SKL - HOST - - Intel Sapphire Rapids Xeon Server CPU (AVX512) + - Intel Skylake Client CPUs + * - SKX + - HOST + - Intel Skylake Xeon Server CPUs (AVX512) * - KNC - HOST - Intel Knights Corner Xeon Phi * - KNL - HOST - Intel Knights Landing Xeon Phi + * - SPR + - HOST + - Intel Sapphire Rapids Xeon Server CPUs (AVX512) * - POWER8 - HOST - - IBM POWER8 CPU + - IBM POWER8 CPUs * - POWER9 - HOST - - IBM POWER9 CPU + - IBM POWER9 CPUs + * - ZEN + - HOST + - AMD Zen architecture + * - ZEN2 + - HOST + - AMD Zen2 architecture + * - ZEN3 + - HOST + - AMD Zen3 architecture * - RISCV_SG2042 - HOST - - SG2042 (RISC-V) CPU + - SG2042 (RISC-V) CPUs + * - RISCV_RVA22V + - HOST + - RVA22V (RISC-V) CPUs * - KEPLER30 - GPU - - NVIDIA Kepler generation CC 3.0 GPU + - NVIDIA Kepler generation CC 3.0 * - KEPLER32 - GPU - - NVIDIA Kepler generation CC 3.2 GPU + - NVIDIA Kepler generation CC 3.2 * - KEPLER35 - GPU - - NVIDIA Kepler generation CC 3.5 GPU + - NVIDIA Kepler generation CC 3.5 * - KEPLER37 - GPU - - NVIDIA Kepler generation CC 3.7 GPU + - NVIDIA Kepler generation CC 3.7 * - MAXWELL50 - GPU - - NVIDIA Maxwell generation CC 5.0 GPU + - NVIDIA Maxwell generation CC 5.0 * - MAXWELL52 - GPU - - NVIDIA Maxwell generation CC 5.2 GPU + - NVIDIA Maxwell generation CC 5.2 * - MAXWELL53 - GPU - - NVIDIA Maxwell generation CC 5.3 GPU + - NVIDIA Maxwell generation CC 5.3 * - PASCAL60 - GPU - - NVIDIA Pascal generation CC 6.0 GPU + - NVIDIA Pascal generation CC 6.0 * - PASCAL61 - GPU - - NVIDIA Pascal generation CC 6.1 GPU + - NVIDIA Pascal generation CC 6.1 * - VOLTA70 - GPU - - NVIDIA Volta generation CC 7.0 GPU + - NVIDIA Volta generation CC 7.0 * - VOLTA72 - GPU - - NVIDIA Volta generation CC 7.2 GPU + - NVIDIA Volta generation CC 7.2 * - TURING75 - GPU - - NVIDIA Turing generation CC 7.5 GPU + - NVIDIA Turing generation CC 7.5 * - AMPERE80 - GPU - - NVIDIA Ampere generation CC 8.0 GPU + - NVIDIA Ampere generation CC 8.0 * - AMPERE86 - GPU - - NVIDIA Ampere generation CC 8.6 GPU + - NVIDIA Ampere generation CC 8.6 * - ADA89 - GPU - - NVIDIA Ada Lovelace generation CC 8.9 GPU + - NVIDIA Ada generation CC 8.9 * - HOPPER90 - GPU - - NVIDIA Hopper generation CC 9.0 GPU + - NVIDIA Hopper generation CC 9.0 * - AMD_GFX906 - GPU - - AMD GPU MI50/MI60 + - AMD GPU MI50/60 * - AMD_GFX908 - GPU - AMD GPU MI100 * - AMD_GFX90A - GPU - AMD GPU MI200 + * - AMD_GFX940 + - GPU + - AMD GPU MI300 * - AMD_GFX942 - GPU - AMD GPU MI300 + * - AMD_GFX942_APU + - GPU + - AMD APU MI300A * - AMD_GFX1030 - GPU - AMD GPU V620/W6800 @@ -678,7 +692,7 @@ They must be specified in uppercase. - AMD GPU RX7900XTX * - AMD_GFX1103 - GPU - - AMD Phoenix APU with Radeon 740M/760M/780M/880M/890M + - AMD APU Phoenix * - INTEL_GEN - GPU - SPIR64-based devices, e.g. Intel GPUs, using JIT @@ -701,7 +715,7 @@ They must be specified in uppercase. - GPU - Intel GPU Ponte Vecchio -This list was last updated for version 4.3.0 of the Kokkos library. +This list was last updated for version 4.5.1 of the Kokkos library. .. tabs:: @@ -2018,7 +2032,7 @@ TBB and MKL. .. _mdi: MDI package ------------------------------ +----------- .. tabs:: @@ -2045,6 +2059,37 @@ MDI package ---------- +.. _misc: + +MISC package +------------ + +The :doc:`fix imd ` style in this package can be run either +synchronously (communication with IMD clients is done in the main +process) or asynchronously (the fix spawns a separate thread that can +communicate with IMD clients concurrently to the LAMMPS execution). + +.. tabs:: + + .. tab:: CMake build + + .. code-block:: bash + + -D LAMMPS_ASYNC_IMD=value # Run IMD server asynchronously + # value = no (default) or yes + + .. tab:: Traditional make + + To enable asynchronous mode the ``-DLAMMPS_ASYNC_IMD`` define + needs to be added to the ``LMP_INC`` variable in the + ``Makefile.machine`` you are using. For example: + + .. code-block:: make + + LMP_INC = -DLAMMPS_ASYNC_IMD -DLAMMPS_MEMALIGN=64 + +---------- + .. _molfile: MOLFILE package @@ -2191,7 +2236,7 @@ verified to work in February 2020 with Quantum Espresso versions 6.3 to from the sources in the *lib* folder (including the essential libqmmm.a) are not included in the static LAMMPS library and (currently) not installed, while their code is included in the - shared LAMMPS library. Thus a typical command line to configure + shared LAMMPS library. Thus a typical command to configure building LAMMPS for QMMM would be: .. code-block:: bash diff --git a/doc/src/Build_make.rst b/doc/src/Build_make.rst index 932050d410..00f2f0b24d 100644 --- a/doc/src/Build_make.rst +++ b/doc/src/Build_make.rst @@ -8,6 +8,10 @@ Building LAMMPS with traditional makefiles requires that you have a for customizing your LAMMPS build with a number of global compilation options and features. +This build system is slowly being phased out and may not support all +optional features and packages in LAMMPS. It is recommended to switch +to the :doc:`CMake based build system `. + Requirements ^^^^^^^^^^^^ diff --git a/doc/src/Build_package.rst b/doc/src/Build_package.rst index 8b2da8ad30..c4c4889806 100644 --- a/doc/src/Build_package.rst +++ b/doc/src/Build_package.rst @@ -49,6 +49,7 @@ packages: * :ref:`LEPTON ` * :ref:`MACHDYN ` * :ref:`MDI ` + * :ref:`MISC ` * :ref:`ML-HDNNP ` * :ref:`ML-IAP ` * :ref:`ML-PACE ` diff --git a/doc/src/Build_windows.rst b/doc/src/Build_windows.rst index d1748825eb..27562db480 100644 --- a/doc/src/Build_windows.rst +++ b/doc/src/Build_windows.rst @@ -100,9 +100,9 @@ procedure. It is possible to use both the integrated CMake support of the Visual Studio IDE or use an external CMake installation (e.g. downloaded from -cmake.org) to create build files and compile LAMMPS from the command line. +cmake.org) to create build files and compile LAMMPS from the command-line. -Compilation via command line and unit tests are checked automatically +Compilation via command-line and unit tests are checked automatically for the LAMMPS development branch through `GitHub Actions `_. @@ -115,7 +115,7 @@ for the LAMMPS development branch through Please note, that for either approach CMake will create a so-called :ref:`"multi-configuration" build environment `, and -the command lines for building and testing LAMMPS must be adjusted +the commands for building and testing LAMMPS must be adjusted accordingly. The LAMMPS cmake folder contains a ``CMakeSettings.json`` file with diff --git a/doc/src/Classes_lammps.rst b/doc/src/Classes_lammps.rst index dd6798d9a1..167a619fae 100644 --- a/doc/src/Classes_lammps.rst +++ b/doc/src/Classes_lammps.rst @@ -4,7 +4,7 @@ LAMMPS Class The LAMMPS class is encapsulating an MD simulation state and thus it is the class that needs to be created when starting a new simulation system state. The LAMMPS executable essentially creates one instance of this -class and passes the command line flags and tells it to process the +class and passes the command-line flags and tells it to process the provided input (a file or ``stdin``). It shuts the class down when control is returned to it and then exits. When using LAMMPS as a library from another code it is required to create an instance of this diff --git a/doc/src/Commands_bond.rst b/doc/src/Commands_bond.rst index 3be40a6158..40532bdef7 100644 --- a/doc/src/Commands_bond.rst +++ b/doc/src/Commands_bond.rst @@ -90,6 +90,7 @@ OPT. * :doc:`lepton (o) ` * :doc:`mesocnt ` * :doc:`mm3 ` + * :doc:`mwlc ` * :doc:`quartic (o) ` * :doc:`spica (ko) ` * :doc:`table (o) ` diff --git a/doc/src/Commands_compute.rst b/doc/src/Commands_compute.rst index fd68ce3e39..7c73583a4f 100644 --- a/doc/src/Commands_compute.rst +++ b/doc/src/Commands_compute.rst @@ -58,6 +58,7 @@ KOKKOS, o = OPENMP, t = OPT. * :doc:`fep/ta ` * :doc:`force/tally ` * :doc:`fragment/atom ` + * :doc:`gaussian/grid/local (k) ` * :doc:`global/atom ` * :doc:`group/group ` * :doc:`gyration ` @@ -140,8 +141,8 @@ KOKKOS, o = OPENMP, t = OPT. * :doc:`smd/vol ` * :doc:`snap ` * :doc:`sna/atom ` - * :doc:`sna/grid ` - * :doc:`sna/grid/local ` + * :doc:`sna/grid (k) ` + * :doc:`sna/grid/local (k) ` * :doc:`snad/atom ` * :doc:`snav/atom ` * :doc:`sph/e/atom ` diff --git a/doc/src/Commands_fix.rst b/doc/src/Commands_fix.rst index 04d1a9969a..6192029a0a 100644 --- a/doc/src/Commands_fix.rst +++ b/doc/src/Commands_fix.rst @@ -58,6 +58,7 @@ OPT. * :doc:`dt/reset (k) ` * :doc:`edpd/source ` * :doc:`efield (k) ` + * :doc:`efield/lepton ` * :doc:`efield/tip4p ` * :doc:`ehex ` * :doc:`electrode/conp (i) ` diff --git a/doc/src/Commands_input.rst b/doc/src/Commands_input.rst index dc0fb72dd9..0f6e8f94a2 100644 --- a/doc/src/Commands_input.rst +++ b/doc/src/Commands_input.rst @@ -69,7 +69,7 @@ WARNING message is printed. The :doc:`Errors ` page gives more information on what errors mean. The documentation for each command lists restrictions on how the command can be used. -You can use the :ref:`-skiprun ` command line flag +You can use the :ref:`-skiprun ` command-line flag to have LAMMPS skip the execution of any ``run``, ``minimize``, or similar commands to check the entire input for correct syntax to avoid crashes on typos or syntax errors in long runs. diff --git a/doc/src/Commands_pair.rst b/doc/src/Commands_pair.rst index 5f42f53bd5..048a54ed37 100644 --- a/doc/src/Commands_pair.rst +++ b/doc/src/Commands_pair.rst @@ -80,6 +80,7 @@ OPT. * :doc:`coul/tt ` * :doc:`coul/wolf (ko) ` * :doc:`coul/wolf/cs ` + * :doc:`dispersion/d3 ` * :doc:`dpd (giko) ` * :doc:`dpd/coul/slater/long (g) ` * :doc:`dpd/ext (ko) ` @@ -114,7 +115,9 @@ OPT. * :doc:`gw/zbl ` * :doc:`harmonic/cut (o) ` * :doc:`hbond/dreiding/lj (o) ` + * :doc:`hbond/dreiding/lj/angleoffset (o) ` * :doc:`hbond/dreiding/morse (o) ` + * :doc:`hbond/dreiding/morse/angleoffset (o) ` * :doc:`hdnnp ` * :doc:`hippo (g) ` * :doc:`ilp/graphene/hbn (t) ` diff --git a/doc/src/Commands_removed.rst b/doc/src/Commands_removed.rst index ea8b3d4b03..be775f4c19 100644 --- a/doc/src/Commands_removed.rst +++ b/doc/src/Commands_removed.rst @@ -1,6 +1,10 @@ Removed commands and packages ============================= +.. contents:: + +------ + This page lists LAMMPS commands and packages that have been removed from the distribution and provides suggestions for alternatives or replacements. LAMMPS has special dummy styles implemented, that will @@ -8,47 +12,60 @@ stop LAMMPS and print a suitable error message in most cases, when a style/command is used that has been removed or will replace the command with the direct alternative (if available) and print a warning. -restart2data tool ------------------ +LAMMPS shell +------------ -.. versionchanged:: 23Nov2013 +.. versionchanged:: 29Aug2024 -The functionality of the restart2data tool has been folded into the -LAMMPS executable directly instead of having a separate tool. A -combination of the commands :doc:`read_restart ` and -:doc:`write_data ` can be used to the same effect. For -added convenience this conversion can also be triggered by -:doc:`command line flags ` +The LAMMPS shell has been removed from the LAMMPS distribution. Users +are encouraged to use the :ref:`LAMMPS-GUI ` tool instead. -Fix ave/spatial and fix ave/spatial/sphere ------------------------------------------- +i-PI tool +--------- -.. deprecated:: 11Dec2015 +.. versionchanged:: 27Jun2024 -The fixes ave/spatial and ave/spatial/sphere have been removed from LAMMPS -since they were superseded by the more general and extensible "chunk -infrastructure". Here the system is partitioned in one of many possible -ways through the :doc:`compute chunk/atom ` command -and then averaging is done using :doc:`fix ave/chunk `. -Please refer to the :doc:`chunk HOWTO ` section for an overview. +The i-PI tool has been removed from the LAMMPS distribution. Instead, +instructions to install i-PI from PyPI via pip are provided. -Box command ------------ +USER-REAXC package +------------------ -.. deprecated:: 22Dec2022 +.. deprecated:: 7Feb2024 -The *box* command has been removed and the LAMMPS code changed so it won't -be needed. If present, LAMMPS will ignore the command and print a warning. +The USER-REAXC package has been renamed to :ref:`REAXFF `. +In the process also the pair style and related fixes were renamed to use +the "reaxff" string instead of "reax/c". For a while LAMMPS was maintaining +backward compatibility by providing aliases for the styles. These have +been removed, so using "reaxff" is now *required*. -Reset_ids, reset_atom_ids, reset_mol_ids commands -------------------------------------------------- +MPIIO package +------------- -.. deprecated:: 22Dec2022 +.. deprecated:: 21Nov2023 -The *reset_ids*, *reset_atom_ids*, and *reset_mol_ids* commands have -been folded into the :doc:`reset_atoms ` command. If -present, LAMMPS will replace the commands accordingly and print a -warning. +The MPIIO package has been removed from LAMMPS since it was unmaintained +for many years and thus not updated to incorporate required changes that +had been applied to the corresponding non-MPIIO commands. As a +consequence the MPIIO commands had become unreliable and sometimes +crashing LAMMPS or corrupting data. Similar functionality is available +through the :ref:`ADIOS package ` and the :ref:`NETCDF +package `. Also, the :doc:`dump_modify nfile or dump_modify +fileper ` keywords may be used for an efficient way of +writing out dump files when running on large numbers of processors. +Similarly, the "nfile" and "fileper" keywords exist for restarts: +see :doc:`restart `, :doc:`read_restart `, +:doc:`write_restart `. + +MSCG package +------------ + +.. deprecated:: 21Nov2023 + +The MSCG package has been removed from LAMMPS since it was unmaintained +for many years and instead superseded by the `OpenMSCG software +`_ of the Voth group at the +University of Chicago, which can be used independent from LAMMPS. LATTE package ------------- @@ -64,18 +81,6 @@ packages, including LATTE. See the ``examples/QUANTUM`` dir and the with LATTE as a plugin library (similar to the way fix latte worked), as well as on a different set of MPI processors. -MEAM package ------------- - -The MEAM package in Fortran has been replaced by a C++ implementation. -The code in the :ref:`MEAM package ` is a translation of the -Fortran code of MEAM into C++, which removes several restrictions -(e.g. there can be multiple instances in hybrid pair styles) and allows -for some optimizations leading to better performance. The pair style -:doc:`meam ` has the exact same syntax. For a transition -period the C++ version of MEAM was called USER-MEAMC so it could -coexist with the Fortran version. - Minimize style fire/old ----------------------- @@ -97,38 +102,38 @@ The same functionality is available through :doc:`bond style mesocnt ` and :doc:`angle style mesocnt `. -MPIIO package -------------- +Box command +----------- -.. deprecated:: 21Nov2023 +.. deprecated:: 22Dec2022 -The MPIIO package has been removed from LAMMPS since it was unmaintained -for many years and thus not updated to incorporate required changes that -had been applied to the corresponding non-MPIIO commands. As a -consequence the MPIIO commands had become unreliable and sometimes -crashing LAMMPS or corrupting data. Similar functionality is available -through the :ref:`ADIOS package ` and the :ref:`NETCDF -package `. Also, the :doc:`dump_modify nfile or dump_modify -fileper ` keywords may be used for an efficient way of -writing out dump files when running on large numbers of processors. -Similarly, the "nfile" and "fileper" keywords exist for restarts: -see :doc:`restart `, :doc:`read_restart `, -:doc:`write_restart `. +The *box* command has been removed and the LAMMPS code changed so it won't +be needed. If present, LAMMPS will ignore the command and print a warning. +Reset_ids, reset_atom_ids, reset_mol_ids commands +------------------------------------------------- -MSCG package ------------- +.. deprecated:: 22Dec2022 -.. deprecated:: 21Nov2023 +The *reset_ids*, *reset_atom_ids*, and *reset_mol_ids* commands have +been folded into the :doc:`reset_atoms ` command. If +present, LAMMPS will replace the commands accordingly and print a +warning. -The MSCG package has been removed from LAMMPS since it was unmaintained -for many years and instead superseded by the `OpenMSCG software -`_ of the Voth group at the -University of Chicago, which can be used independent from LAMMPS. +MESSAGE package +--------------- + +.. deprecated:: 4May2022 + +The MESSAGE package has been removed since it was superseded by the +:ref:`MDI package `. MDI implements the same functionality +and in a more general way with direct support for more applications. REAX package ------------ +.. deprecated:: 4Jan2019 + The REAX package has been removed since it was superseded by the :ref:`REAXFF package `. The REAXFF package has been tested to yield equivalent results to the REAX package, offers better @@ -138,20 +143,25 @@ syntax compatible with the removed reax pair style, so input files will have to be adapted. The REAXFF package was originally called USER-REAXC. -USER-REAXC package ------------------- +MEAM package +------------ -.. deprecated:: 7Feb2024 +.. deprecated:: 4Jan2019 -The USER-REAXC package has been renamed to :ref:`REAXFF `. -In the process also the pair style and related fixes were renamed to use -the "reaxff" string instead of "reax/c". For a while LAMMPS was maintaining -backward compatibility by providing aliases for the styles. These have -been removed, so using "reaxff" is now *required*. +The MEAM package in Fortran has been replaced by a C++ implementation. +The code in the :ref:`MEAM package ` is a translation of the +Fortran code of MEAM into C++, which removes several restrictions +(e.g. there can be multiple instances in hybrid pair styles) and allows +for some optimizations leading to better performance. The pair style +:doc:`meam ` has the exact same syntax. For a transition +period the C++ version of MEAM was called USER-MEAMC so it could +coexist with the Fortran version. USER-CUDA package ----------------- +.. deprecated:: 31May2016 + The USER-CUDA package had been removed, since it had been unmaintained for a long time and had known bugs and problems. Significant parts of the design were transferred to the @@ -160,19 +170,27 @@ performance characteristics on NVIDIA GPUs. Both, the KOKKOS and the :ref:`GPU package ` are maintained and allow running LAMMPS with GPU acceleration. -i-PI tool ---------- +Fix ave/spatial and fix ave/spatial/sphere +------------------------------------------ -.. versionchanged:: 27Jun2024 +.. deprecated:: 11Dec2015 -The i-PI tool has been removed from the LAMMPS distribution. Instead, -instructions to install i-PI from PyPI via pip are provided. +The fixes ave/spatial and ave/spatial/sphere have been removed from LAMMPS +since they were superseded by the more general and extensible "chunk +infrastructure". Here the system is partitioned in one of many possible +ways through the :doc:`compute chunk/atom ` command +and then averaging is done using :doc:`fix ave/chunk `. +Please refer to the :doc:`chunk HOWTO ` section for an overview. -LAMMPS shell ------------- +restart2data tool +----------------- -.. versionchanged:: 29Aug2024 +.. deprecated:: 23Nov2013 -The LAMMPS shell has been removed from the LAMMPS distribution. Users -are encouraged to use the :ref:`LAMMPS-GUI ` tool instead. +The functionality of the restart2data tool has been folded into the +LAMMPS executable directly instead of having a separate tool. A +combination of the commands :doc:`read_restart ` and +:doc:`write_data ` can be used to the same effect. For +added convenience this conversion can also be triggered by +:doc:`command-line flags ` diff --git a/doc/src/Developer_code_design.rst b/doc/src/Developer_code_design.rst index 974266ec7f..9213efa18f 100644 --- a/doc/src/Developer_code_design.rst +++ b/doc/src/Developer_code_design.rst @@ -203,6 +203,7 @@ processed in the expected order before types are removed from dynamic dispatch. .. admonition:: Important Notes + :class: note In order to be able to detect incompatibilities at compile time and to avoid unexpected behavior, it is crucial that all member functions @@ -300,18 +301,24 @@ Formatting with the {fmt} library The LAMMPS source code includes a copy of the `{fmt} library `_, which is preferred over formatting with the -"printf()" family of functions. The primary reason is that it allows -a typesafe default format for any type of supported data. This is +"printf()" family of functions. The primary reason is that it allows a +typesafe default format for any type of supported data. This is particularly useful for formatting integers of a given size (32-bit or -64-bit) which may require different format strings depending on -compile time settings or compilers/operating systems. Furthermore, -{fmt} gives better performance, has more functionality, a familiar -formatting syntax that has similarities to ``format()`` in Python, and -provides a facility that can be used to integrate format strings and a -variable number of arguments into custom functions in a much simpler -way than the varargs mechanism of the C library. Finally, {fmt} has -been included into the C++20 language standard, so changes to adopt it -are future-proof. +64-bit) which may require different format strings depending on compile +time settings or compilers/operating systems. Furthermore, {fmt} gives +better performance, has more functionality, a familiar formatting syntax +that has similarities to ``format()`` in Python, and provides a facility +that can be used to integrate format strings and a variable number of +arguments into custom functions in a much simpler way than the varargs +mechanism of the C library. Finally, {fmt} has been included into the +C++20 language standard as ``std::format()``, so changes to adopt it are +future-proof, for as long as they are not using any extensions that are +not (yet) included into C++. + +The long-term plan is to switch to using ``std::format()`` instead of +``fmt::format()`` when the minimum C++ standard required for LAMMPS will +be set to C++20. See the :ref:`basic build instructions ` for +more details. Formatted strings are frequently created by calling the ``fmt::format()`` function, which will return a string as a @@ -319,11 +326,13 @@ Formatted strings are frequently created by calling the ``printf()``, the {fmt} library uses ``{}`` to embed format descriptors. In the simplest case, no additional characters are needed, as {fmt} will choose the default format based on the data type of the argument. -Otherwise, the ``fmt::print()`` function may be used instead of -``printf()`` or ``fprintf()``. In addition, several LAMMPS output -functions, that originally accepted a single string as argument have -been overloaded to accept a format string with optional arguments as -well (e.g., ``Error::all()``, ``Error::one()``, ``utils::logmesg()``). +Otherwise, the :cpp:func:`utils::print() ` +function may be used instead of ``printf()`` or ``fprintf()``. In +addition, several LAMMPS output functions, that originally accepted a +single string as argument have been overloaded to accept a format string +with optional arguments as well (e.g., ``Error::all()``, +``Error::one()``, :cpp:func:`utils::logmesg() +`). Summary of the {fmt} format syntax ================================== diff --git a/doc/src/Developer_flow.rst b/doc/src/Developer_flow.rst index 17d75879ca..5efbfaa51d 100644 --- a/doc/src/Developer_flow.rst +++ b/doc/src/Developer_flow.rst @@ -209,7 +209,7 @@ nve, nvt, npt. At the end of the timestep, fixes that contain an ``end_of_step()`` method are invoked. These typically perform a diagnostic calculation, -e.g. the ave/time and ave/spatial fixes. The final operation of the +e.g. the ave/time and ave/chunk fixes. The final operation of the timestep is to perform any requested output, via the ``write()`` method of the Output class. There are 3 kinds of LAMMPS output: thermodynamic output to the screen and log file, snapshots of atom data to a dump diff --git a/doc/src/Developer_notes.rst b/doc/src/Developer_notes.rst index af26b4b913..2d136055a4 100644 --- a/doc/src/Developer_notes.rst +++ b/doc/src/Developer_notes.rst @@ -7,13 +7,7 @@ typically document what a variable stores, what a small section of code does, or what a function does and its input/outputs. The topics on this page are intended to document code functionality at a higher level. -Available topics are: - -- `Reading and parsing of text and text files`_ -- `Requesting and accessing neighbor lists`_ -- `Choosing between a custom atom style, fix property/atom, and fix STORE/ATOM`_ -- `Fix contributions to instantaneous energy, virial, and cumulative energy`_ -- `KSpace PPPM FFT grids`_ +.. contents:: ---- @@ -218,6 +212,146 @@ command: neighbor->add_request(this, "delete_atoms", NeighConst::REQ_FULL); + +Errors, warnings, and informational messages +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +LAMMPS has specialized functionality to handle errors (which should +terminate LAMMPS), warning messages (which should indicate possible +problems *without* terminating LAMMPS), and informational text for +messages about the progress and chosen settings. We *strongly* +encourage using these facilities and to *stay away* from using +``printf()`` or ``fprintf()`` or ``std::cout`` or ``std::cerr`` and +calling ``MPI_Abort()`` or ``exit()`` directly. Warnings and +informational messages should be printed only on MPI rank 0 to avoid +flooding the output when running in parallel with many MPI processes. + +**Errors** + +When LAMMPS encounters an error, for example a syntax error in the +input, then a suitable error message should be printed giving a brief, +one line remark about the reason and then call either ``Error::all()`` +or ``Error::one()``. ``Error::all()`` must be called when the failing +code path is executed by *all* MPI processes and the error condition +will appear for *all* MPI processes the same. If desired, each MPI +process may set a flag to either 0 or 1 and then MPI_Allreduce() +searching for the maximum can be used to determine if there was an error +on *any* of the MPI processes and make this information available to +*all*. ``Error::one()`` in contrast needs to be called when only one or +a few MPI processes execute the code path or can have the error +condition. ``Error::all()`` is generally the preferred option. + +Calling these functions does not abort LAMMPS directly, but rather +throws either a ``LAMMPSException`` (from ``Error::all()``) or a +``LAMMPSAbortException`` (from ``Error::one()``). These exceptions are +caught by the LAMMPS ``main()`` program and then handled accordingly. +The reason for this approach is to support applications, especially +graphical applications like :ref:`LAMMPS-GUI `, that are +linked to the LAMMPS library and have a mechanism to avoid that an error +in LAMMPS terminates the application. By catching the exceptions, the +application can delete the failing LAMMPS class instance and create a +new one to try again. In a similar fashion, the :doc:`LAMMPS Python +module ` checks for this and then re-throws corresponding +Python exception, which in turn can be caught by the calling Python +code. + +There are multiple "signatures" that can be called: + +- ``Error::all(FLERR, "Error message")``: this will abort LAMMPS with + the error message "Error message", followed by the last line of input + that was read and processed before the error condition happened. + +- ``Error::all(FLERR, Error::NOLASTLINE, "Error message")``: this is the + same as before but without the last line of input. This is preferred + for errors that would happen *during* a :doc:`run ` or + :doc:`minimization `, since showing the "run" or "minimize" + command would be the last line, but is unrelated to the error. + +- ``Error::all(FLERR, idx, "Error message")``: this is for argument + parsing where "idx" is the index (starting at 0) of the argument for a + LAMMPS command that is causing the failure (use -1 for the command + itself). The output may also include the last input line *before* and + *after*, if they differ due to substituting variables. A textual + indicator is pointing to the specific word that failed. Using the + constant ``Error::NOPOINTER`` in place of the *idx* argument will + suppress the marker and then the behavior is like the *idx* argument + is not provided. + +FLERR is a macro containing the filename and line where the Error class +is called and that information is appended to the error message. This +allows to quickly find the relevant source code causing the error. For +all three signatures, the single string "Error message" may be replaced +with a format string using '{}' placeholders and followed by a variable +number of arguments, one for each placeholder. This format string and +the arguments are then handed for formatting to the `{fmt} library +`_ (which is bundled with LAMMPS) and thus allow +processing similar to the "format()" functionality in Python. + +.. note:: + + For commands like :doc:`fix ave/time ` that accept + wildcard arguments, the :cpp:func:`utils::expand_args` function + may be passed as an optional argument where the function will provide + a map to the original arguments from the expanded argument indices. + +For complex errors, that can have multiple causes and which cannot be +explained in a single line, you can append to the error message, the +string created by :cpp:func:`utils::errorurl`, which then provides a +URL pointing to a paragraph of the :doc:`Errors_details` that +corresponds to the number provided. Example: + +.. code-block:: c++ + + error->all(FLERR, "Unknown identifier in data file: {}{}", keyword, utils::errorurl(1)); + +This will output something like this: + +.. parsed-literal:: + + ERROR: Unknown identifier in data file: Massess + For more information see https://docs.lammps.org/err0001 (src/read_data.cpp:1482) + Last input line: read_data data.peptide + +Where the URL points to the first paragraph with explanations on +the :doc:`Errors_details` page in the manual. + +**Warnings** + +To print warnings, the ``Errors::warning()`` function should be used. +It also requires the FLERR macros as first argument to easily identify +the location of the warning in the source code. Same as with the error +functions above, the function has two variants: one just taking a single +string as final argument and a second that uses the `{fmt} library +`_ to make it similar to, say, ``fprintf()``. One +motivation to use this function is that it will output warnings with +always the same capitalization of the leading "WARNING" string. A +second is that it has a built in rate limiter. After a given number (by +default 100), that can be set via the :doc:`thermo_modify command +` no more warnings are printed. Also, warnings are +written consistently to both screen and logfile or not, depending on the +settings for :ref:`screen ` or :doc:`logfile ` output. + +.. note:: + + Unlike ``Error::all()``, the warning function will produce output on + *every* MPI process, so it typically would be prefixed with an if + statement testing for ``comm->me == 0``, i.e. limiting output to MPI + rank 0. + +**Informational messages** + +Finally, for informational message LAMMPS has the +:cpp:func:`utils::logmesg() convenience function +`. It also uses the `{fmt} library +`_ to support using a format string followed by a +matching number of arguments. It will output the resulting formatted +text to both, the screen and the logfile and will honor the +corresponding settings about whether this output is active and to which +file it should be send. Same as for ``Error::warning()``, it would +produce output for every MPI process and thus should usually be called +only on MPI rank 0 to avoid flooding the output when running with many +parallel processes. + Choosing between a custom atom style, fix property/atom, and fix STORE/ATOM ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ diff --git a/doc/src/Developer_org.rst b/doc/src/Developer_org.rst index f1804655ae..93632ae03f 100644 --- a/doc/src/Developer_org.rst +++ b/doc/src/Developer_org.rst @@ -94,12 +94,12 @@ represents what is generally referred to as an "instance of LAMMPS". It is a composite holding pointers to instances of other core classes providing the core functionality of the MD engine in LAMMPS and through them abstractions of the required operations. The constructor of the -LAMMPS class will instantiate those instances, process the command line +LAMMPS class will instantiate those instances, process the command-line flags, initialize MPI (if not already done) and set up file pointers for input and output. The destructor will shut everything down and free all associated memory. Thus code for the standalone LAMMPS executable in ``main.cpp`` simply initializes MPI, instantiates a single instance of -LAMMPS while passing it the command line flags and input script. It +LAMMPS while passing it the command-line flags and input script. It deletes the LAMMPS instance after the method reading the input returns and shuts down the MPI environment before it exits the executable. diff --git a/doc/src/Developer_unittest.rst b/doc/src/Developer_unittest.rst index 07ffb0e2d8..bee0c45480 100644 --- a/doc/src/Developer_unittest.rst +++ b/doc/src/Developer_unittest.rst @@ -227,12 +227,12 @@ Tests for the C-style library interface ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Tests for validating the LAMMPS C-style library interface are in the -``unittest/c-library`` folder. They are implemented either to be used -for utility functions or for LAMMPS commands, but use the functions -implemented in the ``src/library.cpp`` file as much as possible. There -may be some overlap with other tests, but only in as much as is required -to test the C-style library API. The tests are distributed over -multiple test programs which try to match the grouping of the +``unittest/c-library`` folder. They text either utility functions or +LAMMPS commands, but use the functions implemented in +``src/library.cpp`` as much as possible. There may be some overlap with +other tests as far as the LAMMPS functionality is concerned, but the +focus is on testing the C-style library API. The tests are distributed +over multiple test programs which try to match the grouping of the functions in the source code and :ref:`in the manual `. This group of tests also includes tests invoking LAMMPS in parallel @@ -258,7 +258,7 @@ Tests for the Python module and package The ``unittest/python`` folder contains primarily tests for classes and functions in the LAMMPS python module but also for commands in the -PYTHON package. These tests are only enabled if the necessary +PYTHON package. These tests are only enabled, if the necessary prerequisites are detected or enabled during configuration and compilation of LAMMPS (shared library build enabled, Python interpreter found, Python development files found). @@ -272,29 +272,30 @@ Tests for the Fortran interface Tests for using the Fortran module are in the ``unittest/fortran`` folder. Since they are also using the GoogleTest library, they require -implementing test wrappers in C++ that will call fortran functions -which provide a C function interface through ISO_C_BINDINGS that will in -turn call the functions in the LAMMPS Fortran module. +test wrappers written in C++ that will call fortran functions with a C +function interface through ISO_C_BINDINGS which will in turn call the +functions in the LAMMPS Fortran module. Tests for the C++-style library interface ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ The tests in the ``unittest/cplusplus`` folder are somewhat similar to the tests for the C-style library interface, but do not need to test the -several convenience and utility functions that are only available through -the C-style interface. Instead it can focus on the more generic features -that are used internally. This part of the unit tests is currently still -mostly in the planning stage. +convenience and utility functions that are only available through the +C-style library interface. Instead they focus on the more generic +features that are used in LAMMPS internally. This part of the unit +tests is currently still mostly in the planning stage. Tests for reading and writing file formats ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ The ``unittest/formats`` folder contains test programs for reading and -writing files like data files, restart files, potential files or dump files. -This covers simple things like the file i/o convenience functions in the -``utils::`` namespace to complex tests of atom styles where creating and -deleting atoms with different properties is tested in different ways -and through script commands or reading and writing of data or restart files. +writing files like data files, restart files, potential files or dump +files. This covers simple things like the file i/o convenience +functions in the ``utils::`` namespace to complex tests of atom styles +where creating and deleting of atoms with different properties is tested +in different ways and through script commands or reading and writing of +data or restart files. Tests for styles computing or modifying forces ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -443,7 +444,7 @@ file for a style that is similar to one to be tested. The file name should follow the naming conventions described above and after copying the file, the first step is to replace the style names where needed. The coefficient values do not have to be meaningful, just in a reasonable range for the -given system. It does not matter if some forces are large, as long as +given system. It does not matter if some forces are large, for as long as they do not diverge. The template input files define a large number of index variables at the top @@ -476,7 +477,7 @@ the tabulated coulomb, to test both code paths. The reference results in the YA files then should be compared manually, if they agree well enough within the limits of those two approximations. -The ``test_pair_style`` and equivalent programs have special command line options +The ``test_pair_style`` and equivalent programs have special command-line options to update the YAML files. Running a command like .. code-block:: bash @@ -531,19 +532,20 @@ Python module. Troubleshooting failed unit tests ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -The are by default no unit tests for newly added features (e.g. pair, fix, -or compute styles) unless your pull request also includes tests for the -added features. If you are modifying some features, you may see failures -for existing tests, if your modifications have some unexpected side effects -or your changes render the existing test invalid. If you are adding an -accelerated version of an existing style, then only tests for INTEL, -KOKKOS (with OpenMP only), OPENMP, and OPT will be run automatically. -Tests for the GPU package are time consuming and thus are only run -*after* a merge, or when a special label, ``gpu_unit_tests`` is added -to the pull request. After the test has started, it is often best to -remove the label since every PR activity will re-trigger the test (that -is a limitation of triggering a test with a label). Support for unit -tests when using KOKKOS with GPU acceleration is currently not supported. +There are by default no unit tests for newly added features (e.g. pair, +fix, or compute styles) unless your pull request also includes tests for +these added features. If you are modifying some existing LAMMPS +features, you may see failures for existing tests, if your modifications +have some unexpected side effects or your changes render the existing +test invalid. If you are adding an accelerated version of an existing +style, then only tests for INTEL, KOKKOS (with OpenMP only), OPENMP, and +OPT will be run automatically. Tests for the GPU package are time +consuming and thus are only run *after* a merge, or when a special +label, ``gpu_unit_tests`` is added to the pull request. After the test +has started, it is often best to remove the label since every PR +activity will re-trigger the test (that is a limitation of triggering a +test with a label). Support for unit tests using KOKKOS with GPU +acceleration is currently not supported. When you see a failed build on GitHub, click on ``Details`` to be taken to the corresponding LAMMPS Jenkins CI web page. Click on the "Exit" @@ -588,7 +590,7 @@ While the epsilon (relative precision) for a single, `IEEE 754 compliant `_, double precision floating point operation is at about 2.2e-16, the achievable precision for the tests is lower due to most numbers being sums over intermediate results -and the non-associativity of floating point math leading to larger +for which the non-associativity of floating point math leads to larger errors. As a rule of thumb, the test epsilon can often be in the range 5.0e-14 to 1.0e-13. But for "noisy" force kernels, e.g. those a larger amount of arithmetic operations involving `exp()`, `log()` or `sin()` @@ -602,14 +604,14 @@ of floating point operations or that some or most intermediate operations may be done using approximations or with single precision floating point math. -To rerun the failed unit test individually, change to the ``build`` directory +To rerun a failed unit test individually, change to the ``build`` directory and run the test with verbose output. For example, .. code-block:: bash env TEST_ARGS=-v ctest -R ^MolPairStyle:lj_cut_coul_long -V -``ctest`` with the ``-V`` flag also shows the exact command line +``ctest`` with the ``-V`` flag also shows the exact command of the test. One can then use ``gdb --args`` to further debug and catch exceptions with the test command, for example, diff --git a/doc/src/Developer_utils.rst b/doc/src/Developer_utils.rst index b7f62d5364..866945fc88 100644 --- a/doc/src/Developer_utils.rst +++ b/doc/src/Developer_utils.rst @@ -133,6 +133,9 @@ and parsing files or arguments. .. doxygenfunction:: trim_comment :project: progguide +.. doxygenfunction:: strcompress + :project: progguide + .. doxygenfunction:: strip_style_suffix :project: progguide @@ -166,6 +169,9 @@ and parsing files or arguments. .. doxygenfunction:: split_lines :project: progguide +.. doxygenfunction:: strsame + :project: progguide + .. doxygenfunction:: strmatch :project: progguide @@ -232,12 +238,21 @@ Convenience functions .. doxygenfunction:: logmesg(LAMMPS *lmp, const std::string &mesg) :project: progguide +.. doxygenfunction:: print(FILE *fp, const std::string &format, Args&&... args) + :project: progguide + +.. doxygenfunction:: print(FILE *fp, const std::string &mesg) + :project: progguide + .. doxygenfunction:: errorurl :project: progguide .. doxygenfunction:: missing_cmd_args :project: progguide +.. doxygenfunction:: point_to_error + :project: progguide + .. doxygenfunction:: flush_buffers(LAMMPS *lmp) :project: progguide diff --git a/doc/src/Developer_write_fix.rst b/doc/src/Developer_write_fix.rst index afa569b05d..1b578823bd 100644 --- a/doc/src/Developer_write_fix.rst +++ b/doc/src/Developer_write_fix.rst @@ -96,8 +96,8 @@ Here the we specify which methods of the fix should be called during MPI_Allreduce(localAvgVel, globalAvgVel, 4, MPI_DOUBLE, MPI_SUM, world); scale3(1.0 / globalAvgVel[3], globalAvgVel); if ((comm->me == 0) && screen) { - fmt::print(screen,"{}, {}, {}\n", - globalAvgVel[0], globalAvgVel[1], globalAvgVel[2]); + utils::print(screen, "{}, {}, {}\n", + globalAvgVel[0], globalAvgVel[1], globalAvgVel[2]); } } diff --git a/doc/src/Developer_write_pair.rst b/doc/src/Developer_write_pair.rst index 5d5e081042..90b716d9c4 100644 --- a/doc/src/Developer_write_pair.rst +++ b/doc/src/Developer_write_pair.rst @@ -310,7 +310,7 @@ the constructor and the destructor. Pair styles are different from most classes in LAMMPS that define a "style", as their constructor only uses the LAMMPS class instance -pointer as an argument, but **not** the command line arguments of the +pointer as an argument, but **not** the arguments of the :doc:`pair_style command `. Instead, those arguments are processed in the ``Pair::settings()`` function (or rather the version in the derived class). The constructor is the place where global defaults @@ -891,7 +891,7 @@ originally created from mixing or not). These data file output functions are only useful for true pair-wise additive potentials, where the potential parameters can be entered through *multiple* :doc:`pair_coeff commands `. Pair styles -that require a single "pair_coeff \* \*" command line are not compatible +that require a single "pair_coeff \* \*" command are not compatible with reading their parameters from data files. For pair styles like *born/gauss* that do support writing to data files, the potential parameters will be read from the data file, if present, and @@ -1122,7 +1122,7 @@ once. Thus, the ``coeff()`` function has to do three tasks, each of which is delegated to a function in the ``PairTersoff`` class: #. map elements to atom types. Those follow the potential file name in the - command line arguments and are processed by the ``map_element2type()`` function. + command arguments and are processed by the ``map_element2type()`` function. #. read and parse the potential parameter file in the ``read_file()`` function. #. Build data structures where the original and derived parameters are indexed by all possible triples of atom types and thus can be looked @@ -1356,8 +1356,8 @@ either 0 or 1. The ``morseflag`` variable defaults to 0 and is set to 1 in the ``PairAIREBOMorse::settings()`` function which is called by the -:doc:`pair_style ` command. This function delegates -all command line processing and setting of other parameters to the +:doc:`pair_style ` command. This function delegates all +command argument processing and setting of other parameters to the ``PairAIREBO::settings()`` function of the base class. .. code-block:: c++ diff --git a/doc/src/Errors_debug.rst b/doc/src/Errors_debug.rst index b3f618dbf7..61fb1f7525 100644 --- a/doc/src/Errors_debug.rst +++ b/doc/src/Errors_debug.rst @@ -83,7 +83,7 @@ Run LAMMPS from within the debugger Running LAMMPS under the control of the debugger as shown below only works for a single MPI rank (for debugging a program running in parallel you usually need a parallel debugger program). A simple way to launch -GDB is to prefix the LAMMPS command line with ``gdb --args`` and then +GDB is to prefix the LAMMPS command-line with ``gdb --args`` and then type the command "run" at the GDB prompt. This will launch the debugger, load the LAMMPS executable and its debug info, and then run it. When it reaches the code causing the segmentation fault, it will @@ -180,7 +180,7 @@ inspect the behavior of a compiled program by essentially emulating a CPU and instrumenting the program while running. This slows down execution quite significantly, but can also report issues that are not resulting in a crash. The default valgrind tool is a memory checker and -you can use it by prefixing the normal command line with ``valgrind``. +you can use it by prefixing the normal command-line with ``valgrind``. Unlike GDB, this will also work for parallel execution, but it is recommended to redirect the valgrind output to a file (e.g. with ``--log-file=crash-%p.txt``, the %p will be substituted with the @@ -235,3 +235,53 @@ from GDB. In addition you get a more specific hint about what cause the segmentation fault, i.e. that it is a NULL pointer dereference. To find out which pointer exactly was NULL, you need to use the debugger, though. +Debugging when LAMMPS appears to be stuck +========================================= + +Sometimes the LAMMPS calculation appears to be stuck, that is the LAMMPS +process or processes are active, but there is no visible progress. This +can have multiple reasons: + +- The selected styles are slow and require a lot of CPU time and the + system is large. When extrapolating the expected speed from smaller + systems, one has to factor in that not all models scale linearly with + system size, e.g. :doc:`kspace styles like ewald or pppm + `. There is very little that can be done in this case. +- The output interval is not set or set to a large value with the + :doc:`thermo ` command. I the first case, there will be output + only at the first and last step. +- The output is block-buffered and instead of line-buffered. The output + will only be written to the screen after 4096 or 8192 characters of + output have accumulated. This most often happens for files but also + with MPI parallel executables for output to the screen, since the + output to the screen is handled by the MPI library so that output from + all processes can be shown. This can be suppressed by using the + ``-nonblock`` or ``-nb`` command-line flag, which turns off buffering + for screen and logfile output. +- An MPI parallel calculation has a bug where a collective MPI function + is called (e.g. ``MPI_Barrier()``, ``MPI_Bcast()``, + ``MPI_Allreduce()`` and so on) before pending point-to-point + communications are completed or when the collective function is only + called from a subset of the MPI processes. This also applies to some + internal LAMMPS functions like ``Error::all()`` which uses + ``MPI_Barrier()`` and thus ``Error::one()`` must be called, if the + error condition does not happen on all MPI processes simultaneously. +- Some function in LAMMPS has a bug where a ``for`` or ``while`` loop + does not trigger the exit condition and thus will loop forever. This + can happen when the wrong variable is incremented or when one value in + a comparison becomes ``NaN`` due to an overflow. + +In the latter two cases, further information and stack traces (see above) +can be obtain by attaching a debugger to a running process. For that the +process ID (PID) is needed; this can be found on Linux machines with the +``top``, ``htop``, ``ps``, or ``pstree`` commands. + +Then running the (GNU) debugger ``gdb`` with the ``-p`` flag followed by +the process id will attach the process to the debugger and stop +execution of that specific process. From there on it is possible to +issue all debugger commands in the same way as when LAMMPS was started +from the debugger (see above). Most importantly it is possible to +obtain a stack trace with the ``where`` command and thus determine where +in the execution of a timestep this process is. Also internal data can +be printed and execution single stepped or continued. When the debugger +is exited, the calculation will resume normally. diff --git a/doc/src/Errors_messages.rst b/doc/src/Errors_messages.rst index 2468b3c628..bfc395067a 100644 --- a/doc/src/Errors_messages.rst +++ b/doc/src/Errors_messages.rst @@ -7774,7 +7774,7 @@ Doc page with :doc:`WARNING messages ` *Too few values in body section of molecule file* Self-explanatory. -*Too many -pk arguments in command line* +*Too many -pk arguments in command-line* The string formed by concatenating the arguments is too long. Use a package command in the input script instead. diff --git a/doc/src/Examples.rst b/doc/src/Examples.rst index 3d2103fd6f..683cbd0500 100644 --- a/doc/src/Examples.rst +++ b/doc/src/Examples.rst @@ -146,6 +146,8 @@ Lowercase directories +-------------+------------------------------------------------------------------+ | streitz | use of Streitz/Mintmire potential with charge equilibration | +-------------+------------------------------------------------------------------+ +| stress_vcm | removing binned rigid body motion from binned stress profile | ++-------------+------------------------------------------------------------------+ | tad | temperature-accelerated dynamics of vacancy diffusion in bulk Si | +-------------+------------------------------------------------------------------+ | threebody | regression test input for a variety of manybody potentials | diff --git a/doc/src/Fortran.rst b/doc/src/Fortran.rst index a1ea9a72df..53dc5b8897 100644 --- a/doc/src/Fortran.rst +++ b/doc/src/Fortran.rst @@ -16,7 +16,7 @@ compiled alongside the code using it from the source code in ``fortran/lammps.f90`` *and* with the same compiler used to build the rest of the Fortran code that interfaces to LAMMPS. When linking, you also need to :doc:`link to the LAMMPS library `. A typical -command line for a simple program using the Fortran interface would be: +command for a simple program using the Fortran interface would be: .. code-block:: bash @@ -91,12 +91,12 @@ function and triggered with the optional logical argument set to CALL lmp%close(.TRUE.) END PROGRAM testlib -It is also possible to pass command line flags from Fortran to C/C++ and +It is also possible to pass command-line flags from Fortran to C/C++ and thus make the resulting executable behave similarly to the standalone executable (it will ignore the `-in/-i` flag, though). This allows -using the command line to configure accelerator and suffix settings, +using the command-line to configure accelerator and suffix settings, configure screen and logfile output, or to set index style variables -from the command line and more. Here is a correspondingly adapted +from the command-line and more. Here is a correspondingly adapted version of the previous example: .. code-block:: fortran @@ -108,7 +108,7 @@ version of the previous example: CHARACTER(LEN=128), ALLOCATABLE :: command_args(:) INTEGER :: i, argc - ! copy command line flags to `command_args()` + ! copy command-line flags to `command_args()` argc = COMMAND_ARGUMENT_COUNT() ALLOCATE(command_args(0:argc)) DO i=0, argc @@ -321,6 +321,14 @@ of the contents of the :f:mod:`LIBLAMMPS` Fortran interface to LAMMPS. :ftype set_string_variable: subroutine :f set_internal_variable: :f:subr:`set_internal_variable` :ftype set_internal_variable: subroutine + :f eval: :f:func:`eval` + :ftype eval: function + :f clearstep_compute: :f:subr:`clearstep_compute` + :ftype clearstep_compute: subroutine + :f addstep_compute: :f:subr:`addstep_compute` + :ftype addstep_compute: subroutine + :f addstep_compute_all: :f:subr:`addstep_compute_all` + :ftype addstep_compute_all: subroutine :f gather_atoms: :f:subr:`gather_atoms` :ftype gather_atoms: subroutine :f gather_atoms_concat: :f:subr:`gather_atoms_concat` @@ -448,7 +456,7 @@ of the contents of the :f:mod:`LIBLAMMPS` Fortran interface to LAMMPS. compiled with MPI support, it will also initialize MPI, if it has not already been initialized before. - The *args* argument with the list of command line parameters is + The *args* argument with the list of command-line parameters is optional and so it the *comm* argument with the MPI communicator. If *comm* is not provided, ``MPI_COMM_WORLD`` is assumed. For more details please see the documentation of :cpp:func:`lammps_open`. @@ -954,6 +962,7 @@ Procedures Bound to the :f:type:`lammps` Derived Type :f:func:`extract_atom` between runs. .. admonition:: Array index order + :class: tip Two-dimensional arrays returned from :f:func:`extract_atom` will be **transposed** from equivalent arrays in C, and they will be indexed @@ -1066,6 +1075,7 @@ Procedures Bound to the :f:type:`lammps` Derived Type you based on data from the :cpp:class:`Compute` class. .. admonition:: Array index order + :class: tip Two-dimensional arrays returned from :f:func:`extract_compute` will be **transposed** from equivalent arrays in C, and they will be indexed @@ -1324,6 +1334,7 @@ Procedures Bound to the :f:type:`lammps` Derived Type :rtype data: polymorphic .. admonition:: Array index order + :class: tip Two-dimensional global, per-atom, or local array data from :f:func:`extract_fix` will be **transposed** from equivalent arrays in @@ -1448,11 +1459,62 @@ Procedures Bound to the :f:type:`lammps` Derived Type an internal-style variable, an error is generated. :p character(len=*) name: name of the variable - :p read(c_double) val: new value to assign to the variable + :p real(c_double) val: new value to assign to the variable :to: :cpp:func:`lammps_set_internal_variable` -------- +.. f:function:: eval(expr) + + This function is a wrapper around :cpp:func:`lammps_eval` that takes a + LAMMPS equal style variable string, evaluates it and returns the resulting + scalar value as a floating-point number. + + .. versionadded:: TBD + + :p character(len=\*) expr: string to be evaluated + :to: :cpp:func:`lammps_eval` + :r value [real(c_double)]: result of the evaluated string + +-------- + +.. f:subroutine:: clearstep_compute() + + Clear whether a compute has been invoked + + .. versionadded:: TBD + + :to: :cpp:func:`lammps_clearstep_compute` + +-------- + +.. f:subroutine:: addstep_compute(nextstep) + + Add timestep to list of future compute invocations + if the compute has been invoked on the current timestep + + .. versionadded:: TBD + + overloaded for 32-bit and 64-bit integer arguments + + :p integer(kind=8 or kind=4) nextstep: next timestep + :to: :cpp:func:`lammps_addstep_compute` + +-------- + +.. f:subroutine:: addstep_compute_all(nextstep) + + Add timestep to list of future compute invocations + + .. versionadded:: TBD + + overloaded for 32-bit and 64-bit integer arguments + + :p integer(kind=8 or kind=4) nextstep: next timestep + :to: :cpp:func:`lammps_addstep_compute_all` + +-------- + .. f:subroutine:: gather_atoms(name, count, data) This function calls :cpp:func:`lammps_gather_atoms` to gather the named diff --git a/doc/src/Howto.rst b/doc/src/Howto.rst index 5a63e2b1c4..df42f6bd9d 100644 --- a/doc/src/Howto.rst +++ b/doc/src/Howto.rst @@ -103,6 +103,7 @@ Tutorials howto Howto_github Howto_lammps_gui Howto_moltemplate + Howto_python Howto_pylammps Howto_wsl diff --git a/doc/src/Howto_barostat.rst b/doc/src/Howto_barostat.rst index 0c25e2c53c..89ff4d183f 100644 --- a/doc/src/Howto_barostat.rst +++ b/doc/src/Howto_barostat.rst @@ -10,20 +10,21 @@ and/or pressure (P) is specified by the user, and the thermostat or barostat attempts to equilibrate the system to the requested T and/or P. -Barostatting in LAMMPS is performed by :doc:`fixes `. Two +Barostatting in LAMMPS is performed by :doc:`fixes `. Three barostatting methods are currently available: Nose-Hoover (npt and -nph) and Berendsen: +nph), Berendsen, and various linear controllers in deform/pressure: * :doc:`fix npt ` * :doc:`fix npt/sphere ` * :doc:`fix npt/asphere ` * :doc:`fix nph ` * :doc:`fix press/berendsen ` +* :doc:`fix deform/pressure ` The :doc:`fix npt ` commands include a Nose-Hoover thermostat and barostat. :doc:`Fix nph ` is just a Nose/Hoover barostat; -it does no thermostatting. Both :doc:`fix nph ` and :doc:`fix press/berendsen ` can be used in conjunction -with any of the thermostatting fixes. +it does no thermostatting. The fixes :doc:`nph `, :doc:`press/berendsen `, and :doc:`deform/pressure ` +can be used in conjunction with any of the thermostatting fixes. As with the :doc:`thermostats `, :doc:`fix npt ` and :doc:`fix nph ` only use translational motion of the @@ -44,9 +45,9 @@ a temperature or pressure compute to a barostatting fix. .. note:: As with the thermostats, the Nose/Hoover methods (:doc:`fix npt ` and :doc:`fix nph `) perform time integration. - :doc:`Fix press/berendsen ` does NOT, so it should - be used with one of the constant NVE fixes or with one of the NVT - fixes. + :doc:`Fix press/berendsen ` and :doc:`fix deform/pressure ` + do NOT, so they should be used with one of the constant NVE fixes or with + one of the NVT fixes. Thermodynamic output, which can be setup via the :doc:`thermo_style ` command, often includes pressure diff --git a/doc/src/Howto_cmake.rst b/doc/src/Howto_cmake.rst index 43aa519293..6b8fc82bad 100644 --- a/doc/src/Howto_cmake.rst +++ b/doc/src/Howto_cmake.rst @@ -56,7 +56,7 @@ using a shell like Bash or Zsh. Visual Studio IDE with the bundled CMake or from the Windows command prompt using a separately installed CMake package, both using the native Microsoft Visual C++ compilers and (optionally) the Microsoft MPI SDK. This tutorial, however, only - covers unix-like command line interfaces. + covers unix-like command-line interfaces. We also assume that you have downloaded and unpacked a recent LAMMPS source code package or used Git to create a clone of the LAMMPS sources on your compilation machine. @@ -277,7 +277,7 @@ Setting options --------------- Options that enable, disable or modify settings are modified by setting -the value of CMake variables. This is done on the command line with the +the value of CMake variables. This is done on the command-line with the *-D* flag in the format ``-D VARIABLE=value``, e.g. ``-D CMAKE_BUILD_TYPE=Release`` or ``-D BUILD_MPI=on``. There is one quirk: when used before the CMake directory, there may be a space between the @@ -376,7 +376,7 @@ Using presets ------------- Since LAMMPS has a lot of optional features and packages, specifying -them all on the command line can be tedious. Or when selecting a +them all on the command-line can be tedious. Or when selecting a different compiler toolchain, multiple options have to be changed consistently and that is rather error prone. Or when enabling certain packages, they require consistent settings to be operated in a @@ -384,7 +384,7 @@ particular mode. For this purpose, we are providing a selection of "preset files" for CMake in the folder ``cmake/presets``. They represent a way to pre-load or override the CMake configuration cache by setting or changing CMake variables. Preset files are loaded using the -*-C* command line flag. You can combine loading multiple preset files or +*-C* command-line flag. You can combine loading multiple preset files or change some variables later with additional *-D* flags. A few examples: .. code-block:: bash diff --git a/doc/src/Howto_github.rst b/doc/src/Howto_github.rst index b81716c09d..78a05f113c 100644 --- a/doc/src/Howto_github.rst +++ b/doc/src/Howto_github.rst @@ -163,7 +163,7 @@ After everything is done, add the files to the branch and commit them: *git rm*, *git mv* for adding, removing, renaming individual files, respectively, and then *git commit* to finalize the commit. Carefully check all pending changes with *git status* before - committing them. If you find doing this on the command line too + committing them. If you find doing this on the command-line too tedious, consider using a GUI, for example the one included in git distributions written in Tk, i.e. use *git gui* (on some Linux distributions it may be required to install an additional package to diff --git a/doc/src/Howto_lammps_gui.rst b/doc/src/Howto_lammps_gui.rst index 21e6a31ccc..63cf859c57 100644 --- a/doc/src/Howto_lammps_gui.rst +++ b/doc/src/Howto_lammps_gui.rst @@ -20,8 +20,11 @@ to the online LAMMPS documentation for known LAMMPS commands and styles. (Ubuntu 20.04LTS or later and compatible), macOS (version 11 aka Big Sur or later), and Windows (version 10 or later) :ref:`are available ` for download. Non-MPI LAMMPS executables (as - ``lmp``) for running LAMMPS from the command line and :doc:`some + ``lmp``) for running LAMMPS from the command-line and :doc:`some LAMMPS tools ` compiled executables are also included. + Also, the pre-compiled LAMMPS-GUI packages include the WHAM executables + from http://membrane.urmc.rochester.edu/content/wham/ for use with + LAMMPS tutorials. The source code for LAMMPS-GUI is included in the LAMMPS source code distribution and can be found in the ``tools/lammps-gui`` folder. It @@ -29,16 +32,16 @@ to the online LAMMPS documentation for known LAMMPS commands and styles. `. LAMMPS-GUI tries to provide an experience similar to what people -traditionally would have running LAMMPS using a command line window and +traditionally would have running LAMMPS using a command-line window and the console LAMMPS executable but just rolled into a single executable: - writing & editing LAMMPS input files with a text editor -- run LAMMPS on those input file with selected command line flags +- run LAMMPS on those input file with selected command-line flags - extract data from the created files and visualize it with and external software That procedure is quite effective for people proficient in using the -command line, as that allows them to use tools for the individual steps +command-line, as that allows them to use tools for the individual steps that they are most comfortable with. In fact, it is often *required* to adopt this workflow when running LAMMPS simulations on high-performance computing facilities. @@ -61,13 +64,18 @@ simple LAMMPS simulations. It is very suitable for tutorials on LAMMPS since you only need to learn how to use a single program for most tasks and thus time can be saved and people can focus on learning LAMMPS. The tutorials at https://lammpstutorials.github.io/ are specifically -updated for use with LAMMPS-GUI. +updated for use with LAMMPS-GUI and can their tutorial materials can +be downloaded and loaded directly from the GUI. Another design goal is to keep the barrier low when replacing part of the functionality of LAMMPS-GUI with external tools. That said, LAMMPS-GUI has some unique functionality that is not found elsewhere: - auto-adapting to features available in the integrated LAMMPS library +- auto-completion for LAMMPS commands and options +- context-sensitive online help +- start and stop of simulations via mouse or keyboard +- monitoring of simulation progress - interactive visualization using the :doc:`dump image ` command with the option to copy-paste the resulting settings - automatic slide show generation from dump image out at runtime @@ -100,10 +108,11 @@ MacOS 11 and later ^^^^^^^^^^^^^^^^^^ After downloading the ``LAMMPS-macOS-multiarch-GUI-.dmg`` -installer package, you need to double-click it and then, in the window -that opens, drag the app bundle as indicated into the "Applications" -folder. The follow the instructions in the "README.txt" file to -get access to the other included executables. +application bundle disk image, you need to double-click it and then, in +the window that opens, drag the app bundle as indicated into the +"Applications" folder. Afterwards, the disk image can be unmounted. +Then follow the instructions in the "README.txt" file to get access to +the other included command-line executables. Linux on x86\_64 ^^^^^^^^^^^^^^^^ @@ -117,15 +126,25 @@ into the "LAMMPS_GUI" folder and execute "./lammps-gui" directly. The second variant uses `flatpak `_ and requires the flatpak management and runtime software to be installed. -After downloading the ``LAMMPS-GUI-Linux-x86_64-GUI-.tar.gz`` +After downloading the ``LAMMPS-GUI-Linux-x86_64-GUI-.flatpak`` flatpak bundle, you can install it with ``flatpak install --user -LAMMPS-GUI-Linux-x86_64-GUI-.tar.gz``. After installation, +LAMMPS-GUI-Linux-x86_64-GUI-.flatpak``. After installation, LAMMPS-GUI should be integrated into your desktop environment under "Applications > Science" but also can be launched from the console with ``flatpak run org.lammps.lammps-gui``. The flatpak bundle also includes the console LAMMPS executable ``lmp`` which can be launched to run -simulations with, for example: ``flatpak run --command=lmp -org.lammps.lammps-gui -in in.melt``. +simulations with, for example with: + +.. code-block:: sh + + flatpak run --command=lmp org.lammps.lammps-gui -in in.melt + +Other bundled command-line executables are run the same way and can be +listed with: + +.. code-block:: sh + + ls $(flatpak info --show-location org.lammps.lammps-gui )/files/bin Compiling from Source @@ -165,9 +184,9 @@ window is stored when exiting and restored when starting again. Opening Files ^^^^^^^^^^^^^ -The LAMMPS-GUI application can be launched without command line arguments +The LAMMPS-GUI application can be launched without command-line arguments and then starts with an empty buffer in the *Editor* window. If arguments -are given LAMMPS will use first command line argument as the file name for +are given LAMMPS will use first command-line argument as the file name for the *Editor* buffer and reads its contents into the buffer, if the file exists. All further arguments are ignored. Files can also be opened via the *File* menu, the `Ctrl-O` (`Command-O` on macOS) keyboard shortcut @@ -261,14 +280,21 @@ Output Window By default, when starting a run, an *Output* window opens that displays the screen output of the running LAMMPS calculation, as shown below. -This text would normally be seen in the command line window. +This text would normally be seen in the command-line window. .. image:: JPG/lammps-gui-log.png :align: center :scale: 50% LAMMPS-GUI captures the screen output from LAMMPS as it is generated and -updates the *Output* window regularly during a run. +updates the *Output* window regularly during a run. If there are any +warnings or errors in the LAMMPS output, they are highlighted by using +bold text colored in red. There is a small panel at the bottom center +of the *Output* window showing how many warnings and errors were +detected and how many lines the entire output has. By clicking on the +button on the right with the warning symbol or by using the keyboard +shortcut `Ctrl-N` (`Command-N` on macOS), you can jump to the next +line with a warning or error. By default, the *Output* window is replaced each time a run is started. The runs are counted and the run number for the current run is displayed @@ -398,7 +424,7 @@ below. Like for the *Output* and *Charts* windows, its content is continuously updated during a run. It will show "(none)" if there are no variables defined. Note that it is also possible to *set* :doc:`index style -variables `, that would normally be set via command line +variables `, that would normally be set via command-line flags, via the "Set Variables..." dialog from the *Run* menu. LAMMPS-GUI automatically defines the variable "gui_run" to the current value of the run counter. That way it is possible to automatically @@ -775,11 +801,11 @@ General Settings: - *Echo input to log:* when checked, all input commands, including variable expansions, are echoed to the *Output* window. This is - equivalent to using `-echo screen` at the command line. There is no + equivalent to using `-echo screen` at the command-line. There is no log *file* produced by default, since LAMMPS-GUI uses `-log none`. - *Include citation details:* when checked full citation info will be included to the log window. This is equivalent to using `-cite - screen` on the command line. + screen` on the command-line. - *Show log window by default:* when checked, the screen output of a LAMMPS run will be collected in a log window during the run - *Show chart window by default:* when checked, the thermodynamic @@ -828,7 +854,7 @@ Accelerators: This tab enables selection of an accelerator package for LAMMPS to use and is equivalent to using the `-suffix` and `-package` flags on the -command line. Only settings supported by the LAMMPS library and local +command-line. Only settings supported by the LAMMPS library and local hardware are available. The `Number of threads` field allows setting the maximum number of threads for the accelerator packages that use threads. diff --git a/doc/src/Howto_peri.rst b/doc/src/Howto_peri.rst index 8a2aa4cab6..29eb685c81 100644 --- a/doc/src/Howto_peri.rst +++ b/doc/src/Howto_peri.rst @@ -738,8 +738,8 @@ command. This can be done, for example, by using the built-in visualizer of the :doc:`dump image or dump movie ` command to create snapshot -images or a movie. Below are example command lines for using dump image -with the :ref:`example listed below ` and a set of images +images or a movie. Below are example command for using dump image with +the :ref:`example listed below ` and a set of images created for steps 300, 600, and 2000 this way. .. code-block:: LAMMPS diff --git a/doc/src/Howto_pylammps.rst b/doc/src/Howto_pylammps.rst index 645434bbab..4a9b985bf6 100644 --- a/doc/src/Howto_pylammps.rst +++ b/doc/src/Howto_pylammps.rst @@ -1,564 +1,6 @@ PyLammps Tutorial ================= -.. contents:: - -Overview --------- - -:py:class:`PyLammps ` is a Python wrapper class for -LAMMPS which can be created on its own or use an existing -:py:class:`lammps Python ` object. It creates a simpler, -more "pythonic" interface to common LAMMPS functionality, in contrast to -the :py:class:`lammps ` wrapper for the LAMMPS :ref:`C -language library interface API ` which is written using -`Python ctypes `_. The :py:class:`lammps ` -wrapper is discussed on the :doc:`Python_head` doc page. - -Unlike the flat `ctypes `_ interface, PyLammps exposes a -discoverable API. It no longer requires knowledge of the underlying C++ -code implementation. Finally, the :py:class:`IPyLammps -` wrapper builds on top of :py:class:`PyLammps -` and adds some additional features for `IPython -integration `_ into `Jupyter notebooks `_, e.g. for -embedded visualization output from :doc:`dump style image `. - -.. _ctypes: https://docs.python.org/3/library/ctypes.html -.. _ipython: https://ipython.org/ -.. _jupyter: https://jupyter.org/ - -Comparison of lammps and PyLammps interfaces -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -lammps.lammps -""""""""""""" - -* uses `ctypes `_ -* direct memory access to native C++ data with optional support for NumPy arrays -* provides functions to send and receive data to LAMMPS -* interface modeled after the LAMMPS :ref:`C language library interface API ` -* requires knowledge of how LAMMPS internally works (C pointers, etc) -* full support for running Python with MPI using `mpi4py `_ -* no overhead from creating a more Python-like interface - -lammps.PyLammps -""""""""""""""" - -* higher-level abstraction built on *top* of the original :py:class:`ctypes based interface ` -* manipulation of Python objects -* communication with LAMMPS is hidden from API user -* shorter, more concise Python -* better IPython integration, designed for quick prototyping -* designed for serial execution -* additional overhead from capturing and parsing the LAMMPS screen output - -Quick Start ------------ - -System-wide Installation -^^^^^^^^^^^^^^^^^^^^^^^^ - -Step 1: Building LAMMPS as a shared library -""""""""""""""""""""""""""""""""""""""""""" - -To use LAMMPS inside of Python it has to be compiled as shared -library. This library is then loaded by the Python interface. In this -example we enable the MOLECULE package and compile LAMMPS with PNG, JPEG -and FFMPEG output support enabled. - -Step 1a: For the CMake based build system, the steps are: - -.. code-block:: bash - - mkdir $LAMMPS_DIR/build-shared - cd $LAMMPS_DIR/build-shared - - # MPI, PNG, Jpeg, FFMPEG are auto-detected - cmake ../cmake -DPKG_MOLECULE=yes -DBUILD_LIB=yes -DBUILD_SHARED_LIBS=yes - make - -Step 1b: For the legacy, make based build system, the steps are: - -.. code-block:: bash - - cd $LAMMPS_DIR/src - - # add packages if necessary - make yes-MOLECULE - - # compile shared library using Makefile - make mpi mode=shlib LMP_INC="-DLAMMPS_PNG -DLAMMPS_JPEG -DLAMMPS_FFMPEG" JPG_LIB="-lpng -ljpeg" - -Step 2: Installing the LAMMPS Python package -"""""""""""""""""""""""""""""""""""""""""""" - -PyLammps is part of the lammps Python package. To install it simply install -that package into your current Python installation with: - -.. code-block:: bash - - make install-python - -.. note:: - - Recompiling the shared library requires re-installing the Python package - -Installation inside of a virtualenv -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -You can use virtualenv to create a custom Python environment specifically tuned -for your workflow. - -Benefits of using a virtualenv -"""""""""""""""""""""""""""""" - -* isolation of your system Python installation from your development installation -* installation can happen in your user directory without root access (useful for HPC clusters) -* installing packages through pip allows you to get newer versions of packages than e.g., through apt-get or yum package managers (and without root access) -* you can even install specific old versions of a package if necessary - -**Prerequisite (e.g. on Ubuntu)** - -.. code-block:: bash - - apt-get install python-virtualenv - -Creating a virtualenv with lammps installed -""""""""""""""""""""""""""""""""""""""""""" - -.. code-block:: bash - - # create virtualenv named 'testing' - virtualenv $HOME/python/testing - - # activate 'testing' environment - source $HOME/python/testing/bin/activate - -Now configure and compile the LAMMPS shared library as outlined above. -When using CMake and the shared library has already been build, you -need to re-run CMake to update the location of the python executable -to the location in the virtual environment with: - -.. code-block:: bash - - cmake . -DPython_EXECUTABLE=$(which python) - - # install LAMMPS package in virtualenv - (testing) make install-python - - # install other useful packages - (testing) pip install matplotlib jupyter mpi4py - - ... - - # return to original shell - (testing) deactivate - -Creating a new instance of PyLammps ------------------------------------ - -To create a PyLammps object you need to first import the class from the lammps -module. By using the default constructor, a new *lammps* instance is created. - -.. code-block:: python - - from lammps import PyLammps - L = PyLammps() - -You can also initialize PyLammps on top of this existing *lammps* object: - -.. code-block:: python - - from lammps import lammps, PyLammps - lmp = lammps() - L = PyLammps(ptr=lmp) - -Commands --------- - -Sending a LAMMPS command with the existing library interfaces is done using -the command method of the lammps object instance. - -For instance, let's take the following LAMMPS command: - -.. code-block:: LAMMPS - - region box block 0 10 0 5 -0.5 0.5 - -In the original interface this command can be executed with the following -Python code if *L* was a lammps instance: - -.. code-block:: python - - L.command("region box block 0 10 0 5 -0.5 0.5") - -With the PyLammps interface, any command can be split up into arbitrary parts -separated by white-space, passed as individual arguments to a region method. - -.. code-block:: python - - L.region("box block", 0, 10, 0, 5, -0.5, 0.5) - -Note that each parameter is set as Python literal floating-point number. In the -PyLammps interface, each command takes an arbitrary parameter list and transparently -merges it to a single command string, separating individual parameters by white-space. - -The benefit of this approach is avoiding redundant command calls and easier -parameterization. In the original interface parameterization needed to be done -manually by creating formatted strings. - -.. code-block:: python - - L.command("region box block %f %f %f %f %f %f" % (xlo, xhi, ylo, yhi, zlo, zhi)) - -In contrast, methods of PyLammps accept parameters directly and will convert -them automatically to a final command string. - -.. code-block:: python - - L.region("box block", xlo, xhi, ylo, yhi, zlo, zhi) - -System state ------------- - -In addition to dispatching commands directly through the PyLammps object, it -also provides several properties which allow you to query the system state. - -L.system - Is a dictionary describing the system such as the bounding box or number of atoms - -L.system.xlo, L.system.xhi - bounding box limits along x-axis - -L.system.ylo, L.system.yhi - bounding box limits along y-axis - -L.system.zlo, L.system.zhi - bounding box limits along z-axis - -L.communication - configuration of communication subsystem, such as the number of threads or processors - -L.communication.nthreads - number of threads used by each LAMMPS process - -L.communication.nprocs - number of MPI processes used by LAMMPS - -L.fixes - List of fixes in the current system - -L.computes - List of active computes in the current system - -L.dump - List of active dumps in the current system - -L.groups - List of groups present in the current system - -Working with LAMMPS variables ------------------------------ - -LAMMPS variables can be both defined and accessed via the PyLammps interface. - -To define a variable you can use the :doc:`variable ` command: - -.. code-block:: python - - L.variable("a index 2") - -A dictionary of all variables is returned by L.variables - -you can access an individual variable by retrieving a variable object from the -L.variables dictionary by name - -.. code-block:: python - - a = L.variables['a'] - -The variable value can then be easily read and written by accessing the value -property of this object. - -.. code-block:: python - - print(a.value) - a.value = 4 - -Retrieving the value of an arbitrary LAMMPS expressions -------------------------------------------------------- - -LAMMPS expressions can be immediately evaluated by using the eval method. The -passed string parameter can be any expression containing global thermo values, -variables, compute or fix data. - -.. code-block:: python - - result = L.eval("ke") # kinetic energy - result = L.eval("pe") # potential energy - - result = L.eval("v_t/2.0") - -Accessing atom data -------------------- - -All atoms in the current simulation can be accessed by using the L.atoms list. -Each element of this list is an object which exposes its properties (id, type, -position, velocity, force, etc.). - -.. code-block:: python - - # access first atom - L.atoms[0].id - L.atoms[0].type - - # access second atom - L.atoms[1].position - L.atoms[1].velocity - L.atoms[1].force - -Some properties can also be used to set: - -.. code-block:: python - - # set position in 2D simulation - L.atoms[0].position = (1.0, 0.0) - - # set position in 3D simulation - L.atoms[0].position = (1.0, 0.0, 1.) - -Evaluating thermo data ----------------------- - -Each simulation run usually produces thermo output based on system state, -computes, fixes or variables. The trajectories of these values can be queried -after a run via the L.runs list. This list contains a growing list of run data. -The first element is the output of the first run, the second element that of -the second run. - -.. code-block:: python - - L.run(1000) - L.runs[0] # data of first 1000 time steps - - L.run(1000) - L.runs[1] # data of second 1000 time steps - -Each run contains a dictionary of all trajectories. Each trajectory is -accessible through its thermo name: - -.. code-block:: python - - L.runs[0].thermo.Step # list of time steps in first run - L.runs[0].thermo.Ke # list of kinetic energy values in first run - -Together with matplotlib plotting data out of LAMMPS becomes simple: - -.. code-block:: python - - import matplotlib.plot as plt - steps = L.runs[0].thermo.Step - ke = L.runs[0].thermo.Ke - plt.plot(steps, ke) - -Error handling with PyLammps ----------------------------- - -Using C++ exceptions in LAMMPS for errors allows capturing them on the -C++ side and rethrowing them on the Python side. This way you can handle -LAMMPS errors through the Python exception handling mechanism. - -.. warning:: - - Capturing a LAMMPS exception in Python can still mean that the - current LAMMPS process is in an illegal state and must be - terminated. It is advised to save your data and terminate the Python - instance as quickly as possible. - -Using PyLammps in IPython notebooks and Jupyter ------------------------------------------------ - -If the LAMMPS Python package is installed for the same Python interpreter as -IPython, you can use PyLammps directly inside of an IPython notebook inside of -Jupyter. Jupyter is a powerful integrated development environment (IDE) for -many dynamic languages like Python, Julia and others, which operates inside of -any web browser. Besides auto-completion and syntax highlighting it allows you -to create formatted documents using Markup, mathematical formulas, graphics and -animations intermixed with executable Python code. It is a great format for -tutorials and showcasing your latest research. - -To launch an instance of Jupyter simply run the following command inside your -Python environment (this assumes you followed the Quick Start instructions): - -.. code-block:: bash - - jupyter notebook - -IPyLammps Examples ------------------- - -Examples of IPython notebooks can be found in the python/examples/pylammps -subdirectory. To open these notebooks launch *jupyter notebook* inside this -directory and navigate to one of them. If you compiled and installed -a LAMMPS shared library with exceptions, PNG, JPEG and FFMPEG support -you should be able to rerun all of these notebooks. - -Validating a dihedral potential -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -This example showcases how an IPython Notebook can be used to compare a simple -LAMMPS simulation of a harmonic dihedral potential to its analytical solution. -Four atoms are placed in the simulation and the dihedral potential is applied on -them using a datafile. Then one of the atoms is rotated along the central axis by -setting its position from Python, which changes the dihedral angle. - -.. code-block:: python - - phi = [d \* math.pi / 180 for d in range(360)] - - pos = [(1.0, math.cos(p), math.sin(p)) for p in phi] - - pe = [] - for p in pos: - L.atoms[3].position = p - L.run(0) - pe.append(L.eval("pe")) - -By evaluating the potential energy for each position we can verify that -trajectory with the analytical formula. To compare both solutions, we plot -both trajectories over each other using matplotlib, which embeds the generated -plot inside the IPython notebook. - -.. image:: JPG/pylammps_dihedral.jpg - :align: center - -Running a Monte Carlo relaxation -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -This second example shows how to use PyLammps to create a 2D Monte Carlo Relaxation -simulation, computing and plotting energy terms and even embedding video output. - -Initially, a 2D system is created in a state with minimal energy. - -.. image:: JPG/pylammps_mc_minimum.jpg - :align: center - -It is then disordered by moving each atom by a random delta. - -.. code-block:: python - - random.seed(27848) - deltaperturb = 0.2 - - for i in range(L.system.natoms): - x, y = L.atoms[i].position - dx = deltaperturb \* random.uniform(-1, 1) - dy = deltaperturb \* random.uniform(-1, 1) - L.atoms[i].position = (x+dx, y+dy) - - L.run(0) - -.. image:: JPG/pylammps_mc_disordered.jpg - :align: center - -Finally, the Monte Carlo algorithm is implemented in Python. It continuously -moves random atoms by a random delta and only accepts certain moves. - -.. code-block:: python - - estart = L.eval("pe") - elast = estart - - naccept = 0 - energies = [estart] - - niterations = 3000 - deltamove = 0.1 - kT = 0.05 - - natoms = L.system.natoms - - for i in range(niterations): - iatom = random.randrange(0, natoms) - current_atom = L.atoms[iatom] - - x0, y0 = current_atom.position - - dx = deltamove \* random.uniform(-1, 1) - dy = deltamove \* random.uniform(-1, 1) - - current_atom.position = (x0+dx, y0+dy) - - L.run(1, "pre no post no") - - e = L.eval("pe") - energies.append(e) - - if e <= elast: - naccept += 1 - elast = e - elif random.random() <= math.exp(natoms\*(elast-e)/kT): - naccept += 1 - elast = e - else: - current_atom.position = (x0, y0) - -The energies of each iteration are collected in a Python list and finally plotted using matplotlib. - -.. image:: JPG/pylammps_mc_energies_plot.jpg - :align: center - -The IPython notebook also shows how to use dump commands and embed video files -inside of the IPython notebook. - -Using PyLammps and mpi4py (Experimental) ----------------------------------------- - -PyLammps can be run in parallel using `mpi4py -`_. This python package can be installed -using - -.. code-block:: bash - - pip install mpi4py - -.. warning:: - - Usually, any :py:class:`PyLammps ` command must be - executed by *all* MPI processes. However, evaluations and querying - the system state is only available on MPI rank 0. Using these - functions from other MPI ranks will raise an exception. - -The following is a short example which reads in an existing LAMMPS input -file and executes it in parallel. You can find in.melt in the -examples/melt folder. Please take note that the -:py:meth:`PyLammps.eval() ` is called only from -MPI rank 0. - -.. code-block:: python - - from mpi4py import MPI - from lammps import PyLammps - - L = PyLammps() - L.file("in.melt") - - if MPI.COMM_WORLD.rank == 0: - print("Potential energy: ", L.eval("pe")) - - MPI.Finalize() - -To run this script (melt.py) in parallel using 4 MPI processes we invoke the -following mpirun command: - -.. code-block:: bash - - mpirun -np 4 python melt.py - -Feedback and Contributing -------------------------- - -If you find this Python interface useful, please feel free to provide feedback -and ideas on how to improve it to Richard Berger (richard.berger@outlook.com). We also -want to encourage people to write tutorial style IPython notebooks showcasing LAMMPS usage -and maybe their latest research results. +The PyLammps interface is deprecated and will be removed in a future release of +LAMMPS. As such, the PyLammps version of this tutorial has been removed and is +replaced by the :doc:`Python_head`. diff --git a/doc/src/Howto_python.rst b/doc/src/Howto_python.rst new file mode 100644 index 0000000000..bfb182d989 --- /dev/null +++ b/doc/src/Howto_python.rst @@ -0,0 +1,441 @@ +LAMMPS Python Tutorial +====================== + +.. contents:: + +----- + +Overview +-------- + +The :py:class:`lammps ` Python module is a wrapper class for the +LAMMPS :ref:`C language library interface API ` which is written using +`Python ctypes `_. The design choice of this wrapper class is to +follow the C language API closely with only small changes related to Python +specific requirements and to better accommodate object oriented programming. + +In addition to this flat `ctypes `_ interface, the +:py:class:`lammps ` wrapper class exposes a discoverable +API that doesn't require as much knowledge of the underlying C language +library interface or LAMMPS C++ code implementation. + +Finally, the API exposes some additional features for `IPython integration +`_ into `Jupyter notebooks `_, e.g. for embedded +visualization output from :doc:`dump style image `. + +.. _ctypes: https://docs.python.org/3/library/ctypes.html +.. _ipython: https://ipython.org/ +.. _jupyter: https://jupyter.org/ + +----- + +Quick Start +----------- + +System-wide or User Installation +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Step 1: Building LAMMPS as a shared library +""""""""""""""""""""""""""""""""""""""""""" + +To use LAMMPS inside of Python it has to be compiled as shared library. +This library is then loaded by the Python interface. In this example we +enable the :ref:`MOLECULE package ` and compile LAMMPS +with :ref:`PNG, JPEG and FFMPEG output support ` enabled. + +.. tabs:: + + .. tab:: CMake build + + .. code-block:: bash + + mkdir $LAMMPS_DIR/build-shared + cd $LAMMPS_DIR/build-shared + + # MPI, PNG, Jpeg, FFMPEG are auto-detected + cmake ../cmake -DPKG_MOLECULE=yes -DPKG_PYTHON=on -DBUILD_SHARED_LIBS=yes + make + + .. tab:: Traditional make + + .. code-block:: bash + + cd $LAMMPS_DIR/src + + # add packages if necessary + make yes-MOLECULE + make yes-PYTHON + + # compile shared library using Makefile + make mpi mode=shlib LMP_INC="-DLAMMPS_PNG -DLAMMPS_JPEG -DLAMMPS_FFMPEG" JPG_LIB="-lpng -ljpeg" + +Step 2: Installing the LAMMPS Python package +"""""""""""""""""""""""""""""""""""""""""""" + +Next install the LAMMPS Python package into your current Python installation with: + +.. code-block:: bash + + make install-python + +This will create a so-called `"wheel" +`_ +and then install the LAMMPS Python module from that "wheel" into either +into a system folder (provided the command is executed with root +privileges) or into your personal Python module folder. + +.. note:: + + Recompiling the shared library requires re-installing the Python + package. + +Installation inside of a virtual environment +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +You can use virtual environments to create a custom Python environment +specifically tuned for your workflow. + +Benefits of using a virtualenv +"""""""""""""""""""""""""""""" + +* isolation of your system Python installation from your development installation +* installation can happen in your user directory without root access (useful for HPC clusters) +* installing packages through pip allows you to get newer versions of packages than e.g., through apt-get or yum package managers (and without root access) +* you can even install specific old versions of a package if necessary + +**Prerequisite (e.g. on Ubuntu)** + +.. code-block:: bash + + apt-get install python-venv + +Creating a virtualenv with lammps installed +""""""""""""""""""""""""""""""""""""""""""" + +.. code-block:: bash + + # create virtual envrionment named 'testing' + python3 -m venv $HOME/python/testing + + # activate 'testing' environment + source $HOME/python/testing/bin/activate + +Now configure and compile the LAMMPS shared library as outlined above. +When using CMake and the shared library has already been build, you +need to re-run CMake to update the location of the python executable +to the location in the virtual environment with: + +.. code-block:: bash + + cmake . -DPython_EXECUTABLE=$(which python) + + # install LAMMPS package in virtualenv + (testing) make install-python + + # install other useful packages + (testing) pip install matplotlib jupyter mpi4py pandas + + ... + + # return to original shell + (testing) deactivate + +------- + +Creating a new lammps instance +------------------------------ + +To create a lammps object you need to first import the class from the lammps +module. By using the default constructor, a new :py:class:`lammps +` instance is created. + +.. code-block:: python + + from lammps import lammps + L = lammps() + +See the :doc:`LAMMPS Python documentation ` for how to customize +the instance creation with optional arguments. + +----- + +Commands +-------- + +Sending a LAMMPS command with the library interface is done using +the ``command`` method of the lammps object. + +For instance, let's take the following LAMMPS command: + +.. code-block:: LAMMPS + + region box block 0 10 0 5 -0.5 0.5 + +This command can be executed with the following Python code if ``L`` is a ``lammps`` +instance: + +.. code-block:: python + + L.command("region box block 0 10 0 5 -0.5 0.5") + +For convenience, the ``lammps`` class also provides a command wrapper ``cmd`` +that turns any LAMMPS command into a regular function call: + +.. code-block:: python + + L.cmd.region("box block", 0, 10, 0, 5, -0.5, 0.5) + +Note that each parameter is set as Python number literal. With +the wrapper each command takes an arbitrary parameter list and transparently +merges it to a single command string, separating individual parameters by +white-space. + +The benefit of this approach is avoiding redundant command calls and easier +parameterization. With the ``command`` function each call needs to be assembled +manually using formatted strings. + +.. code-block:: python + + L.command(f"region box block {xlo} {xhi} {ylo} {yhi} {zlo} {zhi}") + +The wrapper accepts parameters directly and will convert +them automatically to a final command string. + +.. code-block:: python + + L.cmd.region("box block", xlo, xhi, ylo, yhi, zlo, zhi) + +.. note:: + + When running in IPython you can use Tab-completion after ``L.cmd.`` to see + all available LAMMPS commands. + +----- + +Accessing atom data +------------------- + +All per-atom properties that are part of the :doc:`atom style +` in the current simulation can be accessed using the +:py:meth:`extract_atoms() ` method. This +can be retrieved as ctypes objects or as NumPy arrays through the +lammps.numpy module. Those represent the *local* atoms of the +individual sub-domain for the current MPI process and may contain +information for the local ghost atoms or not depending on the property. +Both can be accessed as lists, but for the ctypes list object the size +is not known and hast to be retrieved first to avoid out-of-bounds +accesses. + +.. code-block:: python + + nlocal = L.extract_setting("nlocal") + nall = L.extract_setting("nall") + print("Number of local atoms ", nlocal, " Number of local and ghost atoms ", nall); + + # access via ctypes directly + atom_id = L.extract_atom("id") + print("Atom IDs", atom_id[0:nlocal]) + + # access through numpy wrapper + atom_type = L.numpy.extract_atom("type") + print("Atom types", atom_type) + + x = L.numpy.extract_atom("x") + v = L.numpy.extract_atom("v") + print("positions array shape", x.shape) + print("velocity array shape", v.shape) + # turn on communicating velocities to ghost atoms + L.cmd.comm_modify("vel", "yes") + v = L.numpy.extract_atom('v') + print("velocity array shape", v.shape) + +Some properties can also be set from Python since internally the +data of the C++ code is accessed directly: + +.. code-block:: python + + # set position in 2D simulation + x[0] = (1.0, 0.0) + + # set position in 3D simulation + x[0] = (1.0, 0.0, 1.) + +------ + +Retrieving the values of thermodynamic data and variables +--------------------------------------------------------- + +To access thermodynamic data from the last completed timestep, +you can use the :py:meth:`get_thermo() ` +method, and to extract the value of (compatible) variables, you +can use the :py:meth:`extract_variable() ` +method. + +.. code-block:: python + + result = L.get_thermo("ke") # kinetic energy + result = L.get_thermo("pe") # potential energy + + result = L.extract_variable("t") / 2.0 + +Error handling +-------------- + +We are using C++ exceptions in LAMMPS for errors and the C language +library interface captures and records them. This allows checking +whether errors have happened in Python during a call into LAMMPS and +then re-throw the error as a Python exception. This way you can handle +LAMMPS errors in the conventional way through the Python exception +handling mechanism. + +.. warning:: + + Capturing a LAMMPS exception in Python can still mean that the + current LAMMPS process is in an illegal state and must be + terminated. It is advised to save your data and terminate the Python + instance as quickly as possible. + +Using LAMMPS in IPython notebooks and Jupyter +--------------------------------------------- + +If the LAMMPS Python package is installed for the same Python +interpreter as IPython, you can use LAMMPS directly inside of an IPython +notebook inside of Jupyter. Jupyter is a powerful integrated development +environment (IDE) for many dynamic languages like Python, Julia and +others, which operates inside of any web browser. Besides +auto-completion and syntax highlighting it allows you to create +formatted documents using Markup, mathematical formulas, graphics and +animations intermixed with executable Python code. It is a great format +for tutorials and showcasing your latest research. + +To launch an instance of Jupyter simply run the following command inside your +Python environment (this assumes you followed the Quick Start instructions): + +.. code-block:: bash + + jupyter notebook + +Interactive Python Examples +--------------------------- + +Examples of IPython notebooks can be found in the ``python/examples/ipython`` +subdirectory. To open these notebooks launch ``jupyter notebook`` inside this +directory and navigate to one of them. If you compiled and installed +a LAMMPS shared library with PNG, JPEG and FFMPEG support +you should be able to rerun all of these notebooks. + +Validating a dihedral potential +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +This example showcases how an IPython Notebook can be used to compare a simple +LAMMPS simulation of a harmonic dihedral potential to its analytical solution. +Four atoms are placed in the simulation and the dihedral potential is applied on +them using a datafile. Then one of the atoms is rotated along the central axis by +setting its position from Python, which changes the dihedral angle. + +.. code-block:: python + + phi = [d \* math.pi / 180 for d in range(360)] + + pos = [(1.0, math.cos(p), math.sin(p)) for p in phi] + + x = L.numpy.extract_atom("x") + + pe = [] + for p in pos: + x[3] = p + L.cmd.run(0, "post", "no") + pe.append(L.get_thermo("pe")) + +By evaluating the potential energy for each position we can verify that +trajectory with the analytical formula. To compare both solutions, we plot +both trajectories over each other using matplotlib, which embeds the generated +plot inside the IPython notebook. + +.. image:: JPG/pylammps_dihedral.jpg + :align: center + +Running a Monte Carlo relaxation +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +This second example shows how to use the `lammps` Python interface to create a +2D Monte Carlo Relaxation simulation, computing and plotting energy terms and +even embedding video output. + +Initially, a 2D system is created in a state with minimal energy. + +.. image:: JPG/pylammps_mc_minimum.jpg + :align: center + +It is then disordered by moving each atom by a random delta. + +.. code-block:: python + + random.seed(27848) + deltaperturb = 0.2 + x = L.numpy.extract_atom("x") + natoms = x.shape[0] + + for i in range(natoms): + dx = deltaperturb \* random.uniform(-1, 1) + dy = deltaperturb \* random.uniform(-1, 1) + x[i][0] += dx + x[i][1] += dy + + L.cmd.run(0, "post", "no") + +.. image:: JPG/pylammps_mc_disordered.jpg + :align: center + +Finally, the Monte Carlo algorithm is implemented in Python. It continuously +moves random atoms by a random delta and only accepts certain moves. + +.. code-block:: python + + estart = L.get_thermo("pe") + elast = estart + + naccept = 0 + energies = [estart] + + niterations = 3000 + deltamove = 0.1 + kT = 0.05 + + for i in range(niterations): + x = L.numpy.extract_atom("x") + natoms = x.shape[0] + iatom = random.randrange(0, natoms) + current_atom = x[iatom] + + x0 = current_atom[0] + y0 = current_atom[1] + + dx = deltamove \* random.uniform(-1, 1) + dy = deltamove \* random.uniform(-1, 1) + + current_atom[0] = x0 + dx + current_atom[1] = y0 + dy + + L.cmd.run(1, "pre no post no") + + e = L.get_thermo("pe") + energies.append(e) + + if e <= elast: + naccept += 1 + elast = e + elif random.random() <= math.exp(natoms\*(elast-e)/kT): + naccept += 1 + elast = e + else: + current_atom[0] = x0 + current_atom[1] = y0 + +The energies of each iteration are collected in a Python list and finally plotted using matplotlib. + +.. image:: JPG/pylammps_mc_energies_plot.jpg + :align: center + +The IPython notebook also shows how to use dump commands and embed video files +inside of the IPython notebook. diff --git a/doc/src/Howto_wsl.rst b/doc/src/Howto_wsl.rst index fbb75d9cfd..30e4835e4c 100644 --- a/doc/src/Howto_wsl.rst +++ b/doc/src/Howto_wsl.rst @@ -260,7 +260,7 @@ Switch into the :code:`examples/melt` folder: cd ../examples/melt -To run this example in serial, use the following command line: +To run this example in serial, use the following command: .. code-block:: diff --git a/doc/src/Install_git.rst b/doc/src/Install_git.rst index 35b6429f0c..d01bc6a4c5 100644 --- a/doc/src/Install_git.rst +++ b/doc/src/Install_git.rst @@ -52,6 +52,7 @@ your machine and "release" is one of the 3 branches listed above. between them at any time using "git checkout ".) .. admonition:: Saving time and disk space when using ``git clone`` + :class: note The complete git history of the LAMMPS project is quite large because it contains the entire commit history of the project since fall 2006, @@ -60,7 +61,7 @@ between them at any time using "git checkout ".) files (mostly by accident). If you do not need access to the entire commit history (most people don't), you can speed up the "cloning" process and reduce local disk space requirements by using the - ``--depth`` git command line flag. That will create a "shallow clone" + ``--depth`` git command-line flag. That will create a "shallow clone" of the repository, which contains only a subset of the git history. Using a depth of 1000 is usually sufficient to include the head commits of the *develop*, the *release*, and the *maintenance* diff --git a/doc/src/Intro_authors.rst b/doc/src/Intro_authors.rst index 84470ba3a0..38f1102595 100644 --- a/doc/src/Intro_authors.rst +++ b/doc/src/Intro_authors.rst @@ -8,6 +8,8 @@ send an email to all of them at this address: "developers at lammps.org". General questions about LAMMPS should be posted in the `LAMMPS forum on MatSci `_. +.. We need to keep this file in sync with https://www.lammps.org/authors.html + .. raw:: latex \small @@ -27,7 +29,7 @@ lammps.org". General questions about LAMMPS should be posted in the * - `Steve Plimpton `_ - SNL (retired) - sjplimp at gmail.com - - MD kernels, parallel algorithms & scalability, code structure and design + - original author, MD kernels, parallel algorithms & scalability, code structure and design * - `Aidan Thompson `_ - SNL - athomps at sandia.gov diff --git a/doc/src/JPG/lammps-gui-log.png b/doc/src/JPG/lammps-gui-log.png index 894af371f0..61ab2e0601 100644 Binary files a/doc/src/JPG/lammps-gui-log.png and b/doc/src/JPG/lammps-gui-log.png differ diff --git a/doc/src/Library.rst b/doc/src/Library.rst index 50c28b7fcd..e2021187c2 100644 --- a/doc/src/Library.rst +++ b/doc/src/Library.rst @@ -131,16 +131,15 @@ run LAMMPS in serial mode. .. _lammps_python_api: -LAMMPS Python APIs -================== +LAMMPS Python API +================= The LAMMPS Python module enables calling the LAMMPS C library API from Python by dynamically loading functions in the LAMMPS shared library through the `Python ctypes module `_. Because of the dynamic loading, it is **required** that LAMMPS is compiled in :ref:`"shared" mode `. The Python interface is object-oriented, but -otherwise tries to be very similar to the C library API. Three different -Python classes to run LAMMPS are available and they build on each other. +otherwise tries to be very similar to the C library API. More information on this is in the :doc:`Python_head` section of the manual. Use of the LAMMPS Python module is described in :doc:`Python_module`. diff --git a/doc/src/Library_execute.rst b/doc/src/Library_execute.rst index 44b601ba4c..8560fbb148 100644 --- a/doc/src/Library_execute.rst +++ b/doc/src/Library_execute.rst @@ -7,6 +7,7 @@ This section documents the following functions: - :cpp:func:`lammps_command` - :cpp:func:`lammps_commands_list` - :cpp:func:`lammps_commands_string` +- :cpp:func:`lammps_expand` -------------------- @@ -79,3 +80,8 @@ Below is a short example using some of these functions. .. doxygenfunction:: lammps_commands_string :project: progguide +----------------------- + +.. doxygenfunction:: lammps_expand + :project: progguide + diff --git a/doc/src/Library_objects.rst b/doc/src/Library_objects.rst index 7c0ca824d7..86b6d253f0 100644 --- a/doc/src/Library_objects.rst +++ b/doc/src/Library_objects.rst @@ -1,5 +1,5 @@ -Compute, fixes, variables -========================= +Computes, fixes, variables +========================== This section documents accessing or modifying data stored by computes, fixes, or variables in LAMMPS using the following functions: @@ -12,6 +12,10 @@ fixes, or variables in LAMMPS using the following functions: - :cpp:func:`lammps_set_string_variable` - :cpp:func:`lammps_set_internal_variable` - :cpp:func:`lammps_variable_info` +- :cpp:func:`lammps_eval` +- :cpp:func:`lammps_clearstep_compute` +- :cpp:func:`lammps_addstep_compute_all` +- :cpp:func:`lammps_addstep_compute` ----------------------- @@ -55,6 +59,26 @@ fixes, or variables in LAMMPS using the following functions: ----------------------- +.. doxygenfunction:: lammps_eval + :project: progguide + +----------------------- + +.. doxygenfunction:: lammps_clearstep_compute + :project: progguide + +----------------------- + +.. doxygenfunction:: lammps_addstep_compute_all + :project: progguide + +----------------------- + +.. doxygenfunction:: lammps_addstep_compute + :project: progguide + +----------------------- + .. doxygenenum:: _LMP_DATATYPE_CONST .. doxygenenum:: _LMP_STYLE_CONST diff --git a/doc/src/Modify_compute.rst b/doc/src/Modify_compute.rst index bf580a6561..ea603985dd 100644 --- a/doc/src/Modify_compute.rst +++ b/doc/src/Modify_compute.rst @@ -45,6 +45,8 @@ class. See compute.h for details. +-----------------------+------------------------------------------------------------------+ | pair_tally_callback | callback function for *tally*\ -style computes (optional). | +-----------------------+------------------------------------------------------------------+ +| modify_param | called when a compute_modify request is executed (optional) | ++-----------------------+------------------------------------------------------------------+ | memory_usage | tally memory usage (optional) | +-----------------------+------------------------------------------------------------------+ diff --git a/doc/src/Modify_requirements.rst b/doc/src/Modify_requirements.rst index cbcb3eca13..c3e514a423 100644 --- a/doc/src/Modify_requirements.rst +++ b/doc/src/Modify_requirements.rst @@ -208,20 +208,21 @@ Build system (strict) LAMMPS currently supports two build systems: one that is based on :doc:`traditional Makefiles ` and one that is based on -:doc:`CMake `. Therefore, your contribution must be -compatible with and support both build systems. +:doc:`CMake `. As of fall 2024, it is no longer required +to support the traditional make build system. New packages may choose +to only support building with CMake. Additions to existing packages +must follow the requirements set by that package. For a single pair of header and implementation files that are an independent feature, it is usually only required to add them to ``src/.gitignore``. For traditional make, if your contributed files or package depend on -other LAMMPS style files or packages also being installed -(e.g. because your file is a derived class from the other LAMMPS -class), then an ``Install.sh`` file is also needed to check for those -dependencies and modifications to ``src/Depend.sh`` to trigger the checks. -See other README and Install.sh files in other directories as -examples. +other LAMMPS style files or packages also being installed (e.g. because +your file is a derived class from the other LAMMPS class), then an +``Install.sh`` file is also needed to check for those dependencies and +modifications to ``src/Depend.sh`` to trigger the checks. See other +README and Install.sh files in other directories as examples. Similarly, for CMake support, changes may need to be made to ``cmake/CMakeLists.txt``, some of the files in ``cmake/presets``, and diff --git a/doc/src/Modify_style.rst b/doc/src/Modify_style.rst index 496415237c..69b7cc9350 100644 --- a/doc/src/Modify_style.rst +++ b/doc/src/Modify_style.rst @@ -46,7 +46,7 @@ Include files (varied) but instead should be initialized either in the initializer list of the constructor or explicitly assigned in the body of the constructor. If the member variable is relevant to the functionality of a class - (for example when it stores a value from a command line argument), the + (for example when it stores a value from a command-line argument), the member variable declaration is followed by a brief comment explaining its purpose and what its values can be. Class members that are pointers should always be initialized to ``nullptr`` in the diff --git a/doc/src/Packages_details.rst b/doc/src/Packages_details.rst index 466d4134cb..baefee0185 100644 --- a/doc/src/Packages_details.rst +++ b/doc/src/Packages_details.rst @@ -994,6 +994,7 @@ Additional pair styles that are less commonly used. * ``src/EXTRA-PAIR``: filenames -> commands * :doc:`pair_style ` +* ``examples/PACKAGES/dispersion`` ---------- @@ -2171,8 +2172,8 @@ the :doc:`Build extras ` page. * ``src/OPENMP/README`` * :doc:`Accelerator packages ` * :doc:`OPENMP package ` -* :doc:`Command line option -suffix/-sf omp ` -* :doc:`Command line option -package/-pk omp ` +* :doc:`Command-line option -suffix/-sf omp ` +* :doc:`Command-line option -package/-pk omp ` * :doc:`package omp ` * Search the :doc:`commands ` pages (:doc:`fix `, :doc:`compute `, :doc:`pair `, :doc:`bond, angle, dihedral, improper `, @@ -2789,14 +2790,15 @@ implements smoothed particle hydrodynamics (SPH) for liquids. See the related :ref:`MACHDYN package ` package for smooth Mach dynamics (SMD) for solids. -This package contains ideal gas, Lennard-Jones equation of states, -Tait, and full support for complete (i.e. internal-energy dependent) -equations of state. It allows for plain or Monaghans XSPH integration -of the equations of motion. It has options for density continuity or -density summation to propagate the density field. It has -:doc:`set ` command options to set the internal energy and density -of particles from the input script and allows the same quantities to -be output with thermodynamic output or to dump files via the :doc:`compute property/atom ` command. +This package contains ideal gas, Lennard-Jones equation of states, Tait, +and full support for complete (i.e. internal-energy dependent) equations +of state. It allows for plain or Monaghans XSPH integration of the +equations of motion. It has options for density continuity or density +summation to propagate the density field. It has :doc:`set ` +command options to set the internal energy and density of particles from +the input script and allows the same quantities to be output with +thermodynamic output or to dump files via the :doc:`compute +property/atom ` command. **Author:** Georg Ganzenmuller (Fraunhofer-Institute for High-Speed Dynamics, Ernst Mach Institute, Germany). @@ -2809,6 +2811,17 @@ Dynamics, Ernst Mach Institute, Germany). * ``examples/PACKAGES/sph`` * https://www.lammps.org/movies.html#sph +.. note:: + + Please note that the SPH PDF guide file has not been updated for + many years and thus does not reflect the current *syntax* of the + SPH package commands. For that please refer to the LAMMPS manual. + +.. note:: + + Please also note, that the :ref:`RHEO package ` offers + similar functionality in a more modern and flexible implementation. + ---------- .. _PKG-SPIN: diff --git a/doc/src/Python_atoms.rst b/doc/src/Python_atoms.rst index f01559a524..2d07cc2326 100644 --- a/doc/src/Python_atoms.rst +++ b/doc/src/Python_atoms.rst @@ -2,14 +2,8 @@ Per-atom properties =================== Similar to what is described in :doc:`Library_atoms`, the instances of -:py:class:`lammps `, :py:class:`PyLammps `, or -:py:class:`IPyLammps ` can be used to extract atom quantities -and modify some of them. The main difference between the interfaces is how the information -is exposed. - -While the :py:class:`lammps ` is just a thin layer that wraps C API calls, -:py:class:`PyLammps ` and :py:class:`IPyLammps ` expose -information as objects and properties. +:py:class:`lammps ` can be used to extract atom quantities +and modify some of them. In some cases the data returned is a direct reference to the original data inside LAMMPS cast to ``ctypes`` pointers. Where possible, the wrappers will @@ -25,57 +19,41 @@ against invalid accesses. accordingly. These arrays can change sizes and order at every neighbor list rebuild and atom sort event as atoms are migrating between subdomains. -.. tabs:: +.. code-block:: python - .. tab:: lammps API + from lammps import lammps - .. code-block:: python + lmp = lammps() + lmp.file("in.sysinit") - from lammps import lammps - lmp = lammps() - lmp.file("in.sysinit") + # Read/Write access via ctypes + nlocal = lmp.extract_global("nlocal") + x = lmp.extract_atom("x") - nlocal = lmp.extract_global("nlocal") - x = lmp.extract_atom("x") + for i in range(nlocal): + print("(x,y,z) = (", x[i][0], x[i][1], x[i][2], ")") - for i in range(nlocal): - print("(x,y,z) = (", x[i][0], x[i][1], x[i][2], ")") + # Read/Write access via NumPy arrays + atom_id = L.numpy.extract_atom("id") + atom_type = L.numpy.extract_atom("type") + x = L.numpy.extract_atom("x") + v = L.numpy.extract_atom("v") + f = L.numpy.extract_atom("f") - lmp.close() + # set position in 2D simulation + x[0] = (1.0, 0.0) - **Methods**: + # set position in 3D simulation + x[0] = (1.0, 0.0, 1.) - * :py:meth:`extract_atom() `: extract a per-atom quantity + lmp.close() - **Numpy Methods**: - * :py:meth:`numpy.extract_atom() `: extract a per-atom quantity as numpy array +**Methods**: - .. tab:: PyLammps/IPyLammps API +* :py:meth:`extract_atom() `: extract a per-atom quantity - All atoms in the current simulation can be accessed by using the :py:attr:`PyLammps.atoms ` property. - Each element of this list is a :py:class:`Atom ` or :py:class:`Atom2D ` object. The attributes of - these objects provide access to their data (id, type, position, velocity, force, etc.): - - .. code-block:: python - - # access first atom - L.atoms[0].id - L.atoms[0].type - - # access second atom - L.atoms[1].position - L.atoms[1].velocity - L.atoms[1].force - - Some attributes can be changed: - - .. code-block:: python - - # set position in 2D simulation - L.atoms[0].position = (1.0, 0.0) - - # set position in 3D simulation - L.atoms[0].position = (1.0, 0.0, 1.0) +**Numpy Methods**: +* :py:meth:`numpy.extract_atom() `: extract a per-atom quantity as numpy array diff --git a/doc/src/Python_create.rst b/doc/src/Python_create.rst index 939aad2f32..9301829ea9 100644 --- a/doc/src/Python_create.rst +++ b/doc/src/Python_create.rst @@ -6,11 +6,10 @@ Creating or deleting a LAMMPS object ==================================== With the Python interface the creation of a :cpp:class:`LAMMPS -` instance is included in the constructors for the -:py:class:`lammps `, :py:class:`PyLammps `, -and :py:class:`IPyLammps ` classes. -Internally it will call either :cpp:func:`lammps_open` or :cpp:func:`lammps_open_no_mpi` from the C -library API to create the class instance. +` instance is included in the constructor for the +:py:class:`lammps ` class. Internally it will call either +:cpp:func:`lammps_open` or :cpp:func:`lammps_open_no_mpi` from the C library +API to create the class instance. All arguments are optional. The *name* argument allows loading a LAMMPS shared library that is named ``liblammps_machine.so`` instead of @@ -26,108 +25,25 @@ to run the Python module like the library interface on a subset of the MPI ranks after splitting the communicator. -Here are simple examples using all three Python interfaces: +Here is a simple example using the LAMMPS Python interface: -.. tabs:: +.. code-block:: python - .. tab:: lammps API + from lammps import lammps - .. code-block:: python + # NOTE: argv[0] is set by the lammps class constructor + args = ["-log", "none"] - from lammps import lammps + # create LAMMPS instance + lmp = lammps(cmdargs=args) - # NOTE: argv[0] is set by the lammps class constructor - args = ["-log", "none"] + # get and print numerical version code + print("LAMMPS Version: ", lmp.version()) - # create LAMMPS instance - lmp = lammps(cmdargs=args) + # explicitly close and delete LAMMPS instance (optional) + lmp.close() - # get and print numerical version code - print("LAMMPS Version: ", lmp.version()) - - # explicitly close and delete LAMMPS instance (optional) - lmp.close() - - .. tab:: PyLammps API - - The :py:class:`PyLammps ` class is a wrapper around the - :py:class:`lammps ` class and all of its lower level functions. - By default, it will create a new instance of :py:class:`lammps ` passing - along all arguments to the constructor of :py:class:`lammps `. - - .. code-block:: python - - from lammps import PyLammps - - # NOTE: argv[0] is set by the lammps class constructor - args = ["-log", "none"] - - # create LAMMPS instance - L = PyLammps(cmdargs=args) - - # get and print numerical version code - print("LAMMPS Version: ", L.version()) - - # explicitly close and delete LAMMPS instance (optional) - L.close() - - :py:class:`PyLammps ` objects can also be created on top of an existing - :py:class:`lammps ` object: - - .. code-block:: python - - from lammps import lammps, PyLammps - ... - # create LAMMPS instance - lmp = lammps(cmdargs=args) - - # create PyLammps instance using previously created LAMMPS instance - L = PyLammps(ptr=lmp) - - This is useful if you have to create the :py:class:`lammps ` - instance is a specific way, but want to take advantage of the - :py:class:`PyLammps ` interface. - - .. tab:: IPyLammps API - - The :py:class:`IPyLammps ` class is an extension of the - :py:class:`PyLammps ` class. It has the same construction behavior. By - default, it will create a new instance of :py:class:`lammps` passing - along all arguments to the constructor of :py:class:`lammps`. - - .. code-block:: python - - from lammps import IPyLammps - - # NOTE: argv[0] is set by the lammps class constructor - args = ["-log", "none"] - - # create LAMMPS instance - L = IPyLammps(cmdargs=args) - - # get and print numerical version code - print("LAMMPS Version: ", L.version()) - - # explicitly close and delete LAMMPS instance (optional) - L.close() - - You can also initialize IPyLammps on top of an existing :py:class:`lammps` or :py:class:`PyLammps` object: - - .. code-block:: python - - from lammps import lammps, IPyLammps - ... - # create LAMMPS instance - lmp = lammps(cmdargs=args) - - # create PyLammps instance using previously created LAMMPS instance - L = PyLammps(ptr=lmp) - - This is useful if you have to create the :py:class:`lammps ` - instance is a specific way, but want to take advantage of the - :py:class:`IPyLammps ` interface. - -In all of the above cases, same as with the :ref:`C library API `, this will use the +Same as with the :ref:`C library API `, this will use the ``MPI_COMM_WORLD`` communicator for the MPI library that LAMMPS was compiled with. diff --git a/doc/src/Python_execute.rst b/doc/src/Python_execute.rst index 01cf0e920f..a9d65133db 100644 --- a/doc/src/Python_execute.rst +++ b/doc/src/Python_execute.rst @@ -1,127 +1,123 @@ Executing commands ================== -Once an instance of the :py:class:`lammps `, -:py:class:`PyLammps `, or -:py:class:`IPyLammps ` class is created, there are +Once an instance of the :py:class:`lammps ` class is created, there are multiple ways to "feed" it commands. In a way that is not very different from running a LAMMPS input script, except that Python has many more facilities for structured programming than the LAMMPS input script syntax. Furthermore it is possible to "compute" what the next LAMMPS command should be. -.. tabs:: +Same as in the equivalent :doc:`C library functions `, +commands can be read from a file, a single string, a list of strings and a +block of commands in a single multi-line string. They are processed under the +same boundary conditions as the C library counterparts. The example below +demonstrates the use of :py:func:`lammps.file()`, :py:func:`lammps.command()`, +:py:func:`lammps.commands_list()`, and :py:func:`lammps.commands_string()`: - .. tab:: lammps API +.. code-block:: python - Same as in the equivalent - :doc:`C library functions `, commands can be read from a file, a - single string, a list of strings and a block of commands in a single - multi-line string. They are processed under the same boundary conditions - as the C library counterparts. The example below demonstrates the use - of :py:func:`lammps.file()`, :py:func:`lammps.command()`, - :py:func:`lammps.commands_list()`, and :py:func:`lammps.commands_string()`: + from lammps import lammps + lmp = lammps() - .. code-block:: python + # read commands from file 'in.melt' + lmp.file('in.melt') - from lammps import lammps - lmp = lammps() + # issue a single command + lmp.command('variable zpos index 1.0') - # read commands from file 'in.melt' - lmp.file('in.melt') + # create 10 groups with 10 atoms each + cmds = [f"group g{i} id {10*i+1}:{10*(i+1)}" for i in range(10)] + lmp.commands_list(cmds) - # issue a single command - lmp.command('variable zpos index 1.0') + # run commands from a multi-line string + block = """ + clear + region box block 0 2 0 2 0 2 + create_box 1 box + create_atoms 1 single 1.0 1.0 ${zpos} + """ + lmp.commands_string(block) - # create 10 groups with 10 atoms each - cmds = ["group g{} id {}:{}".format(i,10*i+1,10*(i+1)) for i in range(10)] - lmp.commands_list(cmds) +For convenience, the :py:class:`lammps ` class also provides a +command wrapper ``cmd`` that turns any LAMMPS command into a regular function +call. - # run commands from a multi-line string - block = """ - clear - region box block 0 2 0 2 0 2 - create_box 1 box - create_atoms 1 single 1.0 1.0 ${zpos} - """ - lmp.commands_string(block) +For instance, the following LAMMPS command - .. tab:: PyLammps/IPyLammps API +.. code-block:: LAMMPS - Unlike the lammps API, the PyLammps/IPyLammps APIs allow running LAMMPS - commands by calling equivalent member functions of :py:class:`PyLammps ` - and :py:class:`IPyLammps ` instances. + region box block 0 10 0 5 -0.5 0.5 - For instance, the following LAMMPS command +would normally be executed with the following Python code: - .. code-block:: LAMMPS +.. code-block:: python - region box block 0 10 0 5 -0.5 0.5 + from lammps import lammps - can be executed using with the lammps API with the following Python code if ``lmp`` is an - instance of :py:class:`lammps `: + lmp = lammps() + lmp.command("region box block 0 10 0 5 -0.5 0.5") - .. code-block:: python +With the ``cmd`` wrapper, any LAMMPS command can be split up into arbitrary parts. +These parts are then passed to a member function with the name of the :doc:`command `. +For the :doc:`region ` command that means the :code:`region()` method can be called. +The arguments of the command can be passed as one string, or +individually. - from lammps import lammps +.. code-block:: python - lmp = lammps() - lmp.command("region box block 0 10 0 5 -0.5 0.5") + from lammps import lammps - With the PyLammps interface, any LAMMPS command can be split up into arbitrary parts. - These parts are then passed to a member function with the name of the :doc:`command `. - For the :doc:`region ` command that means the :code:`region()` method can be called. - The arguments of the command can be passed as one string, or - individually. + L = lammps() - .. code-block:: python + # pass command parameters as one string + L.cmd.region("box block 0 10 0 5 -0.5 0.5") - from lammps import PyLammps + # OR pass them individually + L.cmd.region("box block", 0, 10, 0, 5, -0.5, 0.5) - L = PyLammps() +In the latter example, all parameters except the first are Python floating-point literals. The +member function takes the entire parameter list and transparently merges it to a single command +string. - # pass command parameters as one string - L.region("box block 0 10 0 5 -0.5 0.5") +The benefit of this approach is avoiding redundant command calls and easier +parameterization. With `command`, `commands_list`, and `commands_string` the +parameterization needed to be done manually by creating formatted command +strings. - # OR pass them individually - L.region("box block", 0, 10, 0, 5, -0.5, 0.5) +.. code-block:: python - In the latter example, all parameters except the first are Python floating-point literals. The - member function takes the entire parameter list and transparently merges it to a single command - string. + lmp.command("region box block %f %f %f %f %f %f" % (xlo, xhi, ylo, yhi, zlo, zhi)) - The benefit of this approach is avoiding redundant command calls and easier - parameterization. In the lammps API parameterization needed to be done - manually by creating formatted command strings. +In contrast, methods of the `cmd` wrapper accept parameters directly and will convert +them automatically to a final command string. - .. code-block:: python +.. code-block:: python - lmp.command("region box block %f %f %f %f %f %f" % (xlo, xhi, ylo, yhi, zlo, zhi)) + L.cmd.region("box block", xlo, xhi, ylo, yhi, zlo, zhi) - In contrast, methods of PyLammps accept parameters directly and will convert - them automatically to a final command string. +.. note:: - .. code-block:: python + When running in IPython you can use Tab-completion after ``L.cmd.`` to see + all available LAMMPS commands. - L.region("box block", xlo, xhi, ylo, yhi, zlo, zhi) +Using these facilities, the previous example shown above can be rewritten as follows: - Using these facilities, the example shown for the lammps API can be rewritten as follows: +.. code-block:: python - .. code-block:: python + from lammps import lammps + L = lammps() - from lammps import PyLammps - L = PyLammps() + # read commands from file 'in.melt' + L.file('in.melt') - # read commands from file 'in.melt' - L.file('in.melt') + # issue a single command + L.cmd.variable('zpos', 'index', 1.0) - # issue a single command - L.variable('zpos', 'index', 1.0) + # create 10 groups with 10 atoms each + for i in range(10): + L.cmd.group(f"g{i}", "id", f"{10*i+1}:{10*(i+1)}") - # create 10 groups with 10 atoms each - for i in range(10): - L.group(f"g{i}", "id", f"{10*i+1}:{10*(i+1)}") - - L.clear() - L.region("box block", 0, 2, 0, 2, 0, 2) - L.create_box(1, "box") - L.create_atoms(1, "single", 1.0, 1.0, "${zpos}") + L.cmd.clear() + L.cmd.region("box block", 0, 2, 0, 2, 0, 2) + L.cmd.create_box(1, "box") + L.cmd.create_atoms(1, "single", 1.0, 1.0, "${zpos}") diff --git a/doc/src/Python_head.rst b/doc/src/Python_head.rst index 3aab3a0d4b..28b6f3d1d4 100644 --- a/doc/src/Python_head.rst +++ b/doc/src/Python_head.rst @@ -15,6 +15,7 @@ together. Python_call Python_formats Python_examples + Python_jupyter Python_error Python_trouble diff --git a/doc/src/Python_jupyter.rst b/doc/src/Python_jupyter.rst new file mode 100644 index 0000000000..9e9a1c917e --- /dev/null +++ b/doc/src/Python_jupyter.rst @@ -0,0 +1,45 @@ +Using LAMMPS in IPython notebooks and Jupyter +============================================= + +If the LAMMPS Python package is installed for the same Python interpreter as +`IPython `_, you can use LAMMPS directly inside of an IPython notebook inside of +Jupyter. `Jupyter `_ is a powerful integrated development environment (IDE) for +many dynamic languages like Python, Julia and others, which operates inside of +any web browser. Besides auto-completion and syntax highlighting it allows you +to create formatted documents using Markup, mathematical formulas, graphics and +animations intermixed with executable Python code. It is a great format for +tutorials and showcasing your latest research. + +The easiest way to install it is via ``pip``: + +.. code-block:: bash + + pip install --user jupyter + +To launch an instance of Jupyter simply run the following command inside your +Python environment: + +.. code-block:: bash + + jupyter notebook + +Interactive Python Examples +--------------------------- + +Examples of IPython notebooks can be found in the ``python/examples/ipython`` +subdirectory. They require LAMMPS to be compiled as shared library with PYTHON, +PNG, JPEG and FFMPEG support. + +To open these notebooks launch ``jupyter notebook index.ipynb`` inside this +directory. The opened file provides an overview of the available examples. + +- Example 1: Using LAMMPS with Python (``simple.ipynb``) +- Example 2: Analyzing LAMMPS thermodynamic data (``thermo.ipynb``) +- Example 3: Working with Per-Atom Data (``atoms.ipynb``) +- Example 4: Working with LAMMPS variables (``variables.ipynb``) +- Example 5: Validating a dihedral potential (``dihedrals/dihedral.ipynb``) +- Example 6: Running a Monte Carlo relaxation (``montecarlo/mc.ipynb``) + +.. note:: + + Typically clicking a link in Jupyter will open a new tab, which might be blocked by your pop-up blocker. diff --git a/doc/src/Python_module.rst b/doc/src/Python_module.rst index c19d4b0345..30e585d143 100644 --- a/doc/src/Python_module.rst +++ b/doc/src/Python_module.rst @@ -10,19 +10,11 @@ be installed into a Python system folder or a user folder with ``make install-python``. Components of the module can then loaded into a Python session with the ``import`` command. -There are multiple Python interface classes in the :py:mod:`lammps` module: +.. warning:: -- the :py:class:`lammps ` class. This is a wrapper around - the C-library interface and its member functions try to replicate the - :ref:`C-library API ` closely. This is the most - feature-complete Python API. -- the :py:class:`PyLammps ` class. This is a more high-level - and more Python style class implemented on top of the - :py:class:`lammps ` class. -- the :py:class:`IPyLammps ` class is derived from - :py:class:`PyLammps ` and adds embedded graphics - features to conveniently include LAMMPS into `Jupyter - `_ notebooks. + Alternative interfaces such as :py:class:`PyLammps ` and + :py:class:`IPyLammps ` classes have been deprecated and + will be removed in a future version of LAMMPS. .. _mpi4py_url: https://mpi4py.readthedocs.io @@ -49,7 +41,7 @@ The ``lammps`` class API ======================== The :py:class:`lammps ` class is the core of the LAMMPS -Python interfaces. It is a wrapper around the :ref:`LAMMPS C library +Python interface. It is a wrapper around the :ref:`LAMMPS C library API ` using the `Python ctypes module `_ and a shared library compiled from the LAMMPS sources code. The individual methods in this @@ -64,40 +56,7 @@ functions. Below is a detailed documentation of the API. .. autoclass:: lammps.numpy_wrapper::numpy_wrapper :members: ----------- - -The ``PyLammps`` class API -========================== - -The :py:class:`PyLammps ` class is a wrapper that creates a -simpler, more "Pythonic" interface to common LAMMPS functionality. LAMMPS -data structures are exposed through objects and properties. This makes Python -scripts shorter and more concise. See the :doc:`PyLammps Tutorial -` for an introduction on how to use this interface. - -.. autoclass:: lammps.PyLammps - :members: - -.. autoclass:: lammps.AtomList - :members: - -.. autoclass:: lammps.Atom - :members: - -.. autoclass:: lammps.Atom2D - :members: - ----------- - -The ``IPyLammps`` class API -=========================== - -The :py:class:`IPyLammps ` class is an extension of -:py:class:`PyLammps `, adding additional functions to -quickly display visualizations such as images and videos inside of IPython. -See the :doc:`PyLammps Tutorial ` for examples. - -.. autoclass:: lammps.IPyLammps +.. autoclass:: lammps.ipython::wrapper :members: ---------- diff --git a/doc/src/Python_objects.rst b/doc/src/Python_objects.rst index 6e3a329a27..c3002ec5e6 100644 --- a/doc/src/Python_objects.rst +++ b/doc/src/Python_objects.rst @@ -4,95 +4,52 @@ Compute, fixes, variables This section documents accessing or modifying data from objects like computes, fixes, or variables in LAMMPS using the :py:mod:`lammps` module. -.. tabs:: +For :py:meth:`lammps.extract_compute() ` and +:py:meth:`lammps.extract_fix() `, the global, per-atom, +or local data calculated by the compute or fix can be accessed. What is returned +depends on whether the compute or fix calculates a scalar or vector or array. +For a scalar, a single double value is returned. If the compute or fix calculates +a vector or array, a pointer to the internal LAMMPS data is returned, which you can +use via normal Python subscripting. - .. tab:: lammps API +The one exception is that for a fix that calculates a +global vector or array, a single double value from the vector or array +is returned, indexed by I (vector) or I and J (array). I,J are +zero-based indices. +See the :doc:`Howto output ` page for a discussion of +global, per-atom, and local data, and of scalar, vector, and array +data types. See the doc pages for individual :doc:`computes ` +and :doc:`fixes ` for a description of what they calculate and +store. - For :py:meth:`lammps.extract_compute() ` and - :py:meth:`lammps.extract_fix() `, the global, per-atom, - or local data calculated by the compute or fix can be accessed. What is returned - depends on whether the compute or fix calculates a scalar or vector or array. - For a scalar, a single double value is returned. If the compute or fix calculates - a vector or array, a pointer to the internal LAMMPS data is returned, which you can - use via normal Python subscripting. +For :py:meth:`lammps.extract_variable() `, +an :doc:`equal-style or atom-style variable ` is evaluated and +its result returned. - The one exception is that for a fix that calculates a - global vector or array, a single double value from the vector or array - is returned, indexed by I (vector) or I and J (array). I,J are - zero-based indices. - See the :doc:`Howto output ` page for a discussion of - global, per-atom, and local data, and of scalar, vector, and array - data types. See the doc pages for individual :doc:`computes ` - and :doc:`fixes ` for a description of what they calculate and - store. +For equal-style variables a single ``c_double`` value is returned and the +group argument is ignored. For atom-style variables, a vector of +``c_double`` is returned, one value per atom, which you can use via normal +Python subscripting. The values will be zero for atoms not in the +specified group. - For :py:meth:`lammps.extract_variable() `, - an :doc:`equal-style or atom-style variable ` is evaluated and - its result returned. +:py:meth:`lammps.numpy.extract_compute() `, +:py:meth:`lammps.numpy.extract_fix() `, and +:py:meth:`lammps.numpy.extract_variable() ` are +equivalent NumPy implementations that return NumPy arrays instead of ``ctypes`` pointers. - For equal-style variables a single ``c_double`` value is returned and the - group argument is ignored. For atom-style variables, a vector of - ``c_double`` is returned, one value per atom, which you can use via normal - Python subscripting. The values will be zero for atoms not in the - specified group. +The :py:meth:`lammps.set_variable() ` method sets an +existing string-style variable to a new string value, so that subsequent LAMMPS +commands can access the variable. - :py:meth:`lammps.numpy.extract_compute() `, - :py:meth:`lammps.numpy.extract_fix() `, and - :py:meth:`lammps.numpy.extract_variable() ` are - equivalent NumPy implementations that return NumPy arrays instead of ``ctypes`` pointers. +**Methods**: - The :py:meth:`lammps.set_variable() ` method sets an - existing string-style variable to a new string value, so that subsequent LAMMPS - commands can access the variable. +* :py:meth:`lammps.extract_compute() `: extract value(s) from a compute +* :py:meth:`lammps.extract_fix() `: extract value(s) from a fix +* :py:meth:`lammps.extract_variable() `: extract value(s) from a variable +* :py:meth:`lammps.set_variable() `: set existing named string-style variable to value - **Methods**: +**NumPy Methods**: - * :py:meth:`lammps.extract_compute() `: extract value(s) from a compute - * :py:meth:`lammps.extract_fix() `: extract value(s) from a fix - * :py:meth:`lammps.extract_variable() `: extract value(s) from a variable - * :py:meth:`lammps.set_variable() `: set existing named string-style variable to value - - **NumPy Methods**: - - * :py:meth:`lammps.numpy.extract_compute() `: extract value(s) from a compute, return arrays as numpy arrays - * :py:meth:`lammps.numpy.extract_fix() `: extract value(s) from a fix, return arrays as numpy arrays - * :py:meth:`lammps.numpy.extract_variable() `: extract value(s) from a variable, return arrays as numpy arrays - - - .. tab:: PyLammps/IPyLammps API - - PyLammps and IPyLammps classes currently do not add any additional ways of - retrieving information out of computes and fixes. This information can still be accessed by using the lammps API: - - .. code-block:: python - - L.lmp.extract_compute(...) - L.lmp.extract_fix(...) - # OR - L.lmp.numpy.extract_compute(...) - L.lmp.numpy.extract_fix(...) - - LAMMPS variables can be both defined and accessed via the :py:class:`PyLammps ` interface. - - To define a variable you can use the :doc:`variable ` command: - - .. code-block:: python - - L.variable("a index 2") - - A dictionary of all variables is returned by the :py:attr:`PyLammps.variables ` property: - - you can access an individual variable by retrieving a variable object from the - ``L.variables`` dictionary by name - - .. code-block:: python - - a = L.variables['a'] - - The variable value can then be easily read and written by accessing the value - property of this object. - - .. code-block:: python - - print(a.value) - a.value = 4 +* :py:meth:`lammps.numpy.extract_compute() `: extract value(s) from a compute, return arrays as numpy arrays +* :py:meth:`lammps.numpy.extract_fix() `: extract value(s) from a fix, return arrays as numpy arrays +* :py:meth:`lammps.numpy.extract_variable() `: extract value(s) from a variable, return arrays as numpy arrays diff --git a/doc/src/Python_overview.rst b/doc/src/Python_overview.rst index a13da0d512..85bc0d3bfa 100644 --- a/doc/src/Python_overview.rst +++ b/doc/src/Python_overview.rst @@ -56,7 +56,7 @@ Below is an example output for Python version 3.8.5. --------- -LAMMPS can work together with Python in three ways. First, Python can +LAMMPS can work together with Python in two ways. First, Python can wrap LAMMPS through the its :doc:`library interface `, so that a Python script can create one or more instances of LAMMPS and launch one or more simulations. In Python terms, this is referred to as @@ -67,22 +67,7 @@ launch one or more simulations. In Python terms, this is referred to as Launching LAMMPS via Python - -Second, the lower-level Python interface in the :py:class:`lammps Python -class ` can be used indirectly through the provided -:py:class:`PyLammps ` and :py:class:`IPyLammps -` wrapper classes, also written in Python. These -wrappers try to simplify the usage of LAMMPS in Python by providing a -more object-based interface to common LAMMPS functionality. They also -reduce the amount of code necessary to parameterize LAMMPS scripts -through Python and make variables and computes directly accessible. - -.. figure:: JPG/pylammps-invoke-lammps.png - :figclass: align-center - - Using the PyLammps / IPyLammps wrappers - -Third, LAMMPS can use the Python interpreter, so that a LAMMPS input +Second, LAMMPS can use the Python interpreter, so that a LAMMPS input script or styles can invoke Python code directly, and pass information back-and-forth between the input script and Python functions you write. This Python code can also call back to LAMMPS to query or change its diff --git a/doc/src/Python_properties.rst b/doc/src/Python_properties.rst index 031461660a..25040ad5bd 100644 --- a/doc/src/Python_properties.rst +++ b/doc/src/Python_properties.rst @@ -2,14 +2,8 @@ System properties ================= Similar to what is described in :doc:`Library_properties`, the instances of -:py:class:`lammps `, :py:class:`PyLammps `, or -:py:class:`IPyLammps ` can be used to extract different kinds -of information about the active LAMMPS instance and also to modify some of it. The -main difference between the interfaces is how the information is exposed. - -While the :py:class:`lammps ` is just a thin layer that wraps C API calls, -:py:class:`PyLammps ` and :py:class:`IPyLammps ` expose -information as objects and properties. +:py:class:`lammps ` can be used to extract different kinds +of information about the active LAMMPS instance and also to modify some of it. In some cases the data returned is a direct reference to the original data inside LAMMPS cast to ``ctypes`` pointers. Where possible, the wrappers will @@ -25,113 +19,38 @@ against invalid accesses. accordingly. These arrays can change sizes and order at every neighbor list rebuild and atom sort event as atoms are migrating between subdomains. -.. tabs:: +.. code-block:: python - .. tab:: lammps API + from lammps import lammps - .. code-block:: python + lmp = lammps() + lmp.file("in.sysinit") - from lammps import lammps + natoms = lmp.get_natoms() + print(f"running simulation with {natoms} atoms") - lmp = lammps() - lmp.file("in.sysinit") + lmp.command("run 1000 post no"); - natoms = lmp.get_natoms() - print(f"running simulation with {natoms} atoms") + for i in range(10): + lmp.command("run 100 pre no post no") + pe = lmp.get_thermo("pe") + ke = lmp.get_thermo("ke") + print(f"PE = {pe}\nKE = {ke}") - lmp.command("run 1000 post no"); + lmp.close() - for i in range(10): - lmp.command("run 100 pre no post no") - pe = lmp.get_thermo("pe") - ke = lmp.get_thermo("ke") - print(f"PE = {pe}\nKE = {ke}") +**Methods**: - lmp.close() +* :py:meth:`version() `: return the numerical version id, e.g. LAMMPS 2 Sep 2015 -> 20150902 +* :py:meth:`get_thermo() `: return current value of a thermo keyword +* :py:meth:`last_thermo() `: return a dictionary of the last thermodynamic output +* :py:meth:`get_natoms() `: total # of atoms as int +* :py:meth:`reset_box() `: reset the simulation box size +* :py:meth:`extract_setting() `: return a global setting +* :py:meth:`extract_global() `: extract a global quantity +* :py:meth:`extract_box() `: extract box info +* :py:meth:`create_atoms() `: create N atoms with IDs, types, x, v, and image flags - **Methods**: +**Properties**: - * :py:meth:`version() `: return the numerical version id, e.g. LAMMPS 2 Sep 2015 -> 20150902 - * :py:meth:`get_thermo() `: return current value of a thermo keyword - * :py:meth:`last_thermo() `: return a dictionary of the last thermodynamic output - * :py:meth:`get_natoms() `: total # of atoms as int - * :py:meth:`reset_box() `: reset the simulation box size - * :py:meth:`extract_setting() `: return a global setting - * :py:meth:`extract_global() `: extract a global quantity - * :py:meth:`extract_box() `: extract box info - * :py:meth:`create_atoms() `: create N atoms with IDs, types, x, v, and image flags - - **Properties**: - - * :py:attr:`last_thermo_step `: the last timestep thermodynamic output was computed - - .. tab:: PyLammps/IPyLammps API - - In addition to the functions provided by :py:class:`lammps `, :py:class:`PyLammps ` objects - have several properties which allow you to query the system state: - - L.system - Is a dictionary describing the system such as the bounding box or number of atoms - - L.system.xlo, L.system.xhi - bounding box limits along x-axis - - L.system.ylo, L.system.yhi - bounding box limits along y-axis - - L.system.zlo, L.system.zhi - bounding box limits along z-axis - - L.communication - configuration of communication subsystem, such as the number of threads or processors - - L.communication.nthreads - number of threads used by each LAMMPS process - - L.communication.nprocs - number of MPI processes used by LAMMPS - - L.fixes - List of fixes in the current system - - L.computes - List of active computes in the current system - - L.dump - List of active dumps in the current system - - L.groups - List of groups present in the current system - - **Retrieving the value of an arbitrary LAMMPS expressions** - - LAMMPS expressions can be immediately evaluated by using the ``eval`` method. The - passed string parameter can be any expression containing global :doc:`thermo` values, - variables, compute or fix data (see :doc:`Howto_output`): - - - .. code-block:: python - - result = L.eval("ke") # kinetic energy - result = L.eval("pe") # potential energy - - result = L.eval("v_t/2.0") - - **Example** - - .. code-block:: python - - from lammps import PyLammps - - L = PyLammps() - L.file("in.sysinit") - - print(f"running simulation with {L.system.natoms} atoms") - - L.run(1000, "post no"); - - for i in range(10): - L.run(100, "pre no post no") - pe = L.eval("pe") - ke = L.eval("ke") - print(f"PE = {pe}\nKE = {ke}") +* :py:attr:`last_thermo_step `: the last timestep thermodynamic output was computed diff --git a/doc/src/Run_basics.rst b/doc/src/Run_basics.rst index 22cec5858b..f18eaff75a 100644 --- a/doc/src/Run_basics.rst +++ b/doc/src/Run_basics.rst @@ -1,8 +1,8 @@ Basics of running LAMMPS ======================== -LAMMPS is run from the command line, reading commands from a file via -the ``-in`` command line flag, or from standard input. Using the ``-in +LAMMPS is run from the command-line, reading commands from a file via +the ``-in`` command-line flag, or from standard input. Using the ``-in in.file`` variant is recommended (see note below). The name of the LAMMPS executable is either ``lmp`` or ``lmp_`` with `` being the machine string used when compiling LAMMPS. This @@ -25,7 +25,7 @@ build LAMMPS: You normally run the LAMMPS command in the directory where your input script is located. That is also where output files are produced by default, unless you provide specific other paths in your input script or -on the command line. As in some of the examples above, the LAMMPS +on the command-line. As in some of the examples above, the LAMMPS executable itself can be placed elsewhere. .. note:: diff --git a/doc/src/Run_options.rst b/doc/src/Run_options.rst index 4f7021cd53..546658a644 100644 --- a/doc/src/Run_options.rst +++ b/doc/src/Run_options.rst @@ -632,7 +632,7 @@ the ``-package omp`` command-line switch or the :doc:`package omp ` com The :doc:`suffix ` command can also be used within an input script to set a suffix, or to turn off or back on any suffix setting -made via the command line. +made via the command-line. ---------- diff --git a/doc/src/Run_output.rst b/doc/src/Run_output.rst index 2025bf5321..28ed891765 100644 --- a/doc/src/Run_output.rst +++ b/doc/src/Run_output.rst @@ -178,3 +178,64 @@ with and without the communication and a Gflop rate is computed. The 3d rate is with communication; the 1d rate is without (just the 1d FFTs). Thus you can estimate what fraction of your FFT time was spent in communication, roughly 75% in the example above. + +Error message output +==================== + +Depending on the error function arguments when it is called in the +source code, there will be one to four lines of error output. + +A single line +^^^^^^^^^^^^^ + +The line starts with "ERROR: ", followed by the error message and +information about the location in the source where the error function +was called in parenthesis on the right (here: line 131 of the file +src/fix_print.cpp). Example: + +.. parsed-literal:: + + ERROR: Fix print timestep variable nevery returned a bad timestep: 9900 (src/fix_print.cpp:131) + +Two lines +^^^^^^^^^ + +In addition to the single line output, also the last line of the input +will be repeated. If a command is spread over multiple lines in the +input using the continuation character '&', then the error will print +the entire concatenated line. For readability all whitespace is +compressed to single blanks. Example: + +.. parsed-literal:: + + ERROR: Unrecognized fix style 'printf' (src/modify.cpp:924) + Last input line: fix 0 all printf v_nevery "Step: $(step) ${step}" + +Three lines +^^^^^^^^^^^ + +In addition to the two line output from above, a third line is added +that uses caret character markers '^' to indicate which "word" in the +input failed. Example: + +.. parsed-literal:: + + ERROR: Illegal fix print nevery value -100; must be > 0 (src/fix_print.cpp:41) + Last input line: fix 0 all print -100 "Step: $(step) ${stepx}" + ^^^^ + +Four lines +^^^^^^^^^^ + +The three line output is expanded to four lines, if the the input is +modified through input pre-processing, e.g. when substituting +variables. Now the last command is printed once in the original form and +a second time after substitutions are applied. The caret character +markers '^' are applied to the second version. Example: + +.. parsed-literal:: + + ERROR: Illegal fix print nevery value -100; must be > 0 (src/fix_print.cpp:41) + Last input line: fix 0 all print ${nevery} 'Step: $(step) ${step}' + --> parsed line: fix 0 all print -100 "Step: $(step) ${step}" + ^^^^ diff --git a/doc/src/Run_windows.rst b/doc/src/Run_windows.rst index 8c2eac2bc6..a03f27c013 100644 --- a/doc/src/Run_windows.rst +++ b/doc/src/Run_windows.rst @@ -20,7 +20,7 @@ To run with 4 threads, you can type this: lmp -in in.lj.lmp -k on t 4 -sf kk Alternately, you can also install a package with LAMMPS-GUI included and -open the LAMMPS-GUI app (the package includes the command line version +open the LAMMPS-GUI app (the package includes the command-line version of LAMMPS as well) and open the input file in the GUI and run it from there. For details on LAMMPS-GUI, see :doc:`Howto_lammps_gui`. diff --git a/doc/src/Speed_gpu.rst b/doc/src/Speed_gpu.rst index 42bd8bf059..c90e3fcc5e 100644 --- a/doc/src/Speed_gpu.rst +++ b/doc/src/Speed_gpu.rst @@ -31,7 +31,8 @@ Coulombics. It has the following general features: (for Nvidia GPUs, AMD GPUs, Intel GPUs, and multicore CPUs). so that the same functionality is supported on a variety of hardware. -**Required hardware/software:** +Required hardware/software +"""""""""""""""""""""""""" To compile and use this package in CUDA mode, you currently need to have an NVIDIA GPU and install the corresponding NVIDIA CUDA @@ -69,12 +70,14 @@ To compile and use this package in HIP mode, you have to have the AMD ROCm software installed. Versions of ROCm older than 3.5 are currently deprecated by AMD. -**Building LAMMPS with the GPU package:** +Building LAMMPS with the GPU package +"""""""""""""""""""""""""""""""""""" See the :ref:`Build extras ` page for instructions. -**Run with the GPU package from the command line:** +Run with the GPU package from the command-line +"""""""""""""""""""""""""""""""""""""""""""""" The ``mpirun`` or ``mpiexec`` command sets the total number of MPI tasks used by LAMMPS (one or multiple per compute node) and the number of MPI @@ -133,7 +136,8 @@ affect the setting for bonded interactions (LAMMPS default is "on"). The "off" setting for pairwise interaction is currently required for GPU package pair styles. -**Or run with the GPU package by editing an input script:** +Run with the GPU package by editing an input script +""""""""""""""""""""""""""""""""""""""""""""""""""" The discussion above for the ``mpirun`` or ``mpiexec`` command, MPI tasks/node, and use of multiple MPI tasks/GPU is the same. @@ -149,7 +153,8 @@ You must also use the :doc:`package gpu ` command to enable the GPU package, unless the ``-sf gpu`` or ``-pk gpu`` :doc:`command-line switches ` were used. It specifies the number of GPUs/node to use, as well as other options. -**Speed-ups to expect:** +Speed-up to expect +"""""""""""""""""" The performance of a GPU versus a multicore CPU is a function of your hardware, which pair style is used, the number of atoms/GPU, and the @@ -176,10 +181,13 @@ better with multiple OMP threads because the inter-process communication is higher for these styles with the GPU package in order to allow deterministic results. -**Guidelines for best performance:** +Guidelines for best performance +""""""""""""""""""""""""""""""" -* Using multiple MPI tasks per GPU will often give the best performance, - as allowed my most multicore CPU/GPU configurations. +* Using multiple MPI tasks (2-10) per GPU will often give the best + performance, as allowed my most multicore CPU/GPU configurations. + Using too many MPI tasks will result in worse performance due to + growing overhead with the growing number of MPI tasks. * If the number of particles per MPI task is small (e.g. 100s of particles), it can be more efficient to run with fewer MPI tasks per GPU, even if you do not use all the cores on the compute node. @@ -199,12 +207,13 @@ deterministic results. :doc:`angle `, :doc:`dihedral `, :doc:`improper `, and :doc:`long-range ` calculations will not be included in the "Pair" time. -* Since only part of the pppm kspace style is GPU accelerated, it - may be faster to only use GPU acceleration for Pair styles with - long-range electrostatics. See the "pair/only" keyword of the - package command for a shortcut to do that. The work between kspace - on the CPU and non-bonded interactions on the GPU can be balanced - through adjusting the coulomb cutoff without loss of accuracy. +* Since only part of the pppm kspace style is GPU accelerated, it may be + faster to only use GPU acceleration for Pair styles with long-range + electrostatics. See the "pair/only" keyword of the :doc:`package + command ` for a shortcut to do that. The distribution of + work between kspace on the CPU and non-bonded interactions on the GPU + can be balanced through adjusting the coulomb cutoff without loss of + accuracy. * When the *mode* setting for the package gpu command is force/neigh, the time for neighbor list calculations on the GPU will be added into the "Pair" time, not the "Neigh" time. An additional breakdown of the @@ -220,4 +229,6 @@ deterministic results. Restrictions """""""""""" -None. +When using :doc:`hybrid pair styles `, the neighbor list +must be generated on the host instead of the GPU and thus the potential +GPU acceleration is reduced. diff --git a/doc/src/Speed_intel.rst b/doc/src/Speed_intel.rst index dd6c27b4e7..78a88f4407 100644 --- a/doc/src/Speed_intel.rst +++ b/doc/src/Speed_intel.rst @@ -1,5 +1,5 @@ INTEL package -================== +============= The INTEL package is maintained by Mike Brown at Intel Corporation. It provides two methods for accelerating simulations, @@ -13,18 +13,18 @@ twice, once on the CPU and once with an offload flag. This allows LAMMPS to run on the CPU cores and co-processor cores simultaneously. Currently Available INTEL Styles -""""""""""""""""""""""""""""""""""""" +"""""""""""""""""""""""""""""""" * Angle Styles: charmm, harmonic -* Bond Styles: fene, fourier, harmonic +* Bond Styles: fene, harmonic * Dihedral Styles: charmm, fourier, harmonic, opls -* Fixes: nve, npt, nvt, nvt/sllod, nve/asphere +* Fixes: nve, npt, nvt, nvt/sllod, nve/asphere, electrode/conp, electrode/conq, electrode/thermo * Improper Styles: cvff, harmonic * Pair Styles: airebo, airebo/morse, buck/coul/cut, buck/coul/long, buck, dpd, eam, eam/alloy, eam/fs, gayberne, lj/charmm/coul/charmm, lj/charmm/coul/long, lj/cut, lj/cut/coul/long, lj/long/coul/long, - rebo, sw, tersoff -* K-Space Styles: pppm, pppm/disp + rebo, snap, sw, tersoff +* K-Space Styles: pppm, pppm/disp, pppm/electrode .. warning:: @@ -33,7 +33,7 @@ Currently Available INTEL Styles input requires it, LAMMPS will abort with an error message. Speed-up to expect -""""""""""""""""""" +"""""""""""""""""" The speedup will depend on your simulation, the hardware, which styles are used, the number of atoms, and the floating-point @@ -312,21 +312,21 @@ almost all cases. recommended, especially when running on a machine with Intel Hyper-Threading technology disabled. -Run with the INTEL package from the command line -""""""""""""""""""""""""""""""""""""""""""""""""""""" +Run with the INTEL package from the command-line +"""""""""""""""""""""""""""""""""""""""""""""""" -To enable INTEL optimizations for all available styles used in -the input script, the ``-sf intel`` :doc:`command-line switch ` can be used without any requirement for -editing the input script. This switch will automatically append -"intel" to styles that support it. It also invokes a default command: -:doc:`package intel 1 `. This package command is used to set -options for the INTEL package. The default package command will -specify that INTEL calculations are performed in mixed precision, -that the number of OpenMP threads is specified by the OMP_NUM_THREADS -environment variable, and that if co-processors are present and the -binary was built with offload support, that 1 co-processor per node -will be used with automatic balancing of work between the CPU and the -co-processor. +To enable INTEL optimizations for all available styles used in the input +script, the ``-sf intel`` :doc:`command-line switch ` can +be used without any requirement for editing the input script. This +switch will automatically append "intel" to styles that support it. It +also invokes a default command: :doc:`package intel 1 `. This +package command is used to set options for the INTEL package. The +default package command will specify that INTEL calculations are +performed in mixed precision, that the number of OpenMP threads is +specified by the OMP_NUM_THREADS environment variable, and that if +co-processors are present and the binary was built with offload support, +that 1 co-processor per node will be used with automatic balancing of +work between the CPU and the co-processor. You can specify different options for the INTEL package by using the ``-pk intel Nphi`` :doc:`command-line switch ` with diff --git a/doc/src/Speed_kokkos.rst b/doc/src/Speed_kokkos.rst index 40e748c9fe..f8379949a4 100644 --- a/doc/src/Speed_kokkos.rst +++ b/doc/src/Speed_kokkos.rst @@ -67,6 +67,14 @@ version 23 November 2023 and Kokkos version 4.2. To build with Kokkos support for AMD GPUs, the AMD ROCm toolkit software version 5.2.0 or later must be installed on your system. +.. admonition:: Intel Data Center GPU support + :class: note + + Support for Kokkos with Intel Data Center GPU accelerators (formerly + known under the code name "Ponte Vecchio") in LAMMPS is still a work + in progress. Only a subset of the functionality works correctly. + Please contact the LAMMPS developers if you run into problems. + .. admonition:: CUDA and MPI library compatibility :class: note @@ -77,16 +85,18 @@ version 23 November 2023 and Kokkos version 4.2. rank. When running with multiple MPI ranks, you may see segmentation faults without GPU-aware MPI support. These can be avoided by adding the flags :doc:`-pk kokkos gpu/aware off ` to the - LAMMPS command line or by using the command :doc:`package kokkos + LAMMPS command-line or by using the command :doc:`package kokkos gpu/aware off ` in the input file. -.. admonition:: Intel Data Center GPU support +.. admonition:: Using multiple MPI ranks per GPU :class: note - Support for Kokkos with Intel Data Center GPU accelerators (formerly - known under the code name "Ponte Vecchio") in LAMMPS is still a work - in progress. Only a subset of the functionality works correctly. - Please contact the LAMMPS developers if you run into problems. + Unlike with the GPU package, there are limited benefits from using + multiple MPI processes per GPU with KOKKOS. But when doing this it + is **required** to enable CUDA MPS (`Multi-Process Service :: GPU + Deployment and Management Documentation + `_ ) to get acceptable + performance. Building LAMMPS with the KOKKOS package """"""""""""""""""""""""""""""""""""""" @@ -365,13 +375,13 @@ one or more nodes, each with two GPUs: .. note:: - When using a GPU, you will achieve the best performance if your - input script does not use fix or compute styles which are not yet + When using a GPU, you will achieve the best performance if your input + script does not use fix or compute styles which are not yet Kokkos-enabled. This allows data to stay on the GPU for multiple timesteps, without being copied back to the host CPU. Invoking a - non-Kokkos fix or compute, or performing I/O for - :doc:`thermo ` or :doc:`dump ` output will cause data - to be copied back to the CPU incurring a performance penalty. + non-Kokkos fix or compute, or performing I/O for :doc:`thermo + ` or :doc:`dump ` output will cause data to be + copied back to the CPU incurring a performance penalty. .. note:: @@ -379,6 +389,56 @@ one or more nodes, each with two GPUs: kspace, etc., you must set the environment variable ``CUDA_LAUNCH_BLOCKING=1``. However, this will reduce performance and is not recommended for production runs. +Troubleshooting segmentation faults on GPUs +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +As noted above, KOKKOS by default assumes that the MPI library is +GPU-aware. This is not always the case and can lead to segmentation +faults when using more than one MPI process. Normally, LAMMPS will +print a warning like "*Turning off GPU-aware MPI since it is not +detected*", or an error message like "*Kokkos with GPU-enabled backend +assumes GPU-aware MPI is available*", OR a **segmentation fault**. To +confirm that a segmentation fault is caused by this, you can turn off +the GPU-aware assumption via the :doc:`package kokkos command ` +or the corresponding command-line flag. + +If you still get a segmentation fault, despite running with only one MPI +process or using the command-line flag to turn off expecting a GPU-aware +MPI library, then using the CMake compile setting +``-DKokkos_ENABLE_DEBUG=on`` or adding ``KOKKOS_DEBUG=yes`` to your +machine makefile for building with traditional make will generate useful +output that can be passed to the LAMMPS developers for further +debugging. + +Troubleshooting memory allocation on GPUs +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +`Kokkos Tools `_ provides a set +of lightweight profiling and debugging utilities, which interface with +instrumentation hooks (eg. `space-time-stack +`_) +built directly into the Kokkos runtime. After compiling a dynamic LAMMPS +library, you then have to set the environment variable ``KOKKOS_TOOLS_LIBS`` +before executing your LAMMPS Kokkos run. Example: + +.. code-block:: bash + + export KOKKOS_TOOLS_LIBS=${HOME}/kokkos-tools/src/tools/memory-events/kp_memory_event.so + mpirun -np 4 lmp_kokkos_cuda_openmpi -in in.lj -k on g 4 -sf kk + +Starting with the NVIDIA Pascal GPU architecture, CUDA supports +`"Unified Virtual Memory" (UVM) +`_ +which enables allocating more memory than a GPU possesses by also using +memory on the host CPU and then CUDA will transparently move data +between CPU and GPU as needed. The resulting LAMMPS performance depends +on `memory access pattern, data residency, and GPU memory +oversubscription +`_ +. The CMake option ``-DKokkos_ENABLE_CUDA_UVM=on`` or the makefile +setting ``KOKKOS_CUDA_OPTIONS=enable_lambda,force_uvm`` enables using +:ref:`UVM with Kokkos ` when compiling LAMMPS. + Run with the KOKKOS package by editing an input script ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -423,7 +483,7 @@ in the ``kokkos-cuda.cmake`` CMake preset file. cmake -DKokkos_ENABLE_CUDA=yes -DKokkos_ENABLE_OPENMP=yes ../cmake The suffix "/kk" is equivalent to "/kk/device", and for Kokkos CUDA, -using the ``-sf kk`` in the command line gives the default CUDA version +using the ``-sf kk`` in the command-line gives the default CUDA version everywhere. However, if the "/kk/host" suffix is added to a specific style in the input script, the Kokkos OpenMP (CPU) version of that specific style will be used instead. Set the number of OpenMP threads @@ -439,7 +499,7 @@ For example, the command to run with 1 GPU and 8 OpenMP threads is then: mpiexec -np 1 lmp_kokkos_cuda_openmpi -in in.lj -k on g 1 t 8 -sf kk -Conversely, if the ``-sf kk/host`` is used in the command line and then +Conversely, if the ``-sf kk/host`` is used in the command-line and then the "/kk" or "/kk/device" suffix is added to a specific style in your input script, then only that specific style will run on the GPU while everything else will run on the CPU in OpenMP mode. Note that the @@ -451,7 +511,7 @@ on the host CPU can overlap with a pair style running on the GPU. First compile with ``--default-stream per-thread`` added to ``CCFLAGS`` in the Kokkos CUDA Makefile. Then explicitly use the "/kk/host" suffix for kspace and bonds, angles, etc. in the input file and the -"kk" suffix (equal to "kk/device") on the command line. Also make +"kk" suffix (equal to "kk/device") on the command-line. Also make sure the environment variable ``CUDA_LAUNCH_BLOCKING`` is not set to "1" so CPU/GPU overlap can occur. diff --git a/doc/src/Speed_omp.rst b/doc/src/Speed_omp.rst index bf0744f0f4..906acf3ce8 100644 --- a/doc/src/Speed_omp.rst +++ b/doc/src/Speed_omp.rst @@ -21,7 +21,7 @@ Building LAMMPS with the OPENMP package See the :ref:`Build extras ` page for instructions. -Run with the OPENMP package from the command line +Run with the OPENMP package from the command-line """"""""""""""""""""""""""""""""""""""""""""""""""" These examples assume one or more 16-core nodes. diff --git a/doc/src/Speed_opt.rst b/doc/src/Speed_opt.rst index e115393017..15e18dad8e 100644 --- a/doc/src/Speed_opt.rst +++ b/doc/src/Speed_opt.rst @@ -17,7 +17,7 @@ Building LAMMPS with the OPT package See the :ref:`Build extras ` page for instructions. -Run with the OPT package from the command line +Run with the OPT package from the command-line """""""""""""""""""""""""""""""""""""""""""""" .. code-block:: bash diff --git a/doc/src/Tools.rst b/doc/src/Tools.rst index ba7cb2035a..d13a6d384f 100644 --- a/doc/src/Tools.rst +++ b/doc/src/Tools.rst @@ -501,7 +501,7 @@ Here are a few highlights of LAMMPS-GUI - Indicator for line that caused an error - Visualization of current state in Image Viewer (via calling :doc:`write_dump image `) - Capture of images created via :doc:`dump image ` in Slide show window -- Dialog to set variables, similar to the LAMMPS command line flag '-v' / '-var' +- Dialog to set variables, similar to the LAMMPS command-line flag '-v' / '-var' - Support for GPU, INTEL, KOKKOS/OpenMP, OPENMAP, and OPT and accelerator packages Parallelization @@ -550,7 +550,7 @@ will be found automatically. 2) you can download the `Flatpak file *flatpak* command: ``flatpak install --user LAMMPS-Linux-x86_64-GUI-.flatpak`` and run it with ``flatpak run org.lammps.lammps-gui``. The flatpak bundle also includes the -command line version of LAMMPS and some LAMMPS tools like msi2lmp. The +command-line version of LAMMPS and some LAMMPS tools like msi2lmp. The can be launched by using the ``--command`` flag. For example to run LAMMPS directly on the ``in.lj`` benchmark input you would type in the ``bench`` folder: ``flatpak run --command=lmp -in in.lj`` The flatpak @@ -608,10 +608,10 @@ would be the ``examples/COUPLE/plugin`` folder of the LAMMPS distribution. When compiling LAMMPS-GUI with plugin support, there is an additional -command line flag (``-p `` or ``--pluginpath ``) which +command-line flag (``-p `` or ``--pluginpath ``) which allows to override the path to LAMMPS shared library used by the GUI. This is usually auto-detected on the first run and can be changed in the -LAMMPS-GUI *Preferences* dialog. The command line flag allows to reset +LAMMPS-GUI *Preferences* dialog. The command-line flag allows to reset this path to a valid value in case the original setting has become invalid. An empty path ("") as argument restores the default setting. @@ -656,7 +656,7 @@ it will create a compressed ``LAMMPS-Win10-amd64.zip`` zip file with the executables and required dependent .dll files. This zip file can be uncompressed and ``lammps-gui.exe`` run directly from there. The uncompressed folder can be added to the ``PATH`` environment and LAMMPS -and LAMMPS-GUI can be launched from anywhere from the command line. +and LAMMPS-GUI can be launched from anywhere from the command-line. **MinGW64 Cross-compiler** @@ -876,7 +876,7 @@ the same ``LAMMPS_CACHING_DIR``. This script does the following: #. Start a simple local HTTP server using Python to host files for CMake Afterwards, it will print out instruction on how to modify the CMake -command line to make sure it uses the local HTTP server. +commands to make sure it uses the local HTTP server. To undo the environment changes and shutdown the local HTTP server, run the ``deactivate_caches`` command. @@ -1025,7 +1025,7 @@ with those in the provided log file with the same number of processors in the same subdirectory. If the differences between the actual and reference values are within specified tolerances, the test is considered passed. For each test batch, that is, a set of example input scripts, -the mpirun command, the LAMMPS command line arguments, and the +the mpirun command, the LAMMPS command-line arguments, and the tolerances for individual thermo quantities can be specified in a configuration file in YAML format. diff --git a/doc/src/angle_mwlc.rst b/doc/src/angle_mwlc.rst new file mode 100644 index 0000000000..9fa4171a63 --- /dev/null +++ b/doc/src/angle_mwlc.rst @@ -0,0 +1,94 @@ +.. index:: angle_style mwlc + +angle_style mwlc command +========================== + +Syntax +"""""" + +.. code-block:: LAMMPS + + angle_style mwlc + +Examples +"""""""" + +.. code-block:: LAMMPS + + angle_style mwlc + angle_coeff * 25 1 10 1 + +Description +""""""""""" + +.. versionadded:: TBD + +The *mwlc* angle style models a meltable wormlike chain and can be used +to model non-linear bending elasticity of polymers, e.g. DNA. *mwlc* +uses a potential that is a canonical-ensemble superposition of a +non-melted and a melted state :ref:`(Farrell) `. The potential +is + +.. math:: + + E = -k_{B}T\,\log [q + q^{m}] + E_{0}, + +where the non-melted and melted partition functions are + +.. math:: + q = \exp [-k_{1}(1+\cos{\theta})/k_{B}T]; \\ + q^{m} = \exp [-(\mu+k_{2}(1+\cos{\theta}))/k_{B}T]. + +:math:`k_1` is the bending elastic constant of the non-melted state, +:math:`k_2` is the bending elastic constant of the melted state, +:math:`\mu` is the melting energy, and +:math:`T` is the reference temperature. +The reference energy, + +.. math:: + E_{0} = -k_{B}T\,\log [1 + \exp[-\mu/k_{B}T]], + +ensures that E is zero for a fully extended chain. + +This potential is a continuous version of the two-state potential +introduced by :ref:`(Yan) `. + +The following coefficients must be defined for each angle type via the +:doc:`angle_coeff ` command as in the example above, or in +the data file or restart files read by the :doc:`read_data ` +or :doc:`read_restart ` commands: + +* :math:`k_1` (energy) +* :math:`k_2` (energy) +* :math:`\mu` (energy) +* :math:`T` (temperature) + +---------- + + +Restrictions +"""""""""""" + +This angle style can only be used if LAMMPS was built with the +EXTRA-MOLECULE package. See the :doc:`Build package ` +doc page for more info. + +Related commands +"""""""""""""""" + +:doc:`angle_coeff ` + +Default +""""""" + +none + +---------- + +.. _Farrell: + +**(Farrell)** `Farrell, Dobnikar, Podgornik, Curk, Phys Rev Lett, 133, 148101 (2024). `_ + +.. _Yan: + +**(Yan)** `Yan, Marko, Phys Rev Lett, 93, 108108 (2004). `_ diff --git a/doc/src/angle_style.rst b/doc/src/angle_style.rst index dc16a3fbaa..ab7671c75d 100644 --- a/doc/src/angle_style.rst +++ b/doc/src/angle_style.rst @@ -94,6 +94,7 @@ of (g,i,k,o,t) to indicate which accelerated styles exist. * :doc:`lepton ` - angle potential from evaluating a string * :doc:`mesocnt ` - piecewise harmonic and linear angle for bending-buckling of nanotubes * :doc:`mm3 ` - anharmonic angle +* :doc:`mwlc ` - meltable wormlike chain * :doc:`quartic ` - angle with cubic and quartic terms * :doc:`spica ` - harmonic angle with repulsive SPICA pair style between 1-3 atoms * :doc:`table ` - tabulated by angle diff --git a/doc/src/bond_bpm_rotational.rst b/doc/src/bond_bpm_rotational.rst index e4cfef3d51..cfbd01dd99 100644 --- a/doc/src/bond_bpm_rotational.rst +++ b/doc/src/bond_bpm_rotational.rst @@ -155,7 +155,7 @@ page on BPMs. If the *break* keyword is set to *no*, LAMMPS assumes bonds should not break during a simulation run. This will prevent some unnecessary calculation. The recommended bond communication distance no longer depends on bond failure -coefficients (which are ignored) but instead corresponds to the typical heurestic +coefficients (which are ignored) but instead corresponds to the typical heuristic maximum strain used by typical non-bpm bond styles. Similar behavior to *break no* can also be attained by setting arbitrarily high values for all four failure coefficients. One cannot use *break no* with *smooth yes*. diff --git a/doc/src/bond_bpm_spring.rst b/doc/src/bond_bpm_spring.rst index c868a47a12..0a43a62159 100644 --- a/doc/src/bond_bpm_spring.rst +++ b/doc/src/bond_bpm_spring.rst @@ -119,7 +119,7 @@ If the *break* keyword is set to *no*, LAMMPS assumes bonds should not break during a simulation run. This will prevent some unnecessary calculation. The recommended bond communication distance no longer depends on the value of :math:`\epsilon_c` (which is ignored) but instead corresponds to the typical -heurestic maximum strain used by typical non-bpm bond styles. Similar behavior +heuristic maximum strain used by typical non-bpm bond styles. Similar behavior to *break no* can also be attained by setting an arbitrarily high value of :math:`\epsilon_c`. One cannot use *break no* with *smooth yes*. diff --git a/doc/src/compute.rst b/doc/src/compute.rst index 082f93a6c4..9a8a1734fb 100644 --- a/doc/src/compute.rst +++ b/doc/src/compute.rst @@ -236,6 +236,7 @@ The individual style names on the :doc:`Commands compute ` pag * :doc:`fep/ta ` - compute free energies for a test area perturbation * :doc:`force/tally ` - force between two groups of atoms via the tally callback mechanism * :doc:`fragment/atom ` - fragment ID for each atom +* :doc:`gaussian/grid/local ` - local array of Gaussian atomic contributions on a regular grid * :doc:`global/atom ` - assign global values to each atom from arrays of global values * :doc:`group/group ` - energy/force between two groups of atoms * :doc:`gyration ` - radius of gyration of group of atoms diff --git a/doc/src/compute_gaussian_grid_local.rst b/doc/src/compute_gaussian_grid_local.rst new file mode 100644 index 0000000000..46fac9fab3 --- /dev/null +++ b/doc/src/compute_gaussian_grid_local.rst @@ -0,0 +1,99 @@ +.. index:: compute gaussian/grid/local +.. index:: compute gaussian/grid/local/kk + +compute gaussian/grid/local command +=================================== + +Accelerator Variants: *gaussian/grid/local/kk* + +Syntax +"""""" + +.. code-block:: LAMMPS + + compute ID group-ID gaussian/grid/local grid nx ny nz rcutfac R_1 R_2 ... sigma_1 sigma_2 + +* ID, group-ID are documented in :doc:`compute ` command +* gaussian/grid/local = style name of this compute command +* *grid* values = nx, ny, nz, number of grid points in x, y, and z directions (positive integer) +* *rcutfac* = scale factor applied to all cutoff radii (positive real) +* *R_1, R_2,...* = list of cutoff radii, one for each type (distance units) +* *sigma_1, sigma_2,...* = Gaussian widths, one for each type (distance units) + +Examples +"""""""" + +.. code-block:: LAMMPS + + compute mygrid all gaussian/grid/local grid 40 40 40 4.0 0.5 0.5 0.4 0.4 + +Description +""""""""""" + +.. versionadded:: TBD + +Define a computation that calculates a Gaussian representation of the ionic +structure. This representation is used for the efficient evaluation +of quantities related to the structure factor in a grid-based workflow, +such as the ML-DFT workflow MALA :ref:`(Ellis) `, for which it was originally +implemented. Usage of the workflow is described in a separate publication :ref:`(Fiedler) `. + +For each LAMMPS type, a separate sum of Gaussians is calculated, using +a separate Gaussian broadening per type. The computation +is always performed on the numerical grid, no atom-based version of this +compute exists. The Gaussian representation can only be executed in a local +fashion, thus the output array only contains rows for grid points +that are local to the processor subdomain. The layout of the grid is the same +as for the see :doc:`sna/grid/local ` command. + +Namely, the array contains one row for each of the +local grid points, looping over the global index *ix* fastest, +then *iy*, and *iz* slowest. Each row of the array contains +the global indexes *ix*, *iy*, and *iz* first, followed by the *x*, *y*, +and *z* coordinates of the grid point, followed by the values of the Gaussians +(one floating point number per type per grid point). + +---------- + + +.. include:: accel_styles.rst + + + +---------- + +Output info +""""""""""" + +Compute *gaussian/grid/local* evaluates a local array. +The array contains one row for each of the +local grid points, looping over the global index *ix* fastest, +then *iy*, and *iz* slowest. The array contains math :math:`ntypes+6` columns, +where *ntypes* is the number of LAMMPS types. The first three columns are +the global indexes *ix*, *iy*, and *iz*, followed by the *x*, *y*, +and *z* coordinates of the grid point, followed by the *ntypes* columns +containing the values of the Gaussians for each type. + +Restrictions +"""""""""""" + +These computes are part of the ML-SNAP package. They are only enabled +if LAMMPS was built with that package. See the :doc:`Build package +` page for more info. + +Related commands +"""""""""""""""" + +:doc:`compute sna/grid/local ` + +---------- + +.. _Ellis2021b: + +**(Ellis)** Ellis, Fiedler, Popoola, Modine, Stephens, Thompson, Cangi, Rajamanickam, `Phys. Rev. B, 104, 035120, (2021) `_ + +.. _Fiedler2023: + +**(Fiedler)** Fiedler, Modine, Schmerler, Vogel, Popoola, Thompson, Rajamanickam, and Cangi, +`npj Comp. Mater., 9, 115 (2023) `_ + diff --git a/doc/src/compute_sna_atom.rst b/doc/src/compute_sna_atom.rst index 179c362dc6..2572093499 100644 --- a/doc/src/compute_sna_atom.rst +++ b/doc/src/compute_sna_atom.rst @@ -3,7 +3,9 @@ .. index:: compute snav/atom .. index:: compute snap .. index:: compute sna/grid +.. index:: compute sna/grid/kk .. index:: compute sna/grid/local +.. index:: compute sna/grid/local/kk compute sna/atom command ======================== @@ -20,9 +22,14 @@ compute snap command compute sna/grid command ======================== +compute sna/grid/kk command +=========================== + compute sna/grid/local command ============================== +Accelerator Variants: *sna/grid/local/kk* + Syntax """""" @@ -33,17 +40,17 @@ Syntax compute ID group-ID snav/atom rcutfac rfac0 twojmax R_1 R_2 ... w_1 w_2 ... keyword values ... compute ID group-ID snap rcutfac rfac0 twojmax R_1 R_2 ... w_1 w_2 ... keyword values ... compute ID group-ID snap rcutfac rfac0 twojmax R_1 R_2 ... w_1 w_2 ... keyword values ... - compute ID group-ID sna/grid nx ny nz rcutfac rfac0 twojmax R_1 R_2 ... w_1 w_2 ... keyword values ... - compute ID group-ID sna/grid/local nx ny nz rcutfac rfac0 twojmax R_1 R_2 ... w_1 w_2 ... keyword values ... + compute ID group-ID sna/grid grid nx ny nz rcutfac rfac0 twojmax R_1 R_2 ... w_1 w_2 ... keyword values ... + compute ID group-ID sna/grid/local grid nx ny nz rcutfac rfac0 twojmax R_1 R_2 ... w_1 w_2 ... keyword values ... * ID, group-ID are documented in :doc:`compute ` command * sna/atom = style name of this compute command -* rcutfac = scale factor applied to all cutoff radii (positive real) -* rfac0 = parameter in distance to angle conversion (0 < rcutfac < 1) -* twojmax = band limit for bispectrum components (non-negative integer) -* R_1, R_2,... = list of cutoff radii, one for each type (distance units) -* w_1, w_2,... = list of neighbor weights, one for each type -* nx, ny, nz = number of grid points in x, y, and z directions (positive integer) +* *rcutfac* = scale factor applied to all cutoff radii (positive real) +* *rfac0* = parameter in distance to angle conversion (0 < rcutfac < 1) +* *twojmax* = band limit for bispectrum components (non-negative integer) +* *R_1, R_2,...* = list of cutoff radii, one for each type (distance units) +* *w_1, w_2,...* = list of neighbor weights, one for each type +* *grid* values = nx, ny, nz, number of grid points in x, y, and z directions (positive integer) * zero or more keyword/value pairs may be appended * keyword = *rmin0* or *switchflag* or *bzeroflag* or *quadraticflag* or *chem* or *bnormflag* or *wselfallflag* or *bikflag* or *switchinnerflag* or *sinner* or *dinner* or *dgradflag* or *nnn* or *wmode* or *delta* @@ -103,7 +110,7 @@ Examples compute snap all snap 1.4 0.95 6 2.0 1.0 compute snap all snap 1.0 0.99363 6 3.81 3.83 1.0 0.93 chem 2 0 1 compute snap all snap 1.0 0.99363 6 3.81 3.83 1.0 0.93 switchinnerflag 1 sinner 1.35 1.6 dinner 0.25 0.3 - compute bgrid all sna/grid/local 200 200 200 1.4 0.95 6 2.0 1.0 + compute bgrid all sna/grid/local grid 200 200 200 1.4 0.95 6 2.0 1.0 compute bnnn all sna/atom 9.0 0.99363 8 0.5 1.0 rmin0 0.0 nnn 24 wmode 1 delta 0.2 Description @@ -252,7 +259,8 @@ for finite-temperature Kohn-Sham density functional theory (:ref:`Ellis et al. `) Neighbor atoms not in the group do not contribute to the bispectrum components of the grid points. The distance cutoff :math:`R_{ii'}` assumes that *i* has the same type as the neighbor atom -*i'*. +*i'*. Both computes can be hardware accelerated with Kokkos by using the +*sna/grid/kk* and *sna/grid/local/kk* commands, respectively. Compute *sna/grid* calculates a global array containing bispectrum components for a regular grid of points. @@ -463,6 +471,12 @@ fluctuations in the resulting local atomic environment fingerprint. The detailed formalism is given in the paper by Lafourcade et al. :ref:`(Lafourcade) `. +---------- + + +.. include:: accel_styles.rst + + ---------- Output info @@ -654,7 +668,7 @@ of Angular Momentum, World Scientific, Singapore (1987). .. _Ellis2021: -**(Ellis)** Ellis, Fiedler, Popoola, Modine, Stephens, Thompson, Cangi, Rajamanickam, Phys Rev B, 104, 035120, (2021) +**(Ellis)** Ellis, Fiedler, Popoola, Modine, Stephens, Thompson, Cangi, Rajamanickam, `Phys. Rev. B, 104, 035120, (2021) `_ .. _Lafourcade2023_2: diff --git a/doc/src/compute_sph_e_atom.rst b/doc/src/compute_sph_e_atom.rst index d557eb7920..44ff765a6a 100644 --- a/doc/src/compute_sph_e_atom.rst +++ b/doc/src/compute_sph_e_atom.rst @@ -33,6 +33,12 @@ particle. See `this PDF guide `_ to using SPH in LAMMPS. +.. note:: + + Please note that the SPH PDF guide file has not been updated for + many years and thus does not reflect the current *syntax* of the + SPH package commands. For that please refer to the LAMMPS manual. + The value of the internal energy will be 0.0 for atoms not in the specified compute group. diff --git a/doc/src/compute_sph_rho_atom.rst b/doc/src/compute_sph_rho_atom.rst index 6e8579476a..6ed6bf70c1 100644 --- a/doc/src/compute_sph_rho_atom.rst +++ b/doc/src/compute_sph_rho_atom.rst @@ -32,6 +32,12 @@ kernel function interpolation using "pair style sph/rhosum". See `this PDF guide `_ to using SPH in LAMMPS. +.. note:: + + Please note that the SPH PDF guide file has not been updated for + many years and thus does not reflect the current *syntax* of the + SPH package commands. For that please refer to the LAMMPS manual. + The value of the SPH density will be 0.0 for atoms not in the specified compute group. diff --git a/doc/src/compute_sph_t_atom.rst b/doc/src/compute_sph_t_atom.rst index 3383c26145..8bbab4ca7d 100644 --- a/doc/src/compute_sph_t_atom.rst +++ b/doc/src/compute_sph_t_atom.rst @@ -37,6 +37,12 @@ particles, i.e. a Smooth-Particle Hydrodynamics particle. See `this PDF guide `_ to using SPH in LAMMPS. +.. note:: + + Please note that the SPH PDF guide file has not been updated for + many years and thus does not reflect the current *syntax* of the + SPH package commands. For that please refer to the LAMMPS manual. + The value of the internal energy will be 0.0 for atoms not in the specified compute group. diff --git a/doc/src/compute_temp_chunk.rst b/doc/src/compute_temp_chunk.rst index 33eab04343..3d97ad1699 100644 --- a/doc/src/compute_temp_chunk.rst +++ b/doc/src/compute_temp_chunk.rst @@ -184,11 +184,24 @@ temp/chunk calculation to a file is to use the The keyword/value option pairs are used in the following ways. The *com* keyword can be used with a value of *yes* to subtract the -velocity of the center-of-mass for each chunk from the velocity of the -atoms in that chunk, before calculating either the global or per-chunk -temperature. This can be useful if the atoms are streaming or +velocity of the center-of-mass (VCM) for each chunk from the velocity of +the atoms in that chunk, before calculating either the global or per-chunk +temperature. This can be useful if the atoms are streaming or otherwise moving collectively, and you wish to calculate only the -thermal temperature. +thermal temperature. This per-chunk VCM bias can be used in other fixes and +computes that can incorporate a temperature bias. If this compute is used +as a temperature bias in other commands then this bias is subtracted from +each atom, the command runs with the remaining thermal velocities, and +then the bias is added back in. This includes thermostatting +fixes like :doc:`fix nvt `, +:doc:`fix temp/rescale `, +:doc:`fix temp/berendsen `, and +:doc:`fix langevin `, and computes like +:doc:`compute stress/atom ` and +:doc:`compute pressure `. See the input script in +examples/stress_vcm for an example of how to use the *com* keyword in +conjunction with compute stress/atom to create a stress profile of a rigid +body while removing the overall motion of the rigid body. For the *bias* keyword, *bias-ID* refers to the ID of a temperature compute that removes a "bias" velocity from each atom. This also diff --git a/doc/src/delete_bonds.rst b/doc/src/delete_bonds.rst index e03c4b3ac7..e6825ded33 100644 --- a/doc/src/delete_bonds.rst +++ b/doc/src/delete_bonds.rst @@ -62,6 +62,18 @@ For all styles, by default, an interaction is only turned off (or on) if all the atoms involved are in the specified group. See the *any* keyword to change the behavior. +.. admonition:: Possible errors caused by using *delete_bonds* + :class: warning + + Since this command by default only *turns off* bonded interactions, + their definitions are still present and subject to the limitations + due to LAMMPS' domain decomposition based parallelization. That is, + when a bond is turned off, the two constituent atoms may move apart + and may reach a distance where they can lead to a "bond atoms missing" + error and crash the simulation. Adding the *remove* keyword (see + below) is required to fully remove those interactions and prevent + the error. + Several of the styles (\ *atom*, *bond*, *angle*, *dihedral*, *improper*\ ) take a *type* as an argument. The specified *type* can be a :doc:`type label `. Otherwise, the type should be an @@ -98,15 +110,18 @@ of all interactions in the specified group is simply reported. This is useful for diagnostic purposes if bonds have been turned off by a bond-breaking potential during a previous run. -The default behavior of the delete_bonds command is to turn off -interactions by toggling their type to a negative value, but not to -permanently remove the interaction. For example, a bond_type of 2 is set to -:math:`-2.` The neighbor list creation routines will not include such an -interaction in their interaction lists. The default is also to not -alter the list of 1--2, 1--3, or 1--4 neighbors computed by the -:doc:`special_bonds ` command and used to weight pairwise -force and energy calculations. This means that pairwise computations -will proceed as if the bond (or angle, etc.) were still turned on. +.. admonition:: Impact on special_bonds processing and exclusions + :class: note + + The default behavior of the delete_bonds command is to turn off + interactions by toggling their type to a negative value, but not to + permanently remove the interaction. For example, a bond_type of 2 is set to + :math:`-2.` The neighbor list creation routines will not include such an + interaction in their interaction lists. The default is also to not + alter the list of 1--2, 1--3, or 1--4 neighbors computed by the + :doc:`special_bonds ` command and used to weight pairwise + force and energy calculations. This means that pairwise computations + will proceed as if the bond (or angle, etc.) were still turned on. Several keywords can be appended to the argument list to alter the default behaviors. @@ -138,9 +153,11 @@ operation, after (optional) removal. It re-computes the pairwise 1--2, turned-off bonds the same as turned-on. Thus, turned-off bonds must be removed if you wish to change the weighting list. -Note that the choice of *remove* and *special* options affects how -1--2, 1--3, 1--4 pairwise interactions will be computed across bonds that -have been modified by the delete_bonds command. +.. note:: + + The choice of *remove* and *special* options affects how 1--2, + 1--3, 1--4 pairwise interactions will be computed across bonds + that have been modified by the delete_bonds command. Restrictions """""""""""" diff --git a/doc/src/dump_image.rst b/doc/src/dump_image.rst index 2336a5889a..8313d3cccb 100644 --- a/doc/src/dump_image.rst +++ b/doc/src/dump_image.rst @@ -681,7 +681,7 @@ MPEG or other movie file you can use: * c) Use FFmpeg - FFmpeg is a command line tool that is available on many platforms and + FFmpeg is a command-line tool that is available on many platforms and allows extremely flexible encoding and decoding of movies. .. code-block:: bash diff --git a/doc/src/fix.rst b/doc/src/fix.rst index 9af607601b..49fbb72726 100644 --- a/doc/src/fix.rst +++ b/doc/src/fix.rst @@ -237,6 +237,7 @@ accelerated styles exist. * :doc:`dt/reset ` - reset the timestep based on velocity, forces * :doc:`edpd/source ` - add heat source to eDPD simulations * :doc:`efield ` - impose electric field on system +* :doc:`efield/lepton ` - impose electric field on system using a Lepton expression for the potential * :doc:`efield/tip4p ` - impose electric field on system with TIP4P molecules * :doc:`ehex ` - enhanced heat exchange algorithm * :doc:`electrode/conp ` - impose electric potential diff --git a/doc/src/fix_adapt.rst b/doc/src/fix_adapt.rst index 1ddf80cbdb..93bd8da041 100644 --- a/doc/src/fix_adapt.rst +++ b/doc/src/fix_adapt.rst @@ -406,6 +406,8 @@ sub-style name. The angle styles that currently work with fix adapt are: +--------------------------------------------------------------------+--------------------+-------------+ | :doc:`mm3 ` | k,theta0 | type angles | +--------------------------------------------------------------------+--------------------+-------------+ +| :doc:`mwlc ` | k1,k2,mu,T | type angles | ++--------------------------------------------------------------------+--------------------+-------------+ | :doc:`quartic ` | k2,k3,k4,theta0 | type angles | +--------------------------------------------------------------------+--------------------+-------------+ | :doc:`spica ` | k,theta0 | type angles | diff --git a/doc/src/fix_efield.rst b/doc/src/fix_efield.rst index a870590856..ecdb25fca5 100644 --- a/doc/src/fix_efield.rst +++ b/doc/src/fix_efield.rst @@ -45,8 +45,9 @@ Description Add a force :math:`\vec{F} = q\vec{E}` to each charged atom in the group due to an external electric field being applied to the system. If the system -contains point-dipoles, also add a torque on the dipoles due to the -external electric field. +contains point-dipoles, also add a torque :math:`\vec{T} = \vec{p} \times \vec{E}` on the dipoles due to the +external electric field. This fix does not compute the dipole force :math:`\vec{F} = (\vec{p} \cdot \nabla) \vec{E}`, +and the :doc:`fix efield/lepton ` command should be used instead. .. versionadded:: 28Mar2023 @@ -68,6 +69,7 @@ For point-dipoles, equal-style variables can be used, but atom-style variables are not currently supported, since they imply a spatial gradient in the electric field which means additional terms with gradients of the field are required for the force and torque on dipoles. +The :doc:`fix efield/lepton ` command should be used instead. Equal-style variables can specify formulas with various mathematical functions, and include :doc:`thermo_style ` command @@ -229,7 +231,7 @@ Fix style *efield/tip4p* can only be used with tip4p pair styles. Related commands """""""""""""""" -:doc:`fix addforce ` +:doc:`fix addforce `, :doc:`fix efield/lepton ` Default """"""" diff --git a/doc/src/fix_efield_lepton.rst b/doc/src/fix_efield_lepton.rst new file mode 100644 index 0000000000..5deec4a15e --- /dev/null +++ b/doc/src/fix_efield_lepton.rst @@ -0,0 +1,143 @@ +.. index:: fix efield/lepton + +fix efield/lepton command +========================= + +Syntax +"""""" + +.. code-block:: LAMMPS + + fix ID group-ID efield/lepton V ... + +* ID, group-ID are documented in the :doc:`fix ` command +* style = *efield/lepton* +* V = electric potential (electric field * distance units) +* V must be a Lepton expression (see below) +* zero or more keyword/value pairs may be appended to args +* keyword = *region* or *step* + + .. parsed-literal:: + + *region* value = region-ID + region-ID = ID of region atoms must be in to have effect + *step* value = h + h = step size for numerical differentiation (distance units) + +Examples +"""""""" + +.. code-block:: LAMMPS + + fix ex all efield/lepton "-E*x; E=1" + fix dexx all efield/lepton "-0.5*x^2" step 1 + fix yukawa all efield/lepton "A*exp(-B*r)/r; r=abs(sqrt(x^2+y^2+z^2)); A=1; B=1" step 1e-6 + fix infp all efield/lepton "-abs(x)" step 1 + + variable th equal 2*PI*ramp(0,1) + fix erot all efield/lepton "-(x*cos(v_th)+y*sin(v_th))" + +Description +""""""""""" + +.. versionadded:: TBD + +Add an electric potential :math:`V` that applies to a group of charged atoms a force :math:`\vec{F} = q \vec{E}`, +and to dipoles a force :math:`\vec{F} = (\vec{p} \cdot \nabla) \vec{E}` and torque :math:`\vec{T} = \vec{p} \times \vec{E}`, +where :math:`\vec{E} = - \nabla V`. The fix also evaluates the electrostatic energy (:math:`U_{q} = q V` and :math:`U_{p} = - \vec{p} \cdot \vec{E}`) +due to this potential when the :doc:`fix_modify energy yes ` command is specified (see below). + +.. note:: + + This command should be used instead of :doc:`fix efield ` if you want to impose a non-uniform electric field on a system with dipoles + since the latter does not include the dipole force term. If you only have charges or if the electric field gradient is negligible, + :doc:`fix efield ` should be used since it is faster. + +The `Lepton library `_, that the *efield/lepton* fix style interfaces with, evaluates +the expression string at run time to compute the energy, forces, and torques. It creates an analytical representation +of :math:`V` and :math:`\vec{E}`, while the gradient force is computed using a central difference scheme + +.. math:: + + \vec{F} = \frac{|\vec{p}|}{2h} \left[ \vec{E}(\vec{x} + h \hat{p}) - \vec{E}(\vec{x} - h \hat{p}) \right] . + +The Lepton expression must be either enclosed in quotes or must not contain any whitespace so that LAMMPS +recognizes it as a single keyword. More on valid Lepton expressions below. The final Lepton expression must +be a function of only :math:`x, y, z`, which refer to the current *unwrapped* coordinates of the atoms to ensure continuity. +Special care must be taken when using this fix with periodic boundary conditions or box-changing commands. + +---------- + +.. include:: lepton_expression.rst + +---------- + +If the *region* keyword is used, the atom must also be in the specified +geometric :doc:`region ` in order to be affected by the potential. + +The *step* keyword is required when :doc:`atom_style dipole ` is used and the electric field is non-uniform. + +---------- + +Restart, fix_modify, output, run start/stop, minimize info +""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" + +No information about this fix is written to :doc:`binary restart files +`. + +The :doc:`fix_modify ` *energy* option is supported by this +fix to add the potential energy defined above to the global potential energy +of the system as part of :doc:`thermodynamic output `. +The default setting for this fix is :doc:`fix_modify energy no `. + +The :doc:`fix_modify ` *virial* option is supported by this +fix to add the contribution due to the added ***forces*** on charges and dipoles +to both the global pressure and per-atom stress of the system via the +:doc:`compute pressure ` and :doc:`compute stress/atom +` commands. The former can be accessed by +:doc:`thermodynamic output `. The default setting for +this fix is :doc:`fix_modify virial no `. + +The :doc:`fix_modify ` *respa* option is supported by this +fix. This allows to set at which level of the :doc:`r-RESPA ` +integrator the fix adding its forces. Default is the outermost level. + +This fix computes a global scalar and a global 3-vector of forces, +which can be accessed by various :doc:`output commands `. +The scalar is the potential energy discussed above. +The vector is the total force added to the group of atoms. +The scalar and vector values calculated by this fix are "extensive". + +This fix cannot be used with the *start/stop* keywords of +the :doc:`run ` command. + +The forces due to this fix are imposed during an energy minimization, +invoked by the :doc:`minimize ` command. You should not +specify force components with a variable that has time-dependence for +use with a minimizer, since the minimizer increments the timestep as +the iteration count during the minimization. + +.. note:: + + If you want the electric potential energy to be included in the + total potential energy of the system (the quantity being minimized), + you MUST enable the :doc:`fix_modify ` *energy* option for this fix. + +---------- + +Restrictions +"""""""""""" + +Fix style *efield/lepton* is part of the LEPTON package. It is only enabled if LAMMPS was built with that package. +See the :doc:`Build package ` page for more info. + + +Related commands +"""""""""""""""" + +:doc:`fix efield ` + +Default +""""""" + +none diff --git a/doc/src/fix_imd.rst b/doc/src/fix_imd.rst index 520af505a1..a4d7d9d387 100644 --- a/doc/src/fix_imd.rst +++ b/doc/src/fix_imd.rst @@ -26,6 +26,29 @@ Syntax *nowait* arg = *on* or *off* off = LAMMPS waits to be connected to an IMD client before continuing (default) on = LAMMPS listens for an IMD client, but continues with the run + *version* arg = *2* or *3* + 2 = use IMD protocol version 2 (default) + 3 = use IMD protocol version 3. + + The following keywords are only supported for IMD protocol version 3. + + .. parsed-literal:: + + *time* arg = *on* or *off* + off = simulation time is not transmitted (default) + on = simulation time is transmitted. + *box* arg = *on* or *off* + off = simulation box data is not transmitted (default) + on = simulation box data is transmitted. + *coordinates* arg = *on* or *off* + off = atomic coordinates are not transmitted (default) + on = atomic coordinates are transmitted. + *velocities* arg = *on* or *off* + off = atomic velocities are not transmitted (default) + on = atomic velocities are transmitted. + *forces* arg = *on* or *off* + off = atomic forces are not transmitted (default) + on = atomic forces are transmitted. Examples """""""" @@ -40,16 +63,19 @@ Description This fix implements the "Interactive MD" (IMD) protocol which allows realtime visualization and manipulation of MD simulations through the -IMD protocol, as initially implemented in VMD and NAMD. Specifically -it allows LAMMPS to connect an IMD client, for example the `VMD visualization program `_, so that it can monitor the progress of the -simulation and interactively apply forces to selected atoms. +IMD protocol, as initially implemented in VMD and NAMD. Specifically it +allows LAMMPS to connect an IMD client, for example the `VMD +visualization program `_ (currently only supports IMDv2) or the +`Python IMDClient `_ (supports both IMDv2 and IMDv3), so +that it can monitor the progress of the simulation and interactively +apply forces to selected atoms. -If LAMMPS is compiled with the pre-processor flag -DLAMMPS_ASYNC_IMD -then fix imd will use POSIX threads to spawn a IMD communication -thread on MPI rank 0 in order to offload data reading and writing -from the main execution thread and potentially lower the inferred -latencies for slow communication links. This feature has only been -tested under linux. +If LAMMPS is compiled with the pre-processor flag +:ref:`-DLAMMPS_ASYNC_IMD ` then fix imd will use POSIX threads to +spawn an IMD communication thread on MPI rank 0 in order to offload data +exchange with the IMD client from the main execution thread and +potentially lower the inferred latencies for slow communication +links. This feature has only been tested under linux. The source code for this fix includes code developed by the Theoretical and Computational Biophysics Group in the Beckman Institute for Advanced @@ -94,10 +120,19 @@ with different units or as a measure to tweak the forces generated by the manipulation of the IMD client, this option allows to make adjustments. +.. versionadded:: TBD + +In `IMDv3 `_, the IMD protocol has been extended to allow for +the transmission of simulation time, box dimensions, atomic coordinates, +velocities, and forces. The *version* keyword allows to select the +version of the protocol to be used. The *time*, *box*, *coordinates*, +*velocities*, and *forces* keywords allow to select which data is +transmitted to the IMD client. The default is to transmit all data. + To connect VMD to a listening LAMMPS simulation on the same machine with fix imd enabled, one needs to start VMD and load a coordinate or topology file that matches the fix group. When the VMD command -prompts appears, one types the command line: +prompts appears, one types the command: .. parsed-literal:: @@ -129,6 +164,10 @@ screen output is active. .. _imdvmd: https://www.ks.uiuc.edu/Research/vmd/imd/ +.. _IMDClient: https://github.com/Becksteinlab/imdclient/tree/main/imdclient + +.. _IMDv3: https://imdclient.readthedocs.io/en/latest/protocol_v3.html + Restart, fix_modify, output, run start/stop, minimize info """"""""""""""""""""""""""""""""""""""""""""""""""""""""""" @@ -147,14 +186,14 @@ This fix is part of the MISC package. It is only enabled if LAMMPS was built with that package. See the :doc:`Build package ` page for more info. -When used in combination with VMD, a topology or coordinate file has -to be loaded, which matches (in number and ordering of atoms) the -group the fix is applied to. The fix internally sorts atom IDs by -ascending integer value; in VMD (and thus the IMD protocol) those will -be assigned 0-based consecutive index numbers. +When used in combination with VMD, a topology or coordinate file has to +be loaded, which matches (in number and ordering of atoms) the group the +fix is applied to. The fix internally sorts atom IDs by ascending +integer value; in VMD (and thus the IMD protocol) those will be assigned +0-based consecutive index numbers. When using multiple active IMD connections at the same time, each -needs to use a different port number. +fix instance needs to use a different port number. Related commands """""""""""""""" diff --git a/doc/src/fix_precession_spin.rst b/doc/src/fix_precession_spin.rst index 7440989d7a..88325ba491 100644 --- a/doc/src/fix_precession_spin.rst +++ b/doc/src/fix_precession_spin.rst @@ -135,7 +135,7 @@ directions for the forces. Only the direction of the vector is important; its length is ignored (the entered vectors are normalized). -Those styles can be combined within one single command line. +Those styles can be combined within one single command. .. note:: diff --git a/doc/src/fix_python_invoke.rst b/doc/src/fix_python_invoke.rst index ad55882270..4f33f5483b 100644 --- a/doc/src/fix_python_invoke.rst +++ b/doc/src/fix_python_invoke.rst @@ -66,6 +66,15 @@ gives access to the LAMMPS state from Python. from these callbacks, trying to execute input script commands will in the best case not work or in the worst case result in undefined behavior. +Restart, fix_modify, output, run start/stop, minimize info +""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" + +No information about this fix is written to :doc:`binary restart files `. None of the :doc:`fix_modify ` options +are relevant to this fix. No global or per-atom quantities are stored +by this fix for access by various :doc:`output commands `. +No parameter of this fix can be used with the *start/stop* keywords of +the :doc:`run ` command. This fix is not invoked during :doc:`energy minimization `. + Restrictions """""""""""" diff --git a/doc/src/fix_reaxff_species.rst b/doc/src/fix_reaxff_species.rst index 107695f0f6..068d1de995 100644 --- a/doc/src/fix_reaxff_species.rst +++ b/doc/src/fix_reaxff_species.rst @@ -200,8 +200,8 @@ The 2 values in the global vector are as follows: The per-atom vector stores the molecule ID for each atom as identified by the fix. If an atom is not in a molecule, its ID will be 0. For atoms in the same molecule, the molecule ID for all of them -will be the same and will be equal to the smallest atom ID of -any atom in the molecule. +will be the same, and molecule IDs will range from 1 to the number +of molecules. No parameter of this fix can be used with the *start/stop* keywords of the :doc:`run ` command. diff --git a/doc/src/fix_rheo.rst b/doc/src/fix_rheo.rst index 0214289dee..ef18d19f82 100644 --- a/doc/src/fix_rheo.rst +++ b/doc/src/fix_rheo.rst @@ -64,7 +64,7 @@ Description Perform time integration for RHEO particles, updating positions, velocities, and densities. For a detailed breakdown of the integration timestep and -numerical details, see :ref:`(Palermo) `. For an overview +numerical details, see :ref:`(Palermo) `. For an overview and list of other features available in the RHEO package, see :doc:`the RHEO howto `. @@ -101,7 +101,7 @@ A modified form of Fickian particle shifting can be enabled with the more uniform spatial distribution. By default, shifting does not consider the type of a particle and therefore may be inappropriate in systems consisting of multiple atom types representing multiple fluid phases. However, two -optional subarguments can follow the *shift* keyword, *exclude/type* and +optional sub-arguments can follow the *shift* keyword, *exclude/type* and *scale/cross/type* to adjust shifting at fluid interfaces. The *exclude/type* option lets the user specify a list of atom types which @@ -155,7 +155,7 @@ threshold for this classification is set by the numerical value of By default, RHEO integrates particles' densities using a mass diffusion equation. Alternatively, one can update densities every timestep by performing a kernel summation of the masses of neighboring particles by specifying the *rho/sum* -keyword. Following this keyword, one may include the optional *self/mass* subargument +keyword. Following this keyword, one may include the optional *self/mass* sub-argument which modifies the behavior of the density summation. Typically, the density :math:`\rho` of a particle is calculated as the sum over neighbors @@ -218,11 +218,11 @@ Default ---------- -.. _rheo_palermo: +.. _fix_rheo_palermo: **(Palermo)** Palermo, Wolf, Clemmer, O'Connor, Phys. Fluids, 36, 113337 (2024). -.. _rheo_yang: +.. _fix_rheo_yang: **(Yang)** Yang, Rakhsha, Hu, Negrut, J. Comp. Physics, 458, 111079 (2022). diff --git a/doc/src/fix_sph.rst b/doc/src/fix_sph.rst index 4932485b03..09989d2fa0 100644 --- a/doc/src/fix_sph.rst +++ b/doc/src/fix_sph.rst @@ -32,6 +32,12 @@ Hydrodynamics. See `this PDF guide `_ to using SPH in LAMMPS. +.. note:: + + Please note that the SPH PDF guide file has not been updated for + many years and thus does not reflect the current *syntax* of the + SPH package commands. For that please refer to the LAMMPS manual. + Restart, fix_modify, output, run start/stop, minimize info """"""""""""""""""""""""""""""""""""""""""""""""""""""""""" diff --git a/doc/src/fix_sph_stationary.rst b/doc/src/fix_sph_stationary.rst index 07bedad4c3..57914d22b1 100644 --- a/doc/src/fix_sph_stationary.rst +++ b/doc/src/fix_sph_stationary.rst @@ -32,6 +32,12 @@ space. SPH stands for Smoothed Particle Hydrodynamics. See `this PDF guide `_ to using SPH in LAMMPS. +.. note:: + + Please note that the SPH PDF guide file has not been updated for + many years and thus does not reflect the current *syntax* of the + SPH package commands. For that please refer to the LAMMPS manual. + Restart, fix_modify, output, run start/stop, minimize info """"""""""""""""""""""""""""""""""""""""""""""""""""""""""" diff --git a/doc/src/fix_wall_gran.rst b/doc/src/fix_wall_gran.rst index 25d659241c..81a411ffc8 100644 --- a/doc/src/fix_wall_gran.rst +++ b/doc/src/fix_wall_gran.rst @@ -222,10 +222,10 @@ restart file, so that the operation of the fix continues in an uninterrupted fashion. If the :code:`contacts` option is used, this fix generates a per-atom array -with 8 columns as output, containing the contact information for owned +with at least 8 columns as output, containing the contact information for owned particles (nlocal on each processor). All columns in this per-atom array will -be zero if no contact has occurred. The values of these columns are listed in -the following table: +be zero if no contact has occurred. The first 8 values of these columns are +listed in the following table. +-------+----------------------------------------------------+----------------+ | Index | Value | Units | @@ -248,6 +248,14 @@ the following table: | 8 | Radius :math:`r` of atom | distance units | +-------+----------------------------------------------------+----------------+ +If a granular sub-model calculates additional contact information (e.g. the +heat sub-models calculate the amount of heat exchanged), these quantities +are appended to the end of this array. First, any extra values from the +normal sub-model are appended followed by the damping, tangential, rolling, +twisting, then heat models. See the descriptions of granular sub-models in +the :doc:`pair granular ` page for information on any extra +quantities. + None of the :doc:`fix_modify ` options are relevant to this fix. No parameter of this fix can be used with the *start/stop* keywords of the :doc:`run ` command. This fix is not invoked during :doc:`energy diff --git a/doc/src/fix_wall_gran_region.rst b/doc/src/fix_wall_gran_region.rst index 0a4343493b..4ad3b9d6c5 100644 --- a/doc/src/fix_wall_gran_region.rst +++ b/doc/src/fix_wall_gran_region.rst @@ -243,10 +243,10 @@ uninterrupted fashion. with a different region ID. If the :code:`contacts` option is used, this fix generates a per-atom array -with 8 columns as output, containing the contact information for owned +with at least 8 columns as output, containing the contact information for owned particles (nlocal on each processor). All columns in this per-atom array will -be zero if no contact has occurred. The values of these columns are listed in -the following table: +be zero if no contact has occurred. The first 8 values of these columns are +listed in the following table. +-------+----------------------------------------------------+----------------+ | Index | Value | Units | @@ -269,6 +269,14 @@ the following table: | 8 | Radius :math:`r` of atom | distance units | +-------+----------------------------------------------------+----------------+ +If a granular sub-model calculates additional contact information (e.g. the +heat sub-models calculate the amount of heat exchanged), these quantities +are appended to the end of this array. First, any extra values from the +normal sub-model are appended followed by the damping, tangential, rolling, +twisting, then heat models. See the descriptions of granular sub-models in +the :doc:`pair granular ` page for information on any extra +quantities. + None of the :doc:`fix_modify ` options are relevant to this fix. No parameter of this fix can be used with the *start/stop* keywords of the :doc:`run ` command. This fix is not invoked during :doc:`energy diff --git a/doc/src/kim_commands.rst b/doc/src/kim_commands.rst index ab9b4486a2..20728dee06 100644 --- a/doc/src/kim_commands.rst +++ b/doc/src/kim_commands.rst @@ -1084,10 +1084,11 @@ the form of *key_name_key*-*key_name_value* pairs). For example, kim property modify 1 key mass source-value 26.98154 kim property modify 1 key mass source-unit amu -where the special keyword "key" is followed by a *key_name* ("species" or -"mass" in the above) and one or more key-value pairs. These key-value pairs -may continue until either another "key" keyword is given or the end of the -command line is reached. Thus, the above could equivalently be written as +where the special keyword "key" is followed by a *key_name* ("species" +or "mass" in the above) and one or more key-value pairs. These +key-value pairs may continue until either another "key" keyword is given +or the end of the line is reached. Thus, the above could equivalently +be written as .. code-block:: LAMMPS diff --git a/doc/src/kspace_modify.rst b/doc/src/kspace_modify.rst index c40ba70ef0..b300213a3b 100644 --- a/doc/src/kspace_modify.rst +++ b/doc/src/kspace_modify.rst @@ -412,11 +412,9 @@ slab correction has also been extended to point dipole interactions .. note:: If you wish to apply an electric field in the Z-direction, in - conjunction with the *slab* keyword, you should do it by adding - explicit charged particles to the +/- Z surfaces. If you do it via - the :doc:`fix efield ` command, it will not give the correct - dielectric constant due to the Yeh/Berkowitz :ref:`(Yeh) ` correction - not being compatible with how :doc:`fix efield ` works. + conjunction with the *slab* keyword, you can do it either by + adding explicit oppositely charged particles to the +/- Z surfaces, + or by using the :doc:`fix efield ` command. ---------- diff --git a/doc/src/label.rst b/doc/src/label.rst index bf232431d7..cae1386e90 100644 --- a/doc/src/label.rst +++ b/doc/src/label.rst @@ -24,12 +24,12 @@ Description """"""""""" Label this line of the input script with the chosen ID. Unless a jump -command was used previously, this does nothing. But if a -:doc:`jump ` command was used with a label argument to begin -invoking this script file, then all command lines in the script prior -to this line will be ignored. I.e. execution of the script will begin -at this line. This is useful for looping over a section of the input -script as discussed in the :doc:`jump ` command. +command was used previously, this does nothing. But if a :doc:`jump +` command was used with a label argument to begin invoking this +script file, then all commands in the script prior to this line will be +ignored. I.e. execution of the script will begin at this line. This is +useful for looping over a section of the input script as discussed in +the :doc:`jump ` command. Restrictions """""""""""" diff --git a/doc/src/package.rst b/doc/src/package.rst index 8be6639e72..ddb3656027 100644 --- a/doc/src/package.rst +++ b/doc/src/package.rst @@ -504,7 +504,7 @@ as it is for non-accelerated pair styles The *binsize* keyword sets the size of bins used to bin atoms during neighbor list builds. The same value can be set by the :doc:`neigh_modify binsize ` command. Making it an option -in the package kokkos command allows it to be set from the command line. +in the package kokkos command allows it to be set from the command-line. The default value for CPUs is 0.0, which means the LAMMPS default will be used, which is bins = 1/2 the size of the pairwise cutoff + neighbor skin distance. This is fine when neighbor lists are built on the CPU. For GPU @@ -664,7 +664,7 @@ too. Also note that if the :doc:`-sf hybrid intel omp command-line switch ` is used, it invokes a "package intel" command, followed by a "package omp" command, both with a setting of *Nthreads* = 0. Likewise for a hybrid suffix for gpu and omp. Note that KOKKOS also supports - setting the number of OpenMP threads from the command line using the + setting the number of OpenMP threads from the command-line using the "-k on" :doc:`command-line switch `. The default for KOKKOS is 1 thread per MPI task, so any other number of threads should be explicitly set using the "-k on" command-line switch (and this diff --git a/doc/src/pair_dispersion_d3.rst b/doc/src/pair_dispersion_d3.rst new file mode 100644 index 0000000000..9c8726b2a1 --- /dev/null +++ b/doc/src/pair_dispersion_d3.rst @@ -0,0 +1,158 @@ +.. index:: pair_style dispersion/d3 + +pair_style dispersion/d3 command +================================ + +Syntax +"""""" + +.. code-block:: LAMMPS + + pair_style dispersion/d3 damping functional cutoff cn_cutoff + +* damping = damping function: *zero*, *zerom*, *bj*, or *bjm* +* functional = XC functional form: *pbe*, *pbe0*, ... (see list below) +* cutoff = global cutoff (distance units) +* cn_cutoff = coordination number cutoff (distance units) + +Examples +"""""""" + +.. code-block:: LAMMPS + + pair_style dispersion/d3 zero pbe 30.0 20.0 + pair_coeff * * C + +Description +""""""""""" + +.. versionadded:: TBD + +Style *dispersion/d3* computes the dispersion energy-correction used in +the DFT-D3 method of Grimme :ref:`(Grimme1) `. It would +typically be used with a machine learning (ML) potential that was +trained with results from plain DFT calculations without the dispersion +correction through pair_style hybrid/overlay. ML potentials are often +combined *a posteriori* with dispersion energy-correction schemes (see +*e.g.* :ref:`(Qamar) ` and :ref:`(Batatia) `). + +The energy contribution :math:`E_i` for an atom :math:`i` is given by: + +.. math:: + + E_i = \frac{1}{2} \sum_{j \neq i} \big( + s_6 \frac{C_{6,ij}}{r^6_{ij}} f_6^{damp}(r_{ij}) + + s_8 \frac{C_{8,ij}}{r^8_{ij}} f_8^{damp}(r_{ij}) \big) + +where :math:`C_n` is the averaged, geometry-dependent nth-order +dispersion coefficient for atom pair :math:`ij`, :math:`r_{ij}` their +inter-nuclear distance, :math:`s_n` are XC functional-dependent scaling +factor, and :math:`f_n^{damp}` are damping functions. + +.. note:: + + It is currently *not* possible to calculate three-body dispersion + contributions, according to, for example, the Axilrod-Teller-Muto + model. + +Available damping functions are the original "zero-damping" +:ref:`(Grimme1) `, Becke-Johnson damping :ref:`(Grimme2) +`, and their revised forms :ref:`(Sherrill) `. + +Available XC functional scaling factors are listed in the table below, +and depend on the selected damping function. + ++------------------+--------------------------------------------------------------------------------+ +| Damping function | XC functional | ++==================+================================================================================+ +| | | | slater-dirac-exchange, b-lyp, b-p, b97-d, revpbe, pbe, pbesol, rpw86-pbe, | +| | | | rpbe, tpss, b3-lyp, pbe0, hse06, revpbe38, pw6b95, tpss0, b2-plyp, pwpb95, | +| | zero | | b2gp-plyp, ptpss, hf, mpwlyp, bpbe, bh-lyp, tpssh, pwb6k, b1b95, bop, o-lyp, | +| | | | o-pbe, ssb, revssb, otpss, b3pw91, revpbe0, pbe38, mpw1b95, mpwb1k, bmk, | +| | | | cam-b3lyp, lc-wpbe, m05, m052x, m06l, m06, m062x, m06hf, hcth120 | ++------------------+--------------------------------------------------------------------------------+ +| zerom | b2-plyp, b3-lyp, b97-d, b-lyp, b-p, pbe, pbe0, lc-wpbe | ++------------------+--------------------------------------------------------------------------------+ +| | | | b-p, b-lyp, revpbe, rpbe, b97-d, pbe, rpw86-pbe, b3-lyp, tpss, hf, tpss0, | +| | | | pbe0, hse06, revpbe38, pw6b95, b2-plyp, dsd-blyp, dsd-blyp-fc, bop, mpwlyp, | +| | bj | | o-lyp, pbesol, bpbe, opbe, ssb, revssb, otpss, b3pw91, bh-lyp, revpbe0, | +| | | | tpssh, mpw1b95, pwb6k, b1b95, bmk, cam-b3lyp, lc-wpbe, b2gp-plyp, ptpss, | +| | | | pwpb95, hf/mixed, hf/sv, hf/minis, b3lyp/6-31gd, hcth120, pw1pw, pwgga, | +| | | | hsesol, hf3c, hf3cv, pbeh3c, pbeh-3c | ++------------------+--------------------------------------------------------------------------------+ +| bjm | b2-plyp, b3-lyp, b97-d, b-lyp, b-p, pbe, pbe0, lc-wpbe | ++------------------+--------------------------------------------------------------------------------+ + + +This style is primarily supposed to be used combined with a +machine-learned interatomic potential trained on a DFT dataset (the +selected XC functional should be chosen accordingly) via the +:doc:`pair_style hybrid ` command. + +Coefficients +"""""""""""" + +All the required coefficients are already stored internally (in the +``src/EXTRA-PAIR/d3_parameters.h`` file). The only information to +provide are the chemical symbols of the atoms. The number of chemical +symbols given must be equal to the number of atom types used and must +match their ordering as atom types. + + +Mixing, shift, table, tail correction, restart, rRESPA info +""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" + +This pair style does not support mixing since all parameters are +explicit for each pair of atom types. + +This pair style does not support the :doc:`pair_modify` shift, table, +and tail options. + +This pair style does not write its information to :doc:`binary restart +files `. + +This pair style can only be used via the *pair* keyword of the +:doc:`run_style respa ` command. It does not support the +*inner*, *middle*, *outer* keywords. + +Restrictions +"""""""""""" + +Style *dispersion/d3* is part of the EXTRA-PAIR package. It is only +enabled if LAMMPS was built with that package. See the :doc:`Build +package ` page for more info. + +It is currently *not* possible to calculate three-body dispersion +contributions according to, for example, the Axilrod-Teller-Muto model. + +Related commands +"""""""""""""""" + +:doc:`pair_coeff ` + +Default +""""""" + +none + +---------- + +.. _Grimme1: + +**(Grimme1)** S. Grimme, J. Antony, S. Ehrlich, and H. Krieg, J. Chem. Phys. 132, 154104 (2010). + +.. _Qamar: + +**(Qamar)** M. Qamar, M. Mrovec, T. Lysogorskiy, A. Bochkarev, and R. Drautz, J. Chem. Theory Comput. 19, 5151 (2023). + +.. _Batatia: + +**(Batatia)** I. Batatia, *et al.*, arXiv:2401.0096 (2023). + +.. _Grimme2: + +**(Grimme2)** S. Grimme, S. Ehrlich and L. Goerigk, J. Comput. Chem. 32, 1456 (2011). + +.. _Sherrill: + +**(Sherrill)** D. G. A. Smith, L. A. Burns, K. Patkowski, and C. D. Sherrill, J. Phys. Chem. Lett., 7, 2197, (2016). diff --git a/doc/src/pair_granular.rst b/doc/src/pair_granular.rst index 11a460e9ca..f54079daf1 100644 --- a/doc/src/pair_granular.rst +++ b/doc/src/pair_granular.rst @@ -40,6 +40,9 @@ Examples pair_style granular pair_coeff * * hertz 1000.0 50.0 tangential mindlin 1000.0 1.0 0.4 heat area 0.1 + pair_style granular + pair_coeff * * mdr 5e6 0.4 1.9e5 2.0 0.5 0.5 tangential linear_history 940.0 0.0 0.7 rolling sds 2.7e5 0.0 0.6 damping none + Description """"""""""" @@ -82,6 +85,7 @@ and their required arguments are: 3. *hertz/material* : E, :math:`\eta_{n0}` (or :math:`e`), :math:`\nu` 4. *dmt* : E, :math:`\eta_{n0}` (or :math:`e`), :math:`\nu`, :math:`\gamma` 5. *jkr* : E, :math:`\eta_{n0}` (or :math:`e`), :math:`\nu`, :math:`\gamma` +6. *mdr* : :math:`E`, :math:`\nu`, :math:`Y`, :math:`\Delta\gamma`, :math:`\psi_b`, :math:`e` Here, :math:`k_n` is spring stiffness (with units that depend on model choice, see below); :math:`\eta_{n0}` is a damping prefactor (or, in its @@ -162,6 +166,144 @@ initially will not experience force until they come into contact experience a tensile force up to :math:`3\pi\gamma R`, at which point they lose contact. +The *mdr* model is a mechanically-derived contact model designed to capture the +contact response between adhesive elastic-plastic particles into large deformation. +The theoretical foundations of the *mdr* model are detailed in the +two-part series :ref:`Zunker and Kamrin Part I ` and +:ref:`Zunker and Kamrin Part II `. Further development +and demonstrations of its application to industrially relevant powder +compaction processes are presented in :ref:`Zunker et al. `. + +The model requires the following inputs: + + 1. *Young's modulus* :math:`E > 0` : The Young's modulus is commonly reported + for various powders. + + 2. *Poisson's ratio* :math:`0 \le \nu \le 0.5` : The Poisson's ratio is commonly + reported for various powders. + + 3. *Yield stress* :math:`Y \ge 0` : The yield stress is often known for powders + composed of materials such as metals but may be unreported for ductile organic + materials, in which case it can be treated as a free parameter. + + 4. *Effective surface energy* :math:`\Delta\gamma \ge 0` : The effective surface + energy for powder compaction applications is most easily determined through its + relation to the more commonly reported critical stress intensity factor + :math:`K_{Ic} = \sqrt{2\Delta\gamma E/(1-\nu^2)}`. + + 5. *Critical confinement ratio* :math:`0 \le \psi_b \le 1` : The critical confinement + ratio is a tunable parameter that determines when the bulk elastic response is + triggered. Lower values of :math:`\psi_b` delay the onset of the bulk elastic + response. + + 6. *Coefficient of restitution* :math:`0 \le e \le 1` : The coefficient of + restitution is a tunable parameter that controls damping in the normal direction. + +.. note:: + + The values for :math:`E`, :math:`\nu`, :math:`Y`, and :math:`\Delta\gamma` (i.e., + :math:`K_{Ic}`) should be selected for zero porosity to reflect the intrinsic + material property rather than the bulk powder property. + +The *mdr* model produces a nonlinear force-displacement response, therefore the +critical timestep :math:`\Delta t` depends on the inputs and level of +deformation. As a conservative starting point the timestep can be assumed to be +dictated by the bulk elastic response such that +:math:`\Delta t = 0.35\sqrt{m/k_\textrm{bulk}}`, where :math:`m` is the mass of +the smallest particle and :math:`k_\textrm{bulk} = \kappa R_\textrm{min}` is an +effective stiffness related to the bulk elastic response. +Here, :math:`\kappa = E/(3(1-2\nu))` is the bulk modulus and +:math:`R_\textrm{min}` is the radius of the smallest particle. + +.. note:: + + The *mdr* model requires some specific settings to function properly, + please read the following text carefully to ensure all requirements are + followed. + +The *atom_style* must be set to *sphere 1* to enable dynamic particle +radii. The *mdr* model is designed to respect the incompressibility of +plastic deformation and inherently tracks free surface displacements +induced by all particle contacts. In practice, this means that all particles +begin with an initial radius, however as compaction occurs and plastic +deformation is accumulated, a new enlarged apparent radius is defined to +ensure that that volume change due to plastic deformation is not lost. +This apparent radius is stored as the *atom radius* meaning it is used +for subsequent neighbor list builds and contact detection checks. The +advantage of this is that multi-neighbor dependent effects such as +formation of secondary contacts caused by radial expansion are captured +by the *mdr* model. Setting *atom_style sphere 1* ensures that updates to +the particle radii are properly reflected throughout the simulation. + +.. code-block:: LAMMPS + + atom_style sphere 1 + +Newton's third law must be set to *off*. This ensures that the neighbor lists +are constructed properly for the topological penalty algorithm used to screen +for non-physical contacts occurring through obstructing particles, an issue +prevalent under large deformation conditions. For more information on this +algorithm see :ref:`Zunker et al. `. + +.. code-block:: LAMMPS + + newton off + +The damping model must be set to *none*. The *mdr* model already has a built +in damping model. + +.. code-block:: LAMMPS + + pair_coeff * * mdr 5e6 0.4 1.9e5 2 0.5 0.5 damping none + +The definition of multiple *mdr* models in the *pair_style* is currently not +supported. Similarly, the *mdr* model cannot be combined with a different normal +model in the *pair_style*. Physically this means that only one homogeneous +collection of particles governed by a single *mdr* model is allowed. + +The *mdr* model currently only supports *fix wall/gran/region*, not +*fix wall/gran*. If the *mdr* model is specified for the *pair_style* +any *fix wall/gran/region* commands must also use the *mdr* model. +Additionally, the following *mdr* inputs must match between the +*pair_style* and *fix wall/gran/region* definitions: :math:`E`, +:math:`\nu`, :math:`Y`, :math:`\psi_b`, and :math:`e`. The exception +is :math:`\Delta\gamma`, which may vary, permitting different +adhesive behaviors between particle-particle and particle-wall interactions. + +.. note:: + + The *mdr* model has a number of custom *property/atom* and *pair/local* definitions that + can be called in the input file. The useful properties for visualization + and analysis are described below. + +In addition to contact forces the *mdr* model also tracks the following +quantities for each particle: elastic volume change, average normal +stress components, total surface area involved in +contact, and individual contact areas. In the input script, these quantities are +initialized by calling *run 0* and can then be accessed using subsequent *compute* +commands. The last *compute* command uses *pair/local p13* to calculate the pairwise +contact areas for each active contact in the *group-ID*. Due to the use of an apparent +radius in the *mdr* model, the keyword/arg pair *cutoff radius* must be specified for +*pair/local* to properly detect existing contacts. + +.. code-block:: LAMMPS + + run 0 + compute ID group-ID property/atom d_Velas + compute ID group-ID property/atom d_sigmaxx + compute ID group-ID property/atom d_sigmayy + compute ID group-ID property/atom d_sigmazz + compute ID group-ID property/atom d_Acon1 + compute ID group-ID pair/local p13 cutoff radius + +.. note:: + + The *mdr* model has two example input scripts within the + *examples/granular* directory. The first is a die compaction + simulation involving 200 particles named *in.tableting.200*. + The second is a triaxial compaction simulation involving 12 + particles named *in.triaxial.compaction.12*. + ---------- In addition, the normal force is augmented by a damping term of the @@ -674,7 +816,10 @@ supported are: 2. *radius* : :math:`k_{s}` 3. *area* : :math:`h_{s}` -If the *heat* keyword is not specified, the model defaults to *none*. +If the *heat* keyword is not specified, the model defaults to *none*. All +heat models calculate an additional pairwise quantity accessible by the +single() function (described below) which is the heat conducted between the +two particles. For *heat* *radius*, the heat :math:`Q` conducted between two particles is given by @@ -789,7 +934,7 @@ The single() function of these pair styles returns 0.0 for the energy of a pairwise interaction, since energy is not conserved in these dissipative potentials. It also returns only the normal component of the pairwise interaction force. However, the single() function also -calculates 13 extra pairwise quantities. The first 3 are the +calculates at least 13 extra pairwise quantities. The first 3 are the components of the tangential force between particles I and J, acting on particle I. The fourth is the magnitude of this tangential force. The next 3 (5-7) are the components of the rolling torque acting on @@ -797,9 +942,17 @@ particle I. The next entry (8) is the magnitude of the rolling torque. The next entry (9) is the magnitude of the twisting torque acting about the vector connecting the two particle centers. The next 3 (10-12) are the components of the vector connecting -the centers of the two particles (x_I - x_J). The last quantity (13) -is the heat flow between the two particles, set to 0 if no heat model -is active. +the centers of the two particles (x_I - x_J). If a granular sub-model +calculates additional contact information (e.g. the heat sub-models +calculate the amount of heat exchanged), these quantities are appended +to the end of this list. First, any extra values from the normal sub-model +are appended followed by the damping, tangential, rolling, twisting, then +heat models. See the descriptions of specific granular sub-models above +for information on any extra quantities. If two or more models are +defined by pair coefficients, the size of the array is set by the +maximum number of extra quantities in a model but the order of quantities +is determined by each model's specific set of sub-models. Any unused +quantities are zeroed. These extra quantities can be accessed by the :doc:`compute pair/local ` command, as *p1*, *p2*, ..., *p12*\ . @@ -870,10 +1023,32 @@ solids. Proc. R. Soc. Lond. A, 324(1558), 301-313. .. _DMT1975: -**Derjaguin et al, 1975)** Derjaguin, B. V., Muller, V. M., & Toporov, +**(Derjaguin et al, 1975)** Derjaguin, B. V., Muller, V. M., & Toporov, Y. P. (1975). Effect of contact deformations on the adhesion of particles. Journal of Colloid and interface science, 53(2), 314-326. +.. _Zunker2024I: + +**(Zunker and Kamrin, 2024)** Zunker, W., & Kamrin, K. (2024). +A mechanically-derived contact model for adhesive elastic-perfectly +plastic particles, Part I: Utilizing the method of dimensionality +reduction. Journal of the Mechanics and Physics of Solids, 183, 105492. + +.. _Zunker2024II: + +**(Zunker and Kamrin, 2024)** Zunker, W., & Kamrin, K. (2024). +A mechanically-derived contact model for adhesive elastic-perfectly +plastic particles, Part II: Contact under high compaction-modeling +a bulk elastic response. Journal of the Mechanics and Physics of Solids, +183, 105493. + +.. _Zunker2025: + +**(Zunker et al, 2025)** Zunker, W., Dunatunga, S., Thakur, S., +Tang, P., & Kamrin, K. (2025). Experimentally validated DEM for large +deformation powder compaction: mechanically-derived contact model and +screening of non-physical contacts. + .. _Luding2008: **(Luding, 2008)** Luding, S. (2008). Cohesive, frictional powders: diff --git a/doc/src/pair_hbond_dreiding.rst b/doc/src/pair_hbond_dreiding.rst index ce19ff9e38..e059fdf2ba 100644 --- a/doc/src/pair_hbond_dreiding.rst +++ b/doc/src/pair_hbond_dreiding.rst @@ -1,30 +1,46 @@ .. index:: pair_style hbond/dreiding/lj .. index:: pair_style hbond/dreiding/lj/omp +.. index:: pair_style hbond/dreiding/lj/angleoffset +.. index:: pair_style hbond/dreiding/lj/angleoffset/omp .. index:: pair_style hbond/dreiding/morse .. index:: pair_style hbond/dreiding/morse/omp +.. index:: pair_style hbond/dreiding/morse/angleoffset +.. index:: pair_style hbond/dreiding/morse/angleoffset/omp pair_style hbond/dreiding/lj command ==================================== Accelerator Variants: *hbond/dreiding/lj/omp* +pair_style hbond/dreiding/lj/angleoffset command +================================================ + +Accelerator Variants: *hbond/dreiding/lj/angleoffset/omp* + pair_style hbond/dreiding/morse command ======================================= Accelerator Variants: *hbond/dreiding/morse/omp* +pair_style hbond/dreiding/morse/angleoffset command +=================================================== + +Accelerator Variants: *hbond/dreiding/morse/angleoffset/omp* + + Syntax """""" .. code-block:: LAMMPS - pair_style style N inner_distance_cutoff outer_distance_cutoff angle_cutoff + pair_style style N inner_distance_cutoff outer_distance_cutoff angle_cutoff equilibrium_angle -* style = *hbond/dreiding/lj* or *hbond/dreiding/morse* +* style = *hbond/dreiding/lj* or *hbond/dreiding/morse* or *hbond/dreiding/lj/angleoffset* or *hbond/dreiding/morse/angleoffset* * N = power of cosine of angle theta (integer) * inner_distance_cutoff = global inner cutoff for Donor-Acceptor interactions (distance units) * outer_distance_cutoff = global cutoff for Donor-Acceptor interactions (distance units) * angle_cutoff = global angle cutoff for Acceptor-Hydrogen-Donor interactions (degrees) +* (with style angleoffset) equilibrium_angle = global equilibrium angle for Acceptor-Hydrogen-Donor interactions (degrees) Examples """""""" @@ -40,6 +56,9 @@ Examples labelmap atom 1 C 2 O 3 H pair_coeff C O hbond/dreiding/morse H i 3.88 1.7241379 2.9 2 9.0 11.0 90.0 + pair_style hybrid/overlay lj/cut 10.0 hbond/dreiding/lj 4 9.0 11.0 90 170.0 + pair_coeff 1 2 hbond/dreiding/lj 3 i 9.5 2.75 4 9.0 11.0 90.0 + Description """"""""""" @@ -74,42 +93,53 @@ hydrogen (H) and the donor atoms: .. image:: JPG/dreiding_hbond.jpg :align: center -These 3-body interactions can be defined for pairs of acceptor and -donor atoms, based on atom types. For each donor/acceptor atom pair, -the third atom in the interaction is a hydrogen permanently bonded to -the donor atom, e.g. in a bond list read in from a data file via the +These 3-body interactions can be defined for pairs of acceptor and donor +atoms, based on atom types. For each donor/acceptor atom pair, the +third atom in the interaction is a hydrogen permanently bonded to the +donor atom, e.g. in a bond list read in from a data file via the :doc:`read_data ` command. The atom types of possible hydrogen atoms for each donor/acceptor type pair are specified by the :doc:`pair_coeff ` command (see below). Style *hbond/dreiding/lj* is the original DREIDING potential of -:ref:`(Mayo) `. It uses a LJ 12/10 functional for the Donor-Acceptor -interactions. To match the results in the original paper, use n = 4. +:ref:`(Mayo) `. It uses a LJ 12/10 functional for the +Donor-Acceptor interactions. To match the results in the original paper, +use n = 4. Style *hbond/dreiding/morse* is an improved version using a Morse potential for the Donor-Acceptor interactions. :ref:`(Liu) ` showed that the Morse form gives improved results for Dendrimer simulations, when n = 2. +.. versionadded:: TBD + +The style variants *hbond/dreiding/lj/angleoffset* and +*hbond/dreiding/lj/angleoffset* take the equilibrium angle of the AHD as +input, allowing it to reach 180 degrees. This variant option was added +to account for cases (especially in some coarse-grained models) in which +the equilibrium state of the bonds may equal the minimum energy state. + See the :doc:`Howto bioFF ` page for more information on the DREIDING force field. .. note:: - Because the Dreiding hydrogen bond potential is only one portion - of an overall force field which typically includes other pairwise - interactions, it is common to use it as a sub-style in a :doc:`pair_style hybrid/overlay ` command, where another pair style - provides the repulsive core interaction between pairs of atoms, e.g. a - 1/r\^12 Lennard-Jones repulsion. + Because the Dreiding hydrogen bond potential is only one portion of + an overall force field which typically includes other pairwise + interactions, it is common to use it as a sub-style in a + :doc:`pair_style hybrid/overlay ` command, where another + pair style provides the repulsive core interaction between pairs of + atoms, e.g. a 1/r\^12 Lennard-Jones repulsion. .. note:: - When using the hbond/dreiding pair styles with :doc:`pair_style hybrid/overlay `, you should explicitly define pair + When using the hbond/dreiding pair styles with :doc:`pair_style + hybrid/overlay `, you should explicitly define pair interactions between the donor atom and acceptor atoms, (as well as between these atoms and ALL other atoms in your system). Whenever - :doc:`pair_style hybrid/overlay ` is used, ordinary mixing - rules are not applied to atoms like the donor and acceptor atoms - because they are typically referenced in multiple pair styles. + :doc:`pair_style hybrid/overlay ` is used, ordinary + mixing rules are not applied to atoms like the donor and acceptor + atoms because they are typically referenced in multiple pair styles. Neglecting to do this can cause difficult-to-detect physics problems. .. note:: @@ -119,6 +149,13 @@ on the DREIDING force field. special_bonds command (e.g. "special_bonds lj 0.0 0.0 1.0") to turn these interactions on. +.. note:: + + For the *angleoffset* variants, the referenced angle offset is the + supplementary angle of the equilibrium angle parameter. It means if + the equilibrium angle is 166.6 degrees, the calculated angle offset + is 13.4 degrees. + ---------- The following coefficients must be defined for pairs of eligible @@ -169,7 +206,10 @@ follows: * distance cutoff :math:`r_{out}` (distance units) * angle cutoff (degrees) -A single hydrogen atom type K can be specified, or a wild-card asterisk +For both the *hbond/dreiding/lj/angleoffset* and *hbond/dreiding/morse/angleoffset* styles an additional parameter is added: +* equilibrium angle (degrees) + +For all styles, a single hydrogen atom type K can be specified, or a wild-card asterisk can be used in place of or in conjunction with the K arguments to select multiple types as hydrogen atoms. This takes the form "\*" or "\*n" or "n\*" or "m\*n". See the :doc:`pair_coeff ` @@ -245,8 +285,7 @@ heading) the following commands could be included in an input script: Restrictions """""""""""" -This pair style can only be used if LAMMPS was built with the -MOLECULE package. See the :doc:`Build package ` doc page +The base pair styles can only be used if LAMMPS was built with the MOLECULE package. The *angleoffset* variant also requires the EXTRA-MOLECULE package. See the :doc:`Build package ` doc page for more info. Related commands diff --git a/doc/src/pair_hybrid.rst b/doc/src/pair_hybrid.rst index 617b0c4372..93e5621736 100644 --- a/doc/src/pair_hybrid.rst +++ b/doc/src/pair_hybrid.rst @@ -70,6 +70,12 @@ Examples pair_coeff 1 1 lj/cut 1.0 1.0 2.5 pair_coeff 1 1 morse 1.0 1.0 1.0 2.5 + variable peratom1 atom 1/(1+exp(-$k*vx^2) + variable peratom2 atom 1-v_peratom1 + pair_style hybrid/scaled v_peratom1 lj/cut 2.5 v_peratom2 morse 2.5 + pair_coeff 1 1 lj/cut 1.0 1.0 2.5 + pair_coeff 1 1 morse 1.0 1.0 1.0 2.5 + Description """"""""""" @@ -78,7 +84,7 @@ styles enable the use of multiple pair styles in one simulation. With the *hybrid* style, exactly one pair style is assigned to each pair of atom types. With the *hybrid/overlay* and *hybrid/scaled* styles, one or more pair styles can be assigned to each pair of atom types. With -the hybrid/molecular style, pair styles are assigned to either intra- +the *hybrid/molecular* style, pair styles are assigned to either intra- or inter-molecular interactions. The assignment of pair styles to type pairs is made via the @@ -114,16 +120,26 @@ restrictions discussed below. If the *hybrid/scaled* style is used instead of *hybrid/overlay*, contributions from sub-styles are weighted by their scale factors, which -may be fractional or even negative. Furthermore the scale factors may -be variables that may change during a simulation. This enables +may be fractional or even negative. Furthermore the scale factor for +each sub-style may a constant, an *equal* style variable, or an *atom* +style variable. Variable scale factors may change during the simulation. +Different sub-styles may use different scale factor styles. +In the case of a sub-style scale factor that is an *atom* style variable, +the force contribution to each atom from that sub-style is weighted +by the value of the variable for that atom, while the contribution +from that sub-style to the global potential energy is zero. +All other contributions to the per-atom energy, per-atom +virial, and global virial (if not obtained from forces) +from that sub-style are zero. +This enables switching smoothly between two different pair styles or two different parameter sets during a run in a similar fashion as could be done with :doc:`fix adapt ` or :doc:`fix alchemy `. - All pair styles that will be used are listed as "sub-styles" following the *hybrid* or *hybrid/overlay* keyword, in any order. In case of the *hybrid/scaled* pair style, each sub-style is prefixed with a scale -factor. The scale factor is either a floating point number or an equal +factor. The scale factor is either a floating point number or an +*equal* or *atom* style (or equivalent) variable. Each sub-style's name is followed by its usual arguments, as illustrated in the examples above. See the doc pages of the individual pair styles for a listing and explanation of the @@ -374,7 +390,7 @@ between all atoms of types 1,3,4 will be computed by that potential. Pair_style hybrid allows interactions between type pairs 2-2, 1-2, 2-3, 2-4 to be specified for computation by other pair styles. You could even add a second interaction for 1-1 to be computed by another -pair style, assuming pair_style hybrid/overlay is used. +pair style, assuming pair_style *hybrid/overlay* is used. But you should not, as a general rule, attempt to exclude the many-body interactions for some subset of the type pairs within the set of 1,3,4 @@ -414,7 +430,7 @@ passed to the Tersoff potential, which means it would compute no 3-body interactions containing both type 1 and 2 atoms. Here is another example to use 2 many-body potentials together in an -overlapping manner using hybrid/overlay. Imagine you have CNT (C atoms) +overlapping manner using *hybrid/overlay*. Imagine you have CNT (C atoms) on a Si surface. You want to use Tersoff for Si/Si and Si/C interactions, and AIREBO for C/C interactions. Si atoms are type 1; C atoms are type 2. Something like this will work: diff --git a/doc/src/pair_mgpt.rst b/doc/src/pair_mgpt.rst index 6ef9f52fac..92bf9cd738 100644 --- a/doc/src/pair_mgpt.rst +++ b/doc/src/pair_mgpt.rst @@ -111,8 +111,8 @@ routines. For x-86 machines, there is a provided Makefile.mgptfast which enables the fast algebra routines, i.e. build LAMMPS with "make mgptfast". The user will be informed in the output files of the matrix kernels in use. To further improve speed, on x86 the option -precision single can be added to the :doc:`pair_coeff ` -command line, which improves speed (up to a factor of two) at the cost +*precision single* can be added to the :doc:`pair_coeff ` +command, which improves speed (up to a factor of two) at the cost of doing matrix calculations with 7 digit precision instead of the default 16. For consistency the default option can be specified explicitly by the option precision double. diff --git a/doc/src/pair_mliap.rst b/doc/src/pair_mliap.rst index e325de0aa6..7f8b36bb83 100644 --- a/doc/src/pair_mliap.rst +++ b/doc/src/pair_mliap.rst @@ -145,6 +145,7 @@ per line. The detail of *nn* module implementation can be found at :ref:`(Yanxon) `. .. admonition:: Notes on mliappy models + :class: note When the *model* keyword is *mliappy*, if the filename ends in '.pt', or '.pth', it will be loaded using pytorch; otherwise, it will be diff --git a/doc/src/pair_reaxff.rst b/doc/src/pair_reaxff.rst index 495572dc0e..45532bc2a6 100644 --- a/doc/src/pair_reaxff.rst +++ b/doc/src/pair_reaxff.rst @@ -158,11 +158,36 @@ drops to zero. Optional keywords *safezone*, *mincap*, and *minhbonds* are used for allocating reaxff arrays. Increasing these values can avoid memory problems, such as segmentation faults and bondchk failed errors, that -could occur under certain conditions. These keywords are not used by +could occur under certain conditions. These keywords are **not** used by the Kokkos version, which instead uses a more robust memory allocation scheme that checks if the sizes of the arrays have been exceeded and automatically allocates more memory. +.. admonition:: Memory management problems with ReaxFF + :class: tip + + The LAMMPS implementation of ReaxFF is adapted from a standalone MD + program written in C called `PuReMD + `_. It inherits from this code + a heuristic memory management that is different from what the rest of + LAMMPS uses. It assumes that a system is dense and already well + equilibrated, so that there are no large changes in how many and what + types of neighbors atoms have. However, not all systems are like + that, and thus there can be errors or segmentation faults if the + system changes too much. If you run into problems, here are three + options to avoid them: + + - Use the KOKKOS version of ReaxFF (KOKKOS is not only for GPUs, + but can also be compiled for serial or OpenMP execution) which + uses a different memory management approach. + - Break down a run command during which memory related errors happen + into multiple smaller segments so that the memory management + heuristics are re-initialized for each segment before they become + invalid. + - Increase the values for *safezone*, *mincap*, and *minhbonds* as + needed. This can lead to significant increase of memory consumption + through. + The keyword *tabulate* controls the size of interpolation table for Lennard-Jones and Coulomb interactions. Tabulation may also be set in the control file (see below). If tabulation is set in both the input script and the diff --git a/doc/src/pair_sph_heatconduction.rst b/doc/src/pair_sph_heatconduction.rst index 33472b64c7..5510c546d3 100644 --- a/doc/src/pair_sph_heatconduction.rst +++ b/doc/src/pair_sph_heatconduction.rst @@ -30,6 +30,12 @@ The transport model is the diffusion equation for the internal energy. See `this PDF guide `_ to using SPH in LAMMPS. +.. note:: + + Please note that the SPH PDF guide file has not been updated for + many years and thus does not reflect the current *syntax* of the + SPH package commands. For that please refer to the LAMMPS manual. + The following coefficients must be defined for each pair of atoms types via the :doc:`pair_coeff ` command as in the examples above. diff --git a/doc/src/pair_sph_idealgas.rst b/doc/src/pair_sph_idealgas.rst index 02949849c4..3bf357e2d2 100644 --- a/doc/src/pair_sph_idealgas.rst +++ b/doc/src/pair_sph_idealgas.rst @@ -36,6 +36,12 @@ particles from interpenetrating :ref:`(Monaghan) `. See `this PDF guide `_ to using SPH in LAMMPS. +.. note:: + + Please note that the SPH PDF guide file has not been updated for + many years and thus does not reflect the current *syntax* of the + SPH package commands. For that please refer to the LAMMPS manual. + The following coefficients must be defined for each pair of atoms types via the :doc:`pair_coeff ` command as in the examples above. diff --git a/doc/src/pair_sph_lj.rst b/doc/src/pair_sph_lj.rst index 4dbccc41de..53264c609b 100644 --- a/doc/src/pair_sph_lj.rst +++ b/doc/src/pair_sph_lj.rst @@ -34,6 +34,12 @@ interpenetrating :ref:`(Monaghan) `. See `this PDF guide `_ to using SPH in LAMMPS. +.. note:: + + Please note that the SPH PDF guide file has not been updated for + many years and thus does not reflect the current *syntax* of the + SPH package commands. For that please refer to the LAMMPS manual. + The following coefficients must be defined for each pair of atoms types via the :doc:`pair_coeff ` command as in the examples above. diff --git a/doc/src/pair_sph_rhosum.rst b/doc/src/pair_sph_rhosum.rst index ecbff98486..6194bacc09 100644 --- a/doc/src/pair_sph_rhosum.rst +++ b/doc/src/pair_sph_rhosum.rst @@ -29,6 +29,12 @@ SPH particles by kernel function interpolation, every Nstep timesteps. See `this PDF guide `_ to using SPH in LAMMPS. +.. note:: + + Please note that the SPH PDF guide file has not been updated for + many years and thus does not reflect the current *syntax* of the + SPH package commands. For that please refer to the LAMMPS manual. + The following coefficients must be defined for each pair of atoms types via the :doc:`pair_coeff ` command as in the examples above. diff --git a/doc/src/pair_sph_taitwater.rst b/doc/src/pair_sph_taitwater.rst index f25b69a969..bb88361f4b 100644 --- a/doc/src/pair_sph_taitwater.rst +++ b/doc/src/pair_sph_taitwater.rst @@ -41,6 +41,12 @@ prevent particles from interpenetrating :ref:`(Monaghan) `. See `this PDF guide `_ to using SPH in LAMMPS. +.. note:: + + Please note that the SPH PDF guide file has not been updated for + many years and thus does not reflect the current *syntax* of the + SPH package commands. For that please refer to the LAMMPS manual. + The following coefficients must be defined for each pair of atoms types via the :doc:`pair_coeff ` command as in the examples above. diff --git a/doc/src/pair_sph_taitwater_morris.rst b/doc/src/pair_sph_taitwater_morris.rst index cd8346364b..adef2c09c2 100644 --- a/doc/src/pair_sph_taitwater_morris.rst +++ b/doc/src/pair_sph_taitwater_morris.rst @@ -37,6 +37,12 @@ This pair style also computes laminar viscosity :ref:`(Morris) `. See `this PDF guide `_ to using SPH in LAMMPS. +.. note:: + + Please note that the SPH PDF guide file has not been updated for + many years and thus does not reflect the current *syntax* of the + SPH package commands. For that please refer to the LAMMPS manual. + The following coefficients must be defined for each pair of atoms types via the :doc:`pair_coeff ` command as in the examples above. diff --git a/doc/src/pair_style.rst b/doc/src/pair_style.rst index fc2ed2b9a9..bdf06d6b66 100644 --- a/doc/src/pair_style.rst +++ b/doc/src/pair_style.rst @@ -172,6 +172,7 @@ accelerated styles exist. * :doc:`coul/tt ` - damped charge-dipole Coulomb for Drude dipoles * :doc:`coul/wolf ` - Coulomb via Wolf potential * :doc:`coul/wolf/cs ` - Coulomb via Wolf potential with core/shell adjustments +* :doc:`dispersion/d3 ` - Dispersion correction for potentials derived from DFT functionals * :doc:`dpd ` - dissipative particle dynamics (DPD) * :doc:`dpd/coul/slater/long ` - dissipative particle dynamics (DPD) with electrostatic interactions * :doc:`dpd/ext ` - generalized force field for DPD @@ -206,7 +207,9 @@ accelerated styles exist. * :doc:`gw/zbl ` - Gao-Weber potential with a repulsive ZBL core * :doc:`harmonic/cut ` - repulsive-only harmonic potential * :doc:`hbond/dreiding/lj ` - DREIDING hydrogen bonding LJ potential +* :doc:`hbond/dreiding/lj/angleoffset ` - DREIDING hydrogen bonding LJ potential with offset for hbond angle * :doc:`hbond/dreiding/morse ` - DREIDING hydrogen bonding Morse potential +* :doc:`hbond/dreiding/morse/angleoffset ` - DREIDING hydrogen bonding Morse potential with offset for hbond angle * :doc:`hdnnp ` - High-dimensional neural network potential * :doc:`hippo ` - * :doc:`ilp/graphene/hbn ` - registry-dependent interlayer potential (ILP) diff --git a/doc/src/pair_thole.rst b/doc/src/pair_thole.rst index a4e8bbb96e..5b0d5653ff 100644 --- a/doc/src/pair_thole.rst +++ b/doc/src/pair_thole.rst @@ -131,7 +131,7 @@ command. * LJ cutoff (distance units) The last two coefficients are optional and default to the global values from -the *pair_style* command line. +the *pair_style* command. ---------- diff --git a/doc/src/read_restart.rst b/doc/src/read_restart.rst index 6a875550a4..eb7f577e4d 100644 --- a/doc/src/read_restart.rst +++ b/doc/src/read_restart.rst @@ -48,9 +48,9 @@ meaning that the trajectories of a restarted run will precisely match those produced by the original run had it continued on. Some information about a restart file can be gathered directly from the -command line when using LAMMPS with the :ref:`-restart2info -` command line flag. On Unix-like operating systems (like -Linux or macOS), one can also :ref:`configure the "file" command line +command-line when using LAMMPS with the :ref:`-restart2info +` command-line flag. On Unix-like operating systems (like +Linux or macOS), one can also :ref:`configure the "file" command-line program ` to display basic information about a restart file The binary restart file format was not designed with backward, forward, @@ -60,9 +60,9 @@ Changes to the architecture, compilation settings, or LAMMPS version can render a restart file unreadable or it may read the data incorrectly. If you want a more portable format, you can use the data file format as created by the :doc:`write_data ` command. Binary restart -files can also be converted into a data file from the command line by +files can also be converted into a data file from the command-line by the LAMMPS executable that wrote them using the :ref:`-restart2data -` command line flag. +` command-line flag. Several things can prevent exact restarts due to round-off effects, in which case the trajectories in the 2 runs will slowly diverge. These diff --git a/doc/src/set.rst b/doc/src/set.rst index 6ef654c5a7..6be590e9b6 100644 --- a/doc/src/set.rst +++ b/doc/src/set.rst @@ -516,6 +516,12 @@ Keywords *sph/e*, *sph/cv*, and *sph/rho* set the energy, heat capacity, and density of smoothed particle hydrodynamics (SPH) particles. See `this PDF guide `_ to using SPH in LAMMPS. +.. note:: + + Please note that the SPH PDF guide file has not been updated for + many years and thus does not reflect the current *syntax* of the + SPH package commands. For that please refer to the LAMMPS manual. + Keyword *smd/mass/density* sets the mass of all selected particles, but it is only applicable to the Smooth Mach Dynamics package MACHDYN. It assumes that the particle volume has already been correctly set and diff --git a/doc/src/suffix.rst b/doc/src/suffix.rst index 6c723657a5..98bb43a02d 100644 --- a/doc/src/suffix.rst +++ b/doc/src/suffix.rst @@ -30,7 +30,7 @@ Description This command allows you to use variants of various styles if they exist. In that respect it operates the same as the :doc:`-suffix command-line switch `. It also has options to turn -off or back on any suffix setting made via the command line. +off or back on any suffix setting made via the command-line. The specified style can be *gpu*, *intel*, *kk*, *omp*, *opt* or *hybrid*\ . These refer to optional packages that LAMMPS can be built diff --git a/doc/src/variable.rst b/doc/src/variable.rst index 1867532efa..2b59aeb65d 100644 --- a/doc/src/variable.rst +++ b/doc/src/variable.rst @@ -56,7 +56,7 @@ Syntax random(x,y,z), normal(x,y,z), ceil(x), floor(x), round(x), ternary(x,y,z), ramp(x,y), stagger(x,y), logfreq(x,y,z), logfreq2(x,y,z), logfreq3(x,y,z), stride(x,y,z), stride2(x,y,z,a,b,c), - vdisplace(x,y), swiggle(x,y,z), cwiggle(x,y,z) + vdisplace(x,y), swiggle(x,y,z), cwiggle(x,y,z), sign(x) group functions = count(group), mass(group), charge(group), xcm(group,dim), vcm(group,dim), fcm(group,dim), bound(group,dir), gyration(group), ke(group), @@ -71,9 +71,9 @@ Syntax feature functions = is_available(category,feature), is_active(category,feature), is_defined(category,id) atom value = id[i], mass[i], type[i], mol[i], x[i], y[i], z[i], vx[i], vy[i], vz[i], fx[i], fy[i], fz[i], q[i] atom vector = id, mass, type, mol, radius, q, x, y, z, vx, vy, vz, fx, fy, fz - custom atom property = i_name, d_name, i_name[i], d_name[i], i2_name[i], d2_name[i], i2_name[i][j], d_name[i][j] - compute references = c_ID, c_ID[i], c_ID[i][j], C_ID, C_ID[i] - fix references = f_ID, f_ID[i], f_ID[i][j], F_ID, F_ID[i] + custom atom property = i_name, d_name, i_name[i], d_name[i], i2_name[i], d2_name[i], i2_name[i][j], d2_name[i][j] + compute references = c_ID, c_ID[i], c_ID[i][j], C_ID, C_ID[i], C_ID[i][j] + fix references = f_ID, f_ID[i], f_ID[i][j], F_ID, F_ID[i], F_ID[i][j] variable references = v_name, v_name[i] vector initialization = [1,3,7,10] (for *vector* variables only) @@ -532,37 +532,37 @@ functions, special functions, feature functions, atom values, atom vectors, custom atom properties, compute references, fix references, and references to other variables. -+------------------------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| Number | 0.2, 100, 1.0e20, -15.4, etc | -+------------------------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| Constant | PI, version, on, off, true, false, yes, no | -+------------------------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| Thermo keywords | vol, pe, ebond, etc | -+------------------------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| Math operators | (), -x, x+y, x-y, x\*y, x/y, x\^y, x%y, x == y, x != y, x < y, x <= y, x > y, x >= y, x && y, x \|\| y, x \|\^ y, !x | -+------------------------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| Math functions | sqrt(x), exp(x), ln(x), log(x), abs(x), sin(x), cos(x), tan(x), asin(x), acos(x), atan(x), atan2(y,x), random(x,y,z), normal(x,y,z), ceil(x), floor(x), round(x), ternary(x,y,z), ramp(x,y), stagger(x,y), logfreq(x,y,z), logfreq2(x,y,z), logfreq3(x,y,z), stride(x,y,z), stride2(x,y,z,a,b,c), vdisplace(x,y), swiggle(x,y,z), cwiggle(x,y,z) | -+------------------------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| Group functions | count(ID), mass(ID), charge(ID), xcm(ID,dim), vcm(ID,dim), fcm(ID,dim), bound(ID,dir), gyration(ID), ke(ID), angmom(ID,dim), torque(ID,dim), inertia(ID,dimdim), omega(ID,dim) | -+------------------------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| Region functions | count(ID,IDR), mass(ID,IDR), charge(ID,IDR), xcm(ID,dim,IDR), vcm(ID,dim,IDR), fcm(ID,dim,IDR), bound(ID,dir,IDR), gyration(ID,IDR), ke(ID,IDR), angmom(ID,dim,IDR), torque(ID,dim,IDR), inertia(ID,dimdim,IDR), omega(ID,dim,IDR) | -+------------------------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| Special functions | sum(x), min(x), max(x), ave(x), trap(x), slope(x), sort(x), rsort(x), gmask(x), rmask(x), grmask(x,y), next(x), is_file(name), is_os(name), extract_setting(name), label2type(kind,label), is_typelabel(kind,label), is_timeout() | -+------------------------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| Feature functions | is_available(category,feature), is_active(category,feature), is_defined(category,id) | -+------------------------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| Atom values | id[i], mass[i], type[i], mol[i], x[i], y[i], z[i], vx[i], vy[i], vz[i], fx[i], fy[i], fz[i], q[i] | -+------------------------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| Atom vectors | id, mass, type, mol, x, y, z, vx, vy, vz, fx, fy, fz, q | -+------------------------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| Custom atom properties | i_name, d_name, i_name[i], d_name[i], i2_name[i], d2_name[i], i2_name[i][j], d_name[i][j] | -+------------------------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| Compute references | c_ID, c_ID[i], c_ID[i][j], C_ID, C_ID[i] | -+------------------------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| Fix references | f_ID, f_ID[i], f_ID[i][j], F_ID, F_ID[i] | -+------------------------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| Other variables | v_name, v_name[i] | -+------------------------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ ++------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| Number | 0.2, 100, 1.0e20, -15.4, etc | ++------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| Constant | PI, version, on, off, true, false, yes, no | ++------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| Thermo keywords | vol, pe, ebond, etc | ++------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| Math operators | (), -x, x+y, x-y, x\*y, x/y, x\^y, x%y, x == y, x != y, x < y, x <= y, x > y, x >= y, x && y, x \|\| y, x \|\^ y, !x | ++------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| Math functions | sqrt(x), exp(x), ln(x), log(x), abs(x), sin(x), cos(x), tan(x), asin(x), acos(x), atan(x), atan2(y,x), random(x,y,z), normal(x,y,z), ceil(x), floor(x), round(x), ternary(x,y,z), ramp(x,y), stagger(x,y), logfreq(x,y,z), logfreq2(x,y,z), logfreq3(x,y,z), stride(x,y,z), stride2(x,y,z,a,b,c), vdisplace(x,y), swiggle(x,y,z), cwiggle(x,y,z), sign(x) | ++------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| Group functions | count(ID), mass(ID), charge(ID), xcm(ID,dim), vcm(ID,dim), fcm(ID,dim), bound(ID,dir), gyration(ID), ke(ID), angmom(ID,dim), torque(ID,dim), inertia(ID,dimdim), omega(ID,dim) | ++------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| Region functions | count(ID,IDR), mass(ID,IDR), charge(ID,IDR), xcm(ID,dim,IDR), vcm(ID,dim,IDR), fcm(ID,dim,IDR), bound(ID,dir,IDR), gyration(ID,IDR), ke(ID,IDR), angmom(ID,dim,IDR), torque(ID,dim,IDR), inertia(ID,dimdim,IDR), omega(ID,dim,IDR) | ++------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| Special functions | sum(x), min(x), max(x), ave(x), trap(x), slope(x), sort(x), rsort(x), gmask(x), rmask(x), grmask(x,y), next(x), is_file(name), is_os(name), extract_setting(name), label2type(kind,label), is_typelabel(kind,label), is_timeout() | ++------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| Feature functions | is_available(category,feature), is_active(category,feature), is_defined(category,id) | ++------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| Atom values | id[i], mass[i], type[i], mol[i], x[i], y[i], z[i], vx[i], vy[i], vz[i], fx[i], fy[i], fz[i], q[i] | ++------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| Atom vectors | id, mass, type, mol, x, y, z, vx, vy, vz, fx, fy, fz, q | ++------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| Custom atom properties | i_name, d_name, i_name[i], d_name[i], i2_name[i], d2_name[i], i2_name[i][j], d_name[i][j] | ++------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| Compute references | c_ID, c_ID[i], c_ID[i][j], C_ID, C_ID[i] | ++------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| Fix references | f_ID, f_ID[i], f_ID[i][j], F_ID, F_ID[i] | ++------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| Other variables | v_name, v_name[i] | ++------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ Most of the formula elements produce a scalar value. Some produce a global or per-atom vector of values. Global vectors can be produced @@ -860,6 +860,9 @@ run, according to one of these formulas, where omega = 2 PI / period: where dt = the timestep size. +The sign(x) function returns 1.0 if the value is greater than or equal +to 0.0, and -1.0 otherwise. + The run begins on startstep. Startstep can span multiple runs, using the *start* keyword of the :doc:`run ` command. See the :doc:`run ` command for details of how to do this. Note that the diff --git a/doc/utils/requirements.txt b/doc/utils/requirements.txt index acf575fe58..d842f47c11 100644 --- a/doc/utils/requirements.txt +++ b/doc/utils/requirements.txt @@ -9,3 +9,4 @@ Pygments six pyyaml linkchecker +ipython diff --git a/doc/utils/sphinx-config/false_positives.txt b/doc/utils/sphinx-config/false_positives.txt index 8e601d6c16..f9fb3a0f8a 100644 --- a/doc/utils/sphinx-config/false_positives.txt +++ b/doc/utils/sphinx-config/false_positives.txt @@ -25,6 +25,7 @@ Ackland acks acolor acos +acs Acta actinide activationfunctions @@ -107,6 +108,7 @@ Andrienko Andzelm Ang anglegrad +angleoffset angletangrad angmom angmomx @@ -248,6 +250,7 @@ basename Bashford bashrc Baskes +Batatia Batra Bayly bb @@ -255,6 +258,7 @@ bcc bcolor bdiam bdw +Becke Beckman Becton Behler @@ -313,6 +317,7 @@ bitrate bitrates Bitzek Bjerrum +bjm Bkappa blabel Blaise @@ -1232,6 +1237,7 @@ fp fphi fPIC fplo +fprintf Fqq Fraige framerate @@ -1342,6 +1348,7 @@ gmres gname gneb GNEB +Goerigk Goga Goldfarb Gompper @@ -1577,6 +1584,7 @@ Impropers imulator includelink incompressible +incompressibility incrementing indenter indenters @@ -1707,6 +1715,7 @@ Jaramillo Jarzynski jatempl javascript +jcc jcp jea jec @@ -1728,6 +1737,7 @@ Jonsson Jorgensen jp jparam +jpclett jpeg jpeglib jpg @@ -1754,6 +1764,7 @@ Kadiri Kai Kalia Kamberaj +Kamrin Kantorovich Kapfer Kapil @@ -1844,6 +1855,7 @@ Krass Kraus Kremer Kress +Krieg Kronik ksh kspace @@ -1916,6 +1928,7 @@ lB lbfgs lbl LBtype +lc lcbop ld lda @@ -2062,6 +2075,7 @@ ly Lybrand Lykotrafitis lyon +lyp Lysogorskiy Lyulin lz @@ -2116,6 +2130,7 @@ Marchi Mariella Marinica Markland +Marko Marrink Marroquin Marsaglia @@ -2198,6 +2213,7 @@ Meissner Melchor Meloni Melrose +meltable mem Mem memalign @@ -2408,6 +2424,7 @@ mV Mvapich mvh mvv +mwlc MxN myCompute myIndex @@ -2499,6 +2516,7 @@ neel Neel Neelov Negre +Negrut nelem Nelement Nelements @@ -2517,6 +2535,7 @@ Nevery Nevins newfile Newns +newstep newtype nextsort Neyts @@ -2823,12 +2842,14 @@ pathangle pathname pathnames Patera +Patkowski Patomtrans Pattnaik Pavese Pavia Paxton pbc +pbe pc pcg pchain @@ -2919,6 +2940,7 @@ ploop PloS plt plumedfile +plyp pmb pmcmoves pme @@ -2928,6 +2950,7 @@ Pmoltrans pN png podd +Podgornik Podhorszki Poiseuille poisson @@ -3060,6 +3083,7 @@ qE qeff qelectron qeq +Qamar QeQ QEq qfactor @@ -3116,6 +3140,7 @@ Rafferty rahman Rahman Rajamanickam +Rakhsha Ralf Raman ramped @@ -3360,6 +3385,7 @@ Schilfgarde Schimansky Schiotz Schlitter +Schmerler Schmid Schnieders Schoen @@ -3426,6 +3452,7 @@ Shardlow shawn Shen Shenderova +Sherrill Shi Shiga Shinoda @@ -3708,6 +3735,7 @@ tgnpt tgnvt th Thakkar +Thakur Thaokar thb thei @@ -3746,6 +3774,7 @@ Tigran Tij Tildesley Timan +timeflag timeI timespan timestamp @@ -3808,6 +3837,7 @@ Tref Tretyakov tri triangleflag +triaxial Tribello triclinic Triclinic @@ -4021,6 +4051,7 @@ VMDARCH VMDHOME vn Voigt +Vogel volfactor Volkov Volpe @@ -4111,6 +4142,8 @@ workflow workflows Workum Worley +wormlike +wpbe Wriggers writedata Wuppertal @@ -4179,6 +4212,7 @@ yaff YAFF Yamada yaml +Yan Yanxon Yaser Yazdani @@ -4230,6 +4264,7 @@ Zemer zenodo Zentrum Zepeda +zerom zflag Zhang Zhao diff --git a/examples/COUPLE/plugin/liblammpsplugin.c b/examples/COUPLE/plugin/liblammpsplugin.c index 5b1308a5cc..619b8828fc 100644 --- a/examples/COUPLE/plugin/liblammpsplugin.c +++ b/examples/COUPLE/plugin/liblammpsplugin.c @@ -117,6 +117,10 @@ liblammpsplugin_t *liblammpsplugin_load(const char *lib) ADDSYM(set_string_variable); ADDSYM(set_internal_variable); ADDSYM(variable_info); + ADDSYM(eval); + ADDSYM(clearstep_compute); + ADDSYM(addstep_compute); + ADDSYM(addstep_compute_all); ADDSYM(gather_atoms); ADDSYM(gather_atoms_concat); diff --git a/examples/COUPLE/plugin/liblammpsplugin.h b/examples/COUPLE/plugin/liblammpsplugin.h index b2df98d630..c765b0adc3 100644 --- a/examples/COUPLE/plugin/liblammpsplugin.h +++ b/examples/COUPLE/plugin/liblammpsplugin.h @@ -163,6 +163,10 @@ struct _liblammpsplugin { int (*set_string_variable)(void *, const char *, const char *); int (*set_internal_variable)(void *, const char *, double); int (*variable_info)(void *, int, char *, int); + double (*eval)(void *, const char *); + void (*clearstep_compute)(void *); + void (*addstep_compute)(void *, void *); + void (*addstep_compute_all)(void *, void *); void (*gather_atoms)(void *, const char *, int, int, void *); void (*gather_atoms_concat)(void *, const char *, int, int, void *); diff --git a/examples/COUPLE/plugin/simple.c b/examples/COUPLE/plugin/simple.c index 9151f92848..5e7c2e98f5 100644 --- a/examples/COUPLE/plugin/simple.c +++ b/examples/COUPLE/plugin/simple.c @@ -21,6 +21,7 @@ #include +#include #include #include #include @@ -63,7 +64,7 @@ int main(int narg, char **arg) nprocs_lammps = atoi(arg[1]); if (nprocs_lammps > nprocs) { if (me == 0) - printf("ERROR: LAMMPS cannot use more procs than available\n"); + printf("ERROR: LAMMPS cannot use more procs than available: %d\n", nprocs); MPI_Abort(MPI_COMM_WORLD,1); } @@ -76,7 +77,7 @@ int main(int narg, char **arg) if (me == 0) { fp = fopen(arg[2],"r"); if (fp == NULL) { - printf("ERROR: Could not open LAMMPS input script\n"); + printf("ERROR: Could not open LAMMPS input script %s: %s\n", arg[2], strerror(errno)); MPI_Abort(MPI_COMM_WORLD,1); } } @@ -87,9 +88,10 @@ int main(int narg, char **arg) all LAMMPS procs call lammps_command() on the line */ if (lammps == 1) { + errno = 0; plugin = liblammpsplugin_load(arg[3]); if (plugin == NULL) { - if (me == 0) printf("ERROR: Could not load shared LAMMPS library\n"); + if (me == 0) printf("ERROR: Could not load shared LAMMPS library file %s: %s\n", arg[3], strerror(errno)); MPI_Abort(MPI_COMM_WORLD,1); } /* must match the plugin ABI version */ diff --git a/examples/LEPTON/in.efield-lepton b/examples/LEPTON/in.efield-lepton new file mode 100644 index 0000000000..176eb3ca63 --- /dev/null +++ b/examples/LEPTON/in.efield-lepton @@ -0,0 +1,50 @@ +# Point dipoles in a 3d box with an external potential (ignoring dipolar interactions) + +units lj +atom_style hybrid sphere dipole +dimension 3 +boundary s s s +region box block -2 2 -2 2 -2 2 + +create_box 1 box +create_atoms 1 random 100 12345 NULL + +# need both mass settings due to hybrid atom style +mass 1 1.0 +set group all mass 1.0 +set group all diameter 0.1 + +set group all dipole/random 98934 0.01 +pair_style none +comm_modify cutoff 3.0 + +velocity all create 0.0 87287 mom yes rot yes + +fix 1 all nve/sphere update dipole + +############################################################################################################### +## Yukawa potential +#fix 2 all efield/lepton "A*exp(-B*r)/r; r=abs(sqrt(x^2+y^2+z^2)); A = 0.1; B = 5" step 1e-8 + +## Gradually increasing uniform field +#variable E equal ramp(0,1) +#fix 2 all efield/lepton "-v_E*(x+y+z)" + +## Linear gradient field +fix 2 all efield/lepton "-1/6*x^3" step 1e-6 + +fix_modify 2 energy yes + +############################################################################################################### + +timestep 1e-3 + +compute erot all erotate/sphere +variable etotal equal "ke + c_erot + pe" # thermo etotal doesn't include erot +thermo_style custom step temp ke c_erot pe v_etotal +thermo 500 +thermo_modify norm no + +#dump 1 all custom 500 dump.dipole id x y z diameter mux muy muz fx fy fz tqx tqy tqz + +run 10000 diff --git a/examples/LEPTON/log.13Jan25.efield-lepton.g++.1 b/examples/LEPTON/log.13Jan25.efield-lepton.g++.1 new file mode 100644 index 0000000000..8954e045ab --- /dev/null +++ b/examples/LEPTON/log.13Jan25.efield-lepton.g++.1 @@ -0,0 +1,115 @@ +LAMMPS (19 Nov 2024 - Development - patch_19Nov2024-283-g742c869534-modified) + using 1 OpenMP thread(s) per MPI task +# Point dipoles in a 3d box with an external potential (ignoring dipolar interactions) + +units lj +atom_style hybrid sphere dipole +WARNING: Atom style hybrid defines both, per-type and per-atom masses; both must be set, but only per-atom masses will be used (src/atom_vec_hybrid.cpp:132) +dimension 3 +boundary s s s +region box block -2 2 -2 2 -2 2 + +create_box 1 box +Created orthogonal box = (-2 -2 -2) to (2 2 2) + 1 by 1 by 1 MPI processor grid +create_atoms 1 random 100 12345 NULL +Created 100 atoms + using lattice units in orthogonal box = (-2.0004 -2.0004 -2.0004) to (2.0004 2.0004 2.0004) + create_atoms CPU = 0.000 seconds + +# need both mass settings due to hybrid atom style +mass 1 1.0 +set group all mass 1.0 +Setting atom values ... + 100 settings made for mass +set group all diameter 0.1 +Setting atom values ... + 100 settings made for diameter + +set group all dipole/random 98934 0.01 +Setting atom values ... + 100 settings made for dipole/random +pair_style none +comm_modify cutoff 3.0 + +velocity all create 0.0 87287 mom yes rot yes + +fix 1 all nve/sphere update dipole + +############################################################################################################### +## Yukawa potential +#fix 2 all efield/lepton "A*exp(-B*r)/r; r=abs(sqrt(x^2+y^2+z^2)); A = 0.1; B = 5" step 1e-8 + +## Gradually increasing uniform field +#variable E equal ramp(0,1) +#fix 2 all efield/lepton "-v_E*(x+y+z)" + +## Linear gradient field +fix 2 all efield/lepton "-1/6*x^3" step 1e-6 + +fix_modify 2 energy yes + +############################################################################################################### + +timestep 1e-3 + +compute erot all erotate/sphere +variable etotal equal "ke + c_erot + pe" # thermo etotal doesn't include erot +thermo_style custom step temp ke c_erot pe v_etotal +thermo 500 +thermo_modify norm no + +#dump 1 all custom 500 dump.dipole id x y z diameter mux muy muz fx fy fz tqx tqy tqz + +run 10000 +WARNING: No pairwise cutoff or binsize set. Atom sorting therefore disabled. (src/atom.cpp:2442) +Per MPI rank memory allocation (min/avg/max) = 4.273 | 4.273 | 4.273 Mbytes + Step Temp KinEng c_erot PotEng v_etotal + 0 0 0 0 0.036419797 0.036419797 + 500 3.7159175e-06 0.00055181374 0.44262618 -0.40675701 0.036420985 + 1000 1.2808438e-05 0.0019020531 0.24499116 -0.21047295 0.036420259 + 1500 2.8343769e-05 0.0042090498 0.26504485 -0.2328336 0.036420307 + 2000 4.8796894e-05 0.0072463388 0.30953526 -0.28036098 0.036420618 + 2500 7.8933715e-05 0.011721657 0.2015076 -0.17680909 0.036420173 + 3000 0.00011381678 0.016901791 0.31002163 -0.29050294 0.036420476 + 3500 0.00015650339 0.023240753 0.27837968 -0.26520001 0.036420418 + 4000 0.00020429109 0.030337227 0.26201101 -0.25592795 0.036420289 + 4500 0.00026362339 0.039148074 0.29769952 -0.3004271 0.036420499 + 5000 0.00033328941 0.049493478 0.21642442 -0.22949776 0.036420131 + 5500 0.00040914224 0.060757622 0.28422322 -0.30856047 0.036420377 + 6000 0.00049425119 0.073396302 0.31767 -0.35464572 0.03642058 + 6500 0.00058508892 0.086885704 0.29079532 -0.34126075 0.036420276 + 7000 0.00069845073 0.10371993 0.25776048 -0.32506015 0.036420262 + 7500 0.0008215656 0.12200249 0.27033777 -0.35591972 0.036420539 + 8000 0.00095528125 0.14185927 0.33943527 -0.44487406 0.036420479 + 8500 0.0011052502 0.16412965 0.26727165 -0.39498109 0.036420218 + 9000 0.0012738298 0.18916373 0.31082058 -0.46356382 0.036420485 + 9500 0.001464197 0.21743325 0.25669856 -0.43771158 0.036420224 + 10000 0.0016627654 0.24692067 0.36273185 -0.57323194 0.036420578 +Loop time of 0.84714 on 1 procs for 10000 steps with 100 atoms + +Performance: 1019901.911 tau/day, 11804.420 timesteps/s, 1.180 Matom-step/s +62.3% CPU use with 1 MPI tasks x 1 OpenMP threads + +MPI task timing breakdown: +Section | min time | avg time | max time |%varavg| %total +--------------------------------------------------------------- +Pair | 0 | 0 | 0 | 0.0 | 0.00 +Neigh | 9.21e-07 | 9.21e-07 | 9.21e-07 | 0.0 | 0.00 +Comm | 0.00094138 | 0.00094138 | 0.00094138 | 0.0 | 0.11 +Output | 0.0001983 | 0.0001983 | 0.0001983 | 0.0 | 0.02 +Modify | 0.84105 | 0.84105 | 0.84105 | 0.0 | 99.28 +Other | | 0.004946 | | | 0.58 + +Nlocal: 100 ave 100 max 100 min +Histogram: 1 0 0 0 0 0 0 0 0 0 +Nghost: 0 ave 0 max 0 min +Histogram: 1 0 0 0 0 0 0 0 0 0 +Neighs: 0 ave 0 max 0 min +Histogram: 1 0 0 0 0 0 0 0 0 0 + +Total # of neighbors = 0 +Ave neighs/atom = 0 +Neighbor list builds = 5 +Dangerous builds = 0 +Total wall time: 0:00:00 diff --git a/examples/LEPTON/log.13Jan25.efield-lepton.g++.4 b/examples/LEPTON/log.13Jan25.efield-lepton.g++.4 new file mode 100644 index 0000000000..44243006a9 --- /dev/null +++ b/examples/LEPTON/log.13Jan25.efield-lepton.g++.4 @@ -0,0 +1,115 @@ +LAMMPS (19 Nov 2024 - Development - patch_19Nov2024-283-g742c869534-modified) + using 1 OpenMP thread(s) per MPI task +# Point dipoles in a 3d box with an external potential (ignoring dipolar interactions) + +units lj +atom_style hybrid sphere dipole +WARNING: Atom style hybrid defines both, per-type and per-atom masses; both must be set, but only per-atom masses will be used (src/atom_vec_hybrid.cpp:132) +dimension 3 +boundary s s s +region box block -2 2 -2 2 -2 2 + +create_box 1 box +Created orthogonal box = (-2 -2 -2) to (2 2 2) + 1 by 2 by 2 MPI processor grid +create_atoms 1 random 100 12345 NULL +Created 100 atoms + using lattice units in orthogonal box = (-2.0004 -2.0004 -2.0004) to (2.0004 2.0004 2.0004) + create_atoms CPU = 0.000 seconds + +# need both mass settings due to hybrid atom style +mass 1 1.0 +set group all mass 1.0 +Setting atom values ... + 100 settings made for mass +set group all diameter 0.1 +Setting atom values ... + 100 settings made for diameter + +set group all dipole/random 98934 0.01 +Setting atom values ... + 100 settings made for dipole/random +pair_style none +comm_modify cutoff 3.0 + +velocity all create 0.0 87287 mom yes rot yes + +fix 1 all nve/sphere update dipole + +############################################################################################################### +## Yukawa potential +#fix 2 all efield/lepton "A*exp(-B*r)/r; r=abs(sqrt(x^2+y^2+z^2)); A = 0.1; B = 5" step 1e-8 + +## Gradually increasing uniform field +#variable E equal ramp(0,1) +#fix 2 all efield/lepton "-v_E*(x+y+z)" + +## Linear gradient field +fix 2 all efield/lepton "-1/6*x^3" step 1e-6 + +fix_modify 2 energy yes + +############################################################################################################### + +timestep 1e-3 + +compute erot all erotate/sphere +variable etotal equal "ke + c_erot + pe" # thermo etotal doesn't include erot +thermo_style custom step temp ke c_erot pe v_etotal +thermo 500 +thermo_modify norm no + +#dump 1 all custom 500 dump.dipole id x y z diameter mux muy muz fx fy fz tqx tqy tqz + +run 10000 +WARNING: No pairwise cutoff or binsize set. Atom sorting therefore disabled. (src/atom.cpp:2442) +Per MPI rank memory allocation (min/avg/max) = 4.289 | 4.289 | 4.289 Mbytes + Step Temp KinEng c_erot PotEng v_etotal + 0 0 0 0 0.036419797 0.036419797 + 500 3.7159175e-06 0.00055181374 0.44262618 -0.40675701 0.036420985 + 1000 1.2808438e-05 0.0019020531 0.24499116 -0.21047295 0.036420259 + 1500 2.8343769e-05 0.0042090498 0.26504485 -0.2328336 0.036420307 + 2000 4.8796894e-05 0.0072463388 0.30953526 -0.28036098 0.036420618 + 2500 7.8933715e-05 0.011721657 0.2015076 -0.17680909 0.036420173 + 3000 0.00011381678 0.016901791 0.31002163 -0.29050294 0.036420476 + 3500 0.00015650339 0.023240753 0.27837968 -0.26520001 0.036420418 + 4000 0.00020429109 0.030337227 0.26201101 -0.25592795 0.036420289 + 4500 0.00026362339 0.039148074 0.29769952 -0.3004271 0.036420499 + 5000 0.00033328941 0.049493478 0.21642442 -0.22949776 0.036420131 + 5500 0.00040914224 0.060757622 0.28422322 -0.30856047 0.036420377 + 6000 0.00049425119 0.073396302 0.31767 -0.35464572 0.03642058 + 6500 0.00058508892 0.086885704 0.29079532 -0.34126075 0.036420276 + 7000 0.00069845073 0.10371993 0.25776048 -0.32506015 0.036420262 + 7500 0.0008215656 0.12200249 0.27033777 -0.35591972 0.036420539 + 8000 0.00095528125 0.14185927 0.33943527 -0.44487406 0.036420479 + 8500 0.0011052502 0.16412965 0.26727165 -0.39498109 0.036420218 + 9000 0.0012738298 0.18916373 0.31082058 -0.46356382 0.036420485 + 9500 0.001464197 0.21743325 0.25669856 -0.43771158 0.036420224 + 10000 0.0016627654 0.24692067 0.36273185 -0.57323194 0.036420578 +Loop time of 0.985035 on 4 procs for 10000 steps with 100 atoms + +Performance: 877125.838 tau/day, 10151.919 timesteps/s, 1.015 Matom-step/s +67.7% CPU use with 4 MPI tasks x 1 OpenMP threads + +MPI task timing breakdown: +Section | min time | avg time | max time |%varavg| %total +--------------------------------------------------------------- +Pair | 0 | 0 | 0 | 0.0 | 0.00 +Neigh | 7.22e-07 | 8.9125e-07 | 1.031e-06 | 0.0 | 0.00 +Comm | 0.09818 | 0.1024 | 0.10798 | 1.1 | 10.40 +Output | 0.00021634 | 0.00028668 | 0.00044312 | 0.0 | 0.03 +Modify | 0.773 | 0.81845 | 0.84055 | 3.0 | 83.09 +Other | | 0.06389 | | | 6.49 + +Nlocal: 25 ave 30 max 23 min +Histogram: 2 1 0 0 0 0 0 0 0 1 +Nghost: 75 ave 77 max 70 min +Histogram: 1 0 0 0 0 0 0 0 1 2 +Neighs: 0 ave 0 max 0 min +Histogram: 4 0 0 0 0 0 0 0 0 0 + +Total # of neighbors = 0 +Ave neighs/atom = 0 +Neighbor list builds = 5 +Dangerous builds = 0 +Total wall time: 0:00:00 diff --git a/examples/PACKAGES/dispersion/README b/examples/PACKAGES/dispersion/README new file mode 100644 index 0000000000..822713f1c1 --- /dev/null +++ b/examples/PACKAGES/dispersion/README @@ -0,0 +1,6 @@ +To run these examples, one needs to compile LAMMPS with the ML-PACE (-DPKG_ML-PACE=ON) and the EXTRA-PAIR packages (-DPKG_EXTRA-PAIR=ON). + +These examples show how to combine a short-ranged ML potential with a dispersion correction scheme. Here we combine a general-purpose ACE potential for carbon (10.1021/acs.jctc.2c01149), with two different dispersion correction schemes: + - D2 (a pure two-body potential, here tabulated) + - D3 (a many-body potential, implemented in LAMMPS). + diff --git a/examples/PACKAGES/dispersion/in.d2 b/examples/PACKAGES/dispersion/in.d2 new file mode 100644 index 0000000000..f2652a9af4 --- /dev/null +++ b/examples/PACKAGES/dispersion/in.d2 @@ -0,0 +1,36 @@ +atom_style atomic +units metal +boundary p p p +atom_modify sort 0 0.0 + +lattice sc 1.0 +region box block 0 10 0 10 0 10 +create_box 1 box +create_atoms 1 region box + +variable l equal 47.6 +change_box all x final 0 $l y final 0 $l z final 0 $l remap +region world block INF INF INF INF INF INF + +### interactions +pair_style hybrid/overlay pace table linear 10000 +pair_coeff * * pace potential_files/c_ace.yace C +pair_coeff * * table potential_files/d2.table D2 9.0 + +mass 1 12.011000 +velocity all create 200 1234 + +compute c1 all pair pace +compute c2 all pair table + +# calculate the e/atom for each pair style individually +variable dUpace equal c_c1/atoms +variable dUd2 equal c_c2/atoms + +### run +timestep 0.001 +fix 1 all nvt temp 200.0 200.0 0.01 +thermo_style custom step temp pe press etotal v_dUpace v_dUd2 +thermo 10 +run 100 + diff --git a/examples/PACKAGES/dispersion/in.d3 b/examples/PACKAGES/dispersion/in.d3 new file mode 100644 index 0000000000..5e0bcd2587 --- /dev/null +++ b/examples/PACKAGES/dispersion/in.d3 @@ -0,0 +1,36 @@ +atom_style atomic +units metal +boundary p p p +atom_modify sort 0 0.0 + +lattice sc 1.0 +region box block 0 10 0 10 0 10 +create_box 1 box +create_atoms 1 region box + +variable l equal 47.6 +change_box all x final 0 $l y final 0 $l z final 0 $l remap +region world block INF INF INF INF INF INF + +### interactions +pair_style hybrid/overlay pace dispersion/d3 bj pbe 16.0 16.0 +pair_coeff * * pace potential_files/c_ace.yace C +pair_coeff * * dispersion/d3 C + +mass 1 12.011000 +velocity all create 200 1234 + +compute c1 all pair pace +compute c2 all pair dispersion/d3 + +# calculate the e/atom for each pair style individually +variable Upace equal c_c1/atoms +variable Ud3 equal c_c2/atoms + +### run +timestep 0.001 +fix 1 all nvt temp 200.0 200.0 0.01 +thermo_style custom step temp pe press etotal v_Upace v_Ud3 +thermo 10 +run 100 + diff --git a/examples/PACKAGES/dispersion/log.20Dec2024.d2.g++.1 b/examples/PACKAGES/dispersion/log.20Dec2024.d2.g++.1 new file mode 100644 index 0000000000..b0d299b980 --- /dev/null +++ b/examples/PACKAGES/dispersion/log.20Dec2024.d2.g++.1 @@ -0,0 +1,122 @@ +LAMMPS (19 Nov 2024 - Development - patch_19Nov2024-125-g095d33dafb) +OMP_NUM_THREADS environment is not set. Defaulting to 1 thread. (src/comm.cpp:99) + using 1 OpenMP thread(s) per MPI task +atom_style atomic +units metal +boundary p p p +atom_modify sort 0 0.0 + +lattice sc 1.0 +Lattice spacing in x,y,z = 1 1 1 +region box block 0 10 0 10 0 10 +create_box 1 box +Created orthogonal box = (0 0 0) to (10 10 10) + 1 by 1 by 1 MPI processor grid +create_atoms 1 region box +Created 1000 atoms + using lattice units in orthogonal box = (0 0 0) to (10 10 10) + create_atoms CPU = 0.000 seconds + +variable l equal 47.6 +change_box all x final 0 $l y final 0 $l z final 0 $l remap +change_box all x final 0 47.6 y final 0 $l z final 0 $l remap +change_box all x final 0 47.6 y final 0 47.6 z final 0 $l remap +change_box all x final 0 47.6 y final 0 47.6 z final 0 47.6 remap +Changing box ... + orthogonal box = (0 0 0) to (47.6 10 10) + orthogonal box = (0 0 0) to (47.6 47.6 10) + orthogonal box = (0 0 0) to (47.6 47.6 47.6) +region world block INF INF INF INF INF INF + +### interactions +pair_style hybrid/overlay pace table linear 10000 +ACE version: 2023.11.25 +Recursive evaluator is used +pair_coeff * * pace potential_files/c_ace.yace C +Loading potential_files/c_ace.yace +Total number of basis functions + C: 20 (r=1) 455 (r>1) +Mapping LAMMPS atom type #1(C) -> ACE species type #0 +pair_coeff * * table potential_files/d2.table D2 9.0 +Reading pair table potential file potential_files/d2.table with DATE: 2021-12-16 +WARNING: 8063 of 20000 force values in table D2 are inconsistent with -dE/dr. +WARNING: Should only be flagged at inflection points (src/pair_table.cpp:466) +WARNING: 2386 of 20000 distance values in table 1e-06 with relative error +WARNING: over D2 to re-computed values (src/pair_table.cpp:474) + +mass 1 12.011000 +velocity all create 200 1234 + +compute c1 all pair pace +compute c2 all pair table + +# calculate the e/atom for each pair style individually +variable dUpace equal c_c1/atoms +variable dUd2 equal c_c2/atoms + +### run +timestep 0.001 +fix 1 all nvt temp 200.0 200.0 0.01 +thermo_style custom step temp pe press etotal v_dUpace v_dUd2 +thermo 10 +run 100 +Neighbor list info ... + update: every = 1 steps, delay = 0 steps, check = yes + max neighbors/atom: 2000, page size: 100000 + master list distance cutoff = 11 + ghost atom cutoff = 11 + binsize = 5.5, bins = 9 9 9 + 2 neighbor lists, perpetual/occasional/extra = 2 0 0 + (1) pair pace, perpetual + attributes: full, newton on, cut 7.5 + pair build: full/bin/atomonly + stencil: full/bin/3d + bin: standard + (2) pair table, perpetual + attributes: half, newton on + pair build: half/bin/atomonly/newton + stencil: half/bin/3d + bin: standard +Per MPI rank memory allocation (min/avg/max) = 3.735 | 3.735 | 3.735 Mbytes + Step Temp PotEng Press TotEng v_dUpace v_dUd2 + 0 200 -262.26589 -9971.6713 -236.43971 -0.2577066 -0.0045592958 + 10 198.01563 -261.95164 -9936.5218 -236.38171 -0.25738489 -0.004566747 + 20 199.80384 -261.06484 -9826.0969 -235.26399 -0.25647577 -0.0045890709 + 30 200.79867 -259.7549 -9655.8924 -233.82559 -0.25512792 -0.0046269853 + 40 194.7303 -258.36397 -9450.9508 -233.21827 -0.25368377 -0.004680203 + 50 197.08802 -257.40377 -9200.5727 -231.95362 -0.25265301 -0.0047507608 + 60 204.21755 -257.66495 -8919.2309 -231.29416 -0.25282305 -0.0048419012 + 70 216.81983 -260.19034 -8702.5441 -232.19221 -0.25523198 -0.0049583602 + 80 242.71952 -266.40641 -8617.9868 -235.06383 -0.26129243 -0.0051139831 + 90 294.45869 -279.46195 -8724.2954 -241.43824 -0.27411961 -0.0053423377 + 100 400.44323 -307.29577 -9070.6387 -255.58618 -0.30165815 -0.0056376175 +Loop time of 2.66184 on 1 procs for 100 steps with 1000 atoms + +Performance: 3.246 ns/day, 7.394 hours/ns, 37.568 timesteps/s, 37.568 katom-step/s +99.6% CPU use with 1 MPI tasks x 1 OpenMP threads + +MPI task timing breakdown: +Section | min time | avg time | max time |%varavg| %total +--------------------------------------------------------------- +Pair | 2.6584 | 2.6584 | 2.6584 | 0.0 | 99.87 +Neigh | 0.0012861 | 0.0012861 | 0.0012861 | 0.0 | 0.05 +Comm | 0.00064617 | 0.00064617 | 0.00064617 | 0.0 | 0.02 +Output | 0.00024173 | 0.00024173 | 0.00024173 | 0.0 | 0.01 +Modify | 0.00099328 | 0.00099328 | 0.00099328 | 0.0 | 0.04 +Other | | 0.0002431 | | | 0.01 + +Nlocal: 1000 ave 1000 max 1000 min +Histogram: 1 0 0 0 0 0 0 0 0 0 +Nghost: 2375 ave 2375 max 2375 min +Histogram: 1 0 0 0 0 0 0 0 0 0 +Neighs: 26027 ave 26027 max 26027 min +Histogram: 1 0 0 0 0 0 0 0 0 0 +FullNghs: 17736 ave 17736 max 17736 min +Histogram: 1 0 0 0 0 0 0 0 0 0 + +Total # of neighbors = 26027 +Ave neighs/atom = 26.027 +Neighbor list builds = 1 +Dangerous builds = 0 + +Total wall time: 0:00:02 diff --git a/examples/PACKAGES/dispersion/log.20Dec2024.d2.g++.4 b/examples/PACKAGES/dispersion/log.20Dec2024.d2.g++.4 new file mode 100644 index 0000000000..b5b4b10275 --- /dev/null +++ b/examples/PACKAGES/dispersion/log.20Dec2024.d2.g++.4 @@ -0,0 +1,122 @@ +LAMMPS (19 Nov 2024 - Development - patch_19Nov2024-125-g095d33dafb) +OMP_NUM_THREADS environment is not set. Defaulting to 1 thread. (src/comm.cpp:99) + using 1 OpenMP thread(s) per MPI task +atom_style atomic +units metal +boundary p p p +atom_modify sort 0 0.0 + +lattice sc 1.0 +Lattice spacing in x,y,z = 1 1 1 +region box block 0 10 0 10 0 10 +create_box 1 box +Created orthogonal box = (0 0 0) to (10 10 10) + 1 by 2 by 2 MPI processor grid +create_atoms 1 region box +Created 1000 atoms + using lattice units in orthogonal box = (0 0 0) to (10 10 10) + create_atoms CPU = 0.000 seconds + +variable l equal 47.6 +change_box all x final 0 $l y final 0 $l z final 0 $l remap +change_box all x final 0 47.6 y final 0 $l z final 0 $l remap +change_box all x final 0 47.6 y final 0 47.6 z final 0 $l remap +change_box all x final 0 47.6 y final 0 47.6 z final 0 47.6 remap +Changing box ... + orthogonal box = (0 0 0) to (47.6 10 10) + orthogonal box = (0 0 0) to (47.6 47.6 10) + orthogonal box = (0 0 0) to (47.6 47.6 47.6) +region world block INF INF INF INF INF INF + +### interactions +pair_style hybrid/overlay pace table linear 10000 +ACE version: 2023.11.25 +Recursive evaluator is used +pair_coeff * * pace potential_files/c_ace.yace C +Loading potential_files/c_ace.yace +Total number of basis functions + C: 20 (r=1) 455 (r>1) +Mapping LAMMPS atom type #1(C) -> ACE species type #0 +pair_coeff * * table potential_files/d2.table D2 9.0 +Reading pair table potential file potential_files/d2.table with DATE: 2021-12-16 +WARNING: 8063 of 20000 force values in table D2 are inconsistent with -dE/dr. +WARNING: Should only be flagged at inflection points (src/pair_table.cpp:466) +WARNING: 2386 of 20000 distance values in table 1e-06 with relative error +WARNING: over D2 to re-computed values (src/pair_table.cpp:474) + +mass 1 12.011000 +velocity all create 200 1234 + +compute c1 all pair pace +compute c2 all pair table + +# calculate the e/atom for each pair style individually +variable dUpace equal c_c1/atoms +variable dUd2 equal c_c2/atoms + +### run +timestep 0.001 +fix 1 all nvt temp 200.0 200.0 0.01 +thermo_style custom step temp pe press etotal v_dUpace v_dUd2 +thermo 10 +run 100 +Neighbor list info ... + update: every = 1 steps, delay = 0 steps, check = yes + max neighbors/atom: 2000, page size: 100000 + master list distance cutoff = 11 + ghost atom cutoff = 11 + binsize = 5.5, bins = 9 9 9 + 2 neighbor lists, perpetual/occasional/extra = 2 0 0 + (1) pair pace, perpetual + attributes: full, newton on, cut 7.5 + pair build: full/bin/atomonly + stencil: full/bin/3d + bin: standard + (2) pair table, perpetual + attributes: half, newton on + pair build: half/bin/atomonly/newton + stencil: half/bin/3d + bin: standard +Per MPI rank memory allocation (min/avg/max) = 3.655 | 3.655 | 3.655 Mbytes + Step Temp PotEng Press TotEng v_dUpace v_dUd2 + 0 200 -262.26589 -9971.6713 -236.43971 -0.2577066 -0.0045592958 + 10 198.00622 -261.95011 -9934.5046 -236.38139 -0.25738304 -0.0045670733 + 20 199.81545 -261.06219 -9818.4051 -235.25985 -0.25647183 -0.0045903655 + 30 200.85902 -259.76256 -9639.9086 -233.82546 -0.25513263 -0.0046299265 + 40 195.00229 -258.4153 -9425.3772 -233.23448 -0.25372979 -0.0046855071 + 50 198.00573 -257.57066 -9164.7658 -232.00201 -0.25281159 -0.0047590772 + 60 206.26759 -258.09159 -8877.0162 -231.45607 -0.25323684 -0.0048547477 + 70 219.81939 -261.10607 -8668.5789 -232.7206 -0.25612771 -0.0049783595 + 80 250.27428 -268.27862 -8601.1343 -235.96048 -0.2631332 -0.0051454143 + 90 308.88167 -283.24793 -8745.8792 -243.36177 -0.27785093 -0.0053969977 + 100 427.60692 -315.05776 -9147.2389 -259.8405 -0.30933434 -0.0057234269 +Loop time of 0.69628 on 4 procs for 100 steps with 1000 atoms + +Performance: 12.409 ns/day, 1.934 hours/ns, 143.620 timesteps/s, 143.620 katom-step/s +99.5% CPU use with 4 MPI tasks x 1 OpenMP threads + +MPI task timing breakdown: +Section | min time | avg time | max time |%varavg| %total +--------------------------------------------------------------- +Pair | 0.67839 | 0.68307 | 0.69054 | 0.6 | 98.10 +Neigh | 0.00034181 | 0.00034811 | 0.00036188 | 0.0 | 0.05 +Comm | 0.0045334 | 0.012031 | 0.016704 | 4.4 | 1.73 +Output | 0.00015123 | 0.00017175 | 0.0002318 | 0.0 | 0.02 +Modify | 0.00041346 | 0.00043062 | 0.00044327 | 0.0 | 0.06 +Other | | 0.0002301 | | | 0.03 + +Nlocal: 250 ave 261 max 246 min +Histogram: 3 0 0 0 0 0 0 0 0 1 +Nghost: 1250 ave 1254 max 1239 min +Histogram: 1 0 0 0 0 0 0 0 0 3 +Neighs: 6501 ave 6778 max 6320 min +Histogram: 1 0 2 0 0 0 0 0 0 1 +FullNghs: 4421.5 ave 4595 max 4332 min +Histogram: 1 2 0 0 0 0 0 0 0 1 + +Total # of neighbors = 26004 +Ave neighs/atom = 26.004 +Neighbor list builds = 1 +Dangerous builds = 0 + +Total wall time: 0:00:00 diff --git a/examples/PACKAGES/dispersion/log.20Dec2024.d3.g++.1 b/examples/PACKAGES/dispersion/log.20Dec2024.d3.g++.1 new file mode 100644 index 0000000000..6700c1f616 --- /dev/null +++ b/examples/PACKAGES/dispersion/log.20Dec2024.d3.g++.1 @@ -0,0 +1,117 @@ +LAMMPS (19 Nov 2024 - Development - patch_19Nov2024-125-g095d33dafb) +OMP_NUM_THREADS environment is not set. Defaulting to 1 thread. (src/comm.cpp:99) + using 1 OpenMP thread(s) per MPI task +atom_style atomic +units metal +boundary p p p +atom_modify sort 0 0.0 + +lattice sc 1.0 +Lattice spacing in x,y,z = 1 1 1 +region box block 0 10 0 10 0 10 +create_box 1 box +Created orthogonal box = (0 0 0) to (10 10 10) + 1 by 1 by 1 MPI processor grid +create_atoms 1 region box +Created 1000 atoms + using lattice units in orthogonal box = (0 0 0) to (10 10 10) + create_atoms CPU = 0.000 seconds + +variable l equal 47.6 +change_box all x final 0 $l y final 0 $l z final 0 $l remap +change_box all x final 0 47.6 y final 0 $l z final 0 $l remap +change_box all x final 0 47.6 y final 0 47.6 z final 0 $l remap +change_box all x final 0 47.6 y final 0 47.6 z final 0 47.6 remap +Changing box ... + orthogonal box = (0 0 0) to (47.6 10 10) + orthogonal box = (0 0 0) to (47.6 47.6 10) + orthogonal box = (0 0 0) to (47.6 47.6 47.6) +region world block INF INF INF INF INF INF + +### interactions +pair_style hybrid/overlay pace dispersion/d3 bj pbe 16.0 16.0 +ACE version: 2023.11.25 +Recursive evaluator is used +pair_coeff * * pace potential_files/c_ace.yace C +Loading potential_files/c_ace.yace +Total number of basis functions + C: 20 (r=1) 455 (r>1) +Mapping LAMMPS atom type #1(C) -> ACE species type #0 +pair_coeff * * dispersion/d3 C + +mass 1 12.011000 +velocity all create 200 1234 + +compute c1 all pair pace +compute c2 all pair dispersion/d3 + +# calculate the e/atom for each pair style individually +variable Upace equal c_c1/atoms +variable Ud3 equal c_c2/atoms + +### run +timestep 0.001 +fix 1 all nvt temp 200.0 200.0 0.01 +thermo_style custom step temp pe press etotal v_Upace v_Ud3 +thermo 10 +run 100 +Neighbor list info ... + update: every = 1 steps, delay = 0 steps, check = yes + max neighbors/atom: 2000, page size: 100000 + master list distance cutoff = 18 + ghost atom cutoff = 18 + binsize = 9, bins = 6 6 6 + 2 neighbor lists, perpetual/occasional/extra = 2 0 0 + (1) pair pace, perpetual + attributes: full, newton on, cut 7.5 + pair build: full/bin/atomonly + stencil: full/bin/3d + bin: standard + (2) pair dispersion/d3, perpetual + attributes: half, newton on + pair build: half/bin/atomonly/newton + stencil: half/bin/3d + bin: standard +Per MPI rank memory allocation (min/avg/max) = 4.225 | 4.225 | 4.225 Mbytes + Step Temp PotEng Press TotEng v_Upace v_Ud3 + 0 200 -269.22784 -10163.81 -243.40166 -0.2577066 -0.011521241 + 10 198.05578 -268.91992 -10128.61 -243.34481 -0.25738487 -0.01153505 + 20 199.85092 -268.05146 -10018.116 -242.24454 -0.25647561 -0.011575851 + 30 201.10902 -266.77119 -9847.2946 -240.80181 -0.2551274 -0.011643795 + 40 195.0686 -265.42225 -9641.6992 -240.23287 -0.25368339 -0.011738855 + 50 197.63706 -264.51951 -9390.1455 -238.99847 -0.25265765 -0.011861864 + 60 205.01072 -264.86268 -9107.4427 -238.38947 -0.25284579 -0.012016888 + 70 217.51797 -267.50863 -8890.9916 -239.42034 -0.25529813 -0.012210496 + 80 244.30754 -273.91051 -8806.154 -242.36286 -0.26145652 -0.01245399 + 90 296.72041 -287.2518 -8913.8963 -248.93603 -0.27448382 -0.012767981 + 100 404.07337 -315.6103 -9266.1292 -263.43195 -0.3024416 -0.013168694 +Loop time of 4.52709 on 1 procs for 100 steps with 1000 atoms + +Performance: 1.909 ns/day, 12.575 hours/ns, 22.089 timesteps/s, 22.089 katom-step/s +99.6% CPU use with 1 MPI tasks x 1 OpenMP threads + +MPI task timing breakdown: +Section | min time | avg time | max time |%varavg| %total +--------------------------------------------------------------- +Pair | 4.5223 | 4.5223 | 4.5223 | 0.0 | 99.89 +Neigh | 0.0023631 | 0.0023631 | 0.0023631 | 0.0 | 0.05 +Comm | 0.00088624 | 0.00088624 | 0.00088624 | 0.0 | 0.02 +Output | 0.00027759 | 0.00027759 | 0.00027759 | 0.0 | 0.01 +Modify | 0.0010211 | 0.0010211 | 0.0010211 | 0.0 | 0.02 +Other | | 0.0002737 | | | 0.01 + +Nlocal: 1000 ave 1000 max 1000 min +Histogram: 1 0 0 0 0 0 0 0 0 0 +Nghost: 3913 ave 3913 max 3913 min +Histogram: 1 0 0 0 0 0 0 0 0 0 +Neighs: 116409 ave 116409 max 116409 min +Histogram: 1 0 0 0 0 0 0 0 0 0 +FullNghs: 17748 ave 17748 max 17748 min +Histogram: 1 0 0 0 0 0 0 0 0 0 + +Total # of neighbors = 116409 +Ave neighs/atom = 116.409 +Neighbor list builds = 1 +Dangerous builds = 0 + +Total wall time: 0:00:04 diff --git a/examples/PACKAGES/dispersion/log.20Dec2024.d3.g++.4 b/examples/PACKAGES/dispersion/log.20Dec2024.d3.g++.4 new file mode 100644 index 0000000000..c5a38f1d6a --- /dev/null +++ b/examples/PACKAGES/dispersion/log.20Dec2024.d3.g++.4 @@ -0,0 +1,117 @@ +LAMMPS (19 Nov 2024 - Development - patch_19Nov2024-125-g095d33dafb) +OMP_NUM_THREADS environment is not set. Defaulting to 1 thread. (src/comm.cpp:99) + using 1 OpenMP thread(s) per MPI task +atom_style atomic +units metal +boundary p p p +atom_modify sort 0 0.0 + +lattice sc 1.0 +Lattice spacing in x,y,z = 1 1 1 +region box block 0 10 0 10 0 10 +create_box 1 box +Created orthogonal box = (0 0 0) to (10 10 10) + 1 by 2 by 2 MPI processor grid +create_atoms 1 region box +Created 1000 atoms + using lattice units in orthogonal box = (0 0 0) to (10 10 10) + create_atoms CPU = 0.000 seconds + +variable l equal 47.6 +change_box all x final 0 $l y final 0 $l z final 0 $l remap +change_box all x final 0 47.6 y final 0 $l z final 0 $l remap +change_box all x final 0 47.6 y final 0 47.6 z final 0 $l remap +change_box all x final 0 47.6 y final 0 47.6 z final 0 47.6 remap +Changing box ... + orthogonal box = (0 0 0) to (47.6 10 10) + orthogonal box = (0 0 0) to (47.6 47.6 10) + orthogonal box = (0 0 0) to (47.6 47.6 47.6) +region world block INF INF INF INF INF INF + +### interactions +pair_style hybrid/overlay pace dispersion/d3 bj pbe 16.0 16.0 +ACE version: 2023.11.25 +Recursive evaluator is used +pair_coeff * * pace potential_files/c_ace.yace C +Loading potential_files/c_ace.yace +Total number of basis functions + C: 20 (r=1) 455 (r>1) +Mapping LAMMPS atom type #1(C) -> ACE species type #0 +pair_coeff * * dispersion/d3 C + +mass 1 12.011000 +velocity all create 200 1234 + +compute c1 all pair pace +compute c2 all pair dispersion/d3 + +# calculate the e/atom for each pair style individually +variable Upace equal c_c1/atoms +variable Ud3 equal c_c2/atoms + +### run +timestep 0.001 +fix 1 all nvt temp 200.0 200.0 0.01 +thermo_style custom step temp pe press etotal v_Upace v_Ud3 +thermo 10 +run 100 +Neighbor list info ... + update: every = 1 steps, delay = 0 steps, check = yes + max neighbors/atom: 2000, page size: 100000 + master list distance cutoff = 18 + ghost atom cutoff = 18 + binsize = 9, bins = 6 6 6 + 2 neighbor lists, perpetual/occasional/extra = 2 0 0 + (1) pair pace, perpetual + attributes: full, newton on, cut 7.5 + pair build: full/bin/atomonly + stencil: full/bin/3d + bin: standard + (2) pair dispersion/d3, perpetual + attributes: half, newton on + pair build: half/bin/atomonly/newton + stencil: half/bin/3d + bin: standard +Per MPI rank memory allocation (min/avg/max) = 3.732 | 3.732 | 3.732 Mbytes + Step Temp PotEng Press TotEng v_Upace v_Ud3 + 0 200 -269.22784 -10163.81 -243.40166 -0.2577066 -0.011521241 + 10 198.04813 -268.91867 -10126.59 -243.34454 -0.25738301 -0.011535654 + 20 199.86491 -268.04994 -10010.421 -242.24121 -0.25647167 -0.011578276 + 30 201.18317 -266.78129 -9831.2837 -240.80233 -0.25513213 -0.011649162 + 40 195.35281 -265.47802 -9616.0833 -240.25194 -0.2537296 -0.011748422 + 50 198.56247 -264.69401 -9354.3017 -239.05347 -0.25281709 -0.011876925 + 60 207.17238 -265.30194 -9065.1196 -238.54959 -0.25326251 -0.012039431 + 70 221.05245 -268.44583 -8856.3622 -239.90114 -0.25620278 -0.012243053 + 80 252.00942 -275.82142 -8789.4126 -243.27922 -0.26332044 -0.012500977 + 90 311.21153 -291.09334 -8935.4036 -250.90632 -0.27825852 -0.012834817 + 100 431.24438 -323.45003 -9344.1963 -267.76306 -0.31019084 -0.013259185 +Loop time of 1.20684 on 4 procs for 100 steps with 1000 atoms + +Performance: 7.159 ns/day, 3.352 hours/ns, 82.861 timesteps/s, 82.861 katom-step/s +99.4% CPU use with 4 MPI tasks x 1 OpenMP threads + +MPI task timing breakdown: +Section | min time | avg time | max time |%varavg| %total +--------------------------------------------------------------- +Pair | 1.2007 | 1.2019 | 1.2029 | 0.1 | 99.60 +Neigh | 0.00060541 | 0.00062493 | 0.00064411 | 0.0 | 0.05 +Comm | 0.0024344 | 0.0033552 | 0.0045996 | 1.4 | 0.28 +Output | 0.00016956 | 0.00017999 | 0.00021054 | 0.0 | 0.01 +Modify | 0.00046946 | 0.00048235 | 0.00049796 | 0.0 | 0.04 +Other | | 0.0002449 | | | 0.02 + +Nlocal: 250 ave 261 max 246 min +Histogram: 3 0 0 0 0 0 0 0 0 1 +Nghost: 2198 ave 2202 max 2187 min +Histogram: 1 0 0 0 0 0 0 0 0 3 +Neighs: 29023.2 ave 29681 max 27646 min +Histogram: 1 0 0 0 0 0 0 1 0 2 +FullNghs: 4421 ave 4595 max 4331 min +Histogram: 1 2 0 0 0 0 0 0 0 1 + +Total # of neighbors = 116093 +Ave neighs/atom = 116.093 +Neighbor list builds = 1 +Dangerous builds = 0 + +Total wall time: 0:00:01 diff --git a/examples/PACKAGES/dispersion/potential_files/c_ace.yace b/examples/PACKAGES/dispersion/potential_files/c_ace.yace new file mode 100644 index 0000000000..51c00eaa7e --- /dev/null +++ b/examples/PACKAGES/dispersion/potential_files/c_ace.yace @@ -0,0 +1,484 @@ +elements: [C] +E0: [0] +deltaSplineBins: 0.001 +embeddings: + 0: {ndensity: 2, FS_parameters: [1, 1, 1, 0.5], npoti: FinnisSinclairShiftedScaled, rho_core_cutoff: 100000, drho_core_cutoff: 250} +bonds: + [0, 0]: {nradmax: 6, lmax: 6, nradbasemax: 20, radbasename: ChebExpCos, radparameters: [5.25], radcoefficients: [[[-0.0041203432560571792, -0.064593121529619893, 0.17582379121718966, 0.6739179112611895, 0.56851987465018261, -0.25019624063914025, -0.48140703633327014, -0.11958789728030561, 0.14211221178069974, 0.25761450203102332, -0.12875117979087, -0.18674000350793851, 0.29989009726500693, -0.025104058134365399, -0.16025666035057917, -0.032974027642877726, 0.17772398924112806, -0.21270014111670607, 0.23859452909386253, -0.12951583047610216], [-0.060025366899507167, 0.15694591364868665, 0.2193836772331868, 0.38169992213099696, 0.26109064286975392, 0.016408289614982537, -0.132732341780402, -0.21137481257567642, -0.069088968633111203, 0.19853953463963173, -0.0050128642822902291, -0.16753927018870979, 0.2315931785609171, 0.1199226726881412, -0.29904431229968176, 0.090636696365066355, 0.33644560840540744, -0.48640128686911982, 0.32762220361321159, -0.11055261565915846], [0.10231447247806104, 0.27862218552545748, 0.54240255458840425, 0.17921421273237026, -0.032947432825776585, -0.11752713808435113, -0.070877448899669404, -0.22156531702035376, -0.057580697844613705, 0.30326365380233294, 0.009728501619093069, -0.26046927263644604, 0.18948368932791229, 0.066338478837012743, -0.27354908017332624, 0.066084278942988295, 0.25634179926987222, -0.42737202266280505, 0.32289829790801183, -0.11418810927821943], [-0.12920639882270368, -0.05478155358326816, 0.45452060258295329, 0.2060206300855931, 0.0053567037157309589, -0.0067182265281378424, -0.015224066864658615, -0.40895812535861026, 0.06668953445591444, 0.56193193641093064, -0.42557605248613928, 0.086600192506724863, 0.24676365511728832, -0.0058249870467681599, -0.21217314766862627, 0.054681250536981307, 0.21375868484759208, -0.3035140278666808, 0.20187595716230461, -0.038586865164910701], [0.397022621397498, -0.062112070621705283, -0.34288284750828596, 0.29893602925732721, 0.21434712151266413, 0.11253651338665337, 0.18946203650891907, 0.066615950664941956, -0.10484481606188209, -0.2001997362804635, -0.036850332352758794, -0.073700912214797312, -0.35610167244740742, 0.14416677369206998, -0.18936254395270613, 0.20340238249692355, -0.099090919360790758, -0.25472952779951546, 0.45805630736247543, -0.32376991073083961], [0.24105671319394023, -0.17638418421119936, 0.07532666930787113, -0.040869940947987009, 0.13142845947162399, 0.32986665306428647, 0.16925200628432155, -0.173862451886416, -0.21390235043815348, 0.01354466375138433, 0.25816203107430513, 0.30312957989822881, 0.10670006480829729, -0.0039516248850588647, 0.0015710791010618691, 0.19219639261941954, 0.098913712353930094, -0.1723525548988081, 0.29505990509039121, -0.36898739767322503], [-0.13865194297266908, -0.19101365808327947, -0.21462005535086229, -0.42072376074666157, -0.054964120967469554, -0.20827701031673324, -0.052100839487758607, -0.19736555044520979, 0.052388840600572299, 0.28118356877320588, 0.10129951233430452, -0.14649753796697371, -0.034452738093982624, 0.10289470295374038, -0.20143086475867239, -0.47180645057648729, -0.056910644327836456, 0.30680510187234467, -0.26163314705381779, 0.23530566059574465]], [[-0.027342691371566325, 0.66822467799597451, 0.16852110685623917, -0.012597845353796036, -0.16845605041322675, 0.098584817243844297, -0.054025259377184054, 0.029167643152356086, -0.20515619927709236, 0.43524073844180799, -0.27260890718389374, -0.09158614028304625, 0.2947483473329211, -0.11044000904420881, -0.2298799757626902, 0.39997308751825811, -0.34494675641191369, 0.18563045903364922, -0.04987030234315154, -0.00077295192397960986], [0.035648057700759406, 0.066872526959904116, 0.1149034413282702, 0.030841622562348247, 0.17350931848446643, 0.039458015406677496, -0.43161786793636503, -0.11825133772053753, 0.68444041999489014, -0.27502945007299517, -0.34271666223964586, 0.32375609670381689, -0.029035853501948535, 0.0082951196308731316, -0.18778809938009772, 0.13965714927686707, 0.012364530359680676, -0.07111217621768337, 0.052128699210167835, -0.016448775135382777], [-0.040145983153081058, 0.11359244166794524, 0.23286975426090101, 0.12560396985107222, -0.18817603622891108, 0.11016986239381468, 0.26533353390337028, -0.0070133698040188283, -0.37955098976300961, -0.1693917054039572, 0.40557573176649986, 0.016055437284206078, -0.21425325748618168, 0.15309430423541273, 0.16984368527727112, -0.30793782178957591, 0.11698291809819342, 0.2426673439210415, -0.35532025049008575, 0.17091691999588543], [-0.12770766978981068, 0.30182015157237263, 0.11488627434326433, 0.09556593093855538, -0.1545665176269046, -0.0057126701018387679, -0.026345361293811317, -0.093642567719133582, -0.11580294855871612, 0.14108730343051673, 0.52961730530098328, -0.41031427378212526, -0.13043355048684716, 0.41717499376782896, -0.066231212410716278, -0.37464683850373393, 0.40921583476999812, -0.19836965816615965, 0.06069475555256585, -0.017502547261504211], [-0.22991135498935303, 0.042851998427167796, -0.041750600522774552, -0.10853487768276213, 0.57783507675971779, -0.20816633544101526, -0.052803466971975153, 0.20904596639140377, 0.42293933612658213, 0.081421856502523857, -0.25858237758049518, -0.25551120256023863, -0.13233155994702467, 0.31358250222181439, 0.082008049690272158, -0.20545903484042755, -0.17108232275211302, 0.45070056021940919, -0.2350367126516798, 0.0053671538752225797], [0.71773410740753252, 0.12238457810989697, -0.091918616058697, 0.053742983320875688, 0.15977201542389213, -0.40207902256030736, -0.095515940849254993, 0.010742190207910599, -0.1546101722873798, -0.2263887824352239, -0.045638033599702338, 0.11447451955618618, 0.039720238361627809, -0.24618642931660714, -0.17180162480927608, -0.017112493825468283, 0.14918716319793832, 0.16484732045906952, -0.30374491160838912, 0.10669390854216979], [0.72184973167330768, -0.017437658646338731, -0.3353422066567881, -0.13760508426837437, -0.00097162567526262668, 0.1635339045626365, -0.27806800356333872, -0.19333881920785334, -0.030974933698004404, 0.0095404092982257407, -0.047271048553268809, -0.034707716486172942, 0.054471062321393995, 0.088031002913413953, 0.033714799726232855, -0.030224012374140321, -0.066718524131929996, -0.044800636931000143, 0.1221249518049973, -0.11983319699566448]], [[0.010751857291005303, 0.18309227386033386, 0.78355272318538627, -0.024207514329207036, -0.32619771732875535, -0.081249213527712721, 0.028437775300322782, -0.25528508106681264, 0.53592032318934402, -0.13733695236048671, -0.18448501211646473, 0.12748214141763548, 0.03391060624540982, -0.068739054163226496, -0.00054301318277701516, -0.079895193684757448, 0.10509398693239369, -0.019681127021796259, -0.018960091667170585, -0.0028051969652502316], [0.044614814887837861, 0.052368682998455041, 0.42703887574067229, 0.43483623399627425, -0.023606045157063386, -0.31556170295804592, -0.018203545776350145, 0.25298700601472285, -0.13641711898596168, -0.020471157589855508, -0.10868678889941849, -0.0028506797879642842, 0.2365179505964789, -0.24936763337726492, 0.15428924465480789, -0.093654487060668104, 0.065770847943912331, 0.056750523480274102, -0.098363724707717815, 0.047112456792732557], [-0.0098838332192876539, -0.15978986079740412, 0.2419308497598498, 0.10709090305845236, 0.32830486047699353, -0.016728549924914154, -0.13018845702349574, -0.16299120583304422, -0.063149938367278377, 0.017418926410264358, -0.0010229962673665876, 0.036727957410122539, 0.3613359588693128, -0.30494002805270415, -0.15175649711607059, 0.40016828909949254, -0.24505732004071917, 0.027726177490953911, -0.0053672372085453298, 0.030904558169923321], [-0.61114127068032797, -0.025885202260506925, 0.12318332305159208, 0.20855611822260611, 0.060240124563291114, -0.10010501288870575, -0.27918743247660294, -0.097442121882156493, 0.22831543493816001, 0.30533379856061238, 0.11748903170092527, 0.030877957096331961, 0.045402791690632542, 0.24666606672316477, 0.19875607733073569, -0.24598677760434706, -0.14039426995139737, 0.56622133202738745, -0.53948222096095522, 0.11738456218836794], [0.22696321285045887, 0.10894271114892085, -0.11690451602908762, -0.53370932978114238, 0.12530721409875475, 0.055107070783691481, 0.27430265940961668, 0.040789165102450224, -0.34101257799817808, -0.31873814056806804, 0.20706516013334056, 0.36188504831133134, -0.32867295910114575, -0.23576437701241792, 0.22983296502417136, 0.11711404999271384, -0.28041065225389211, 0.19366564914310117, -0.06810790478280429, -0.020511448705468133], [0.01896296556437695, 0.16564714248833912, 0.74178419431357856, 0.43653918497985733, 0.2824154070984049, 0.1112487896890651, -0.32740028073885791, -0.31364926751719235, 0.039426946570891255, 0.19143370755577904, -0.049419147116606006, -0.287911058060489, -0.18936134445355823, 0.081627897816940215, 0.081465940297858336, -0.047062252356557033, -0.040251353843627907, 0.025991647626831264, 0.1617052710725311, -0.060380621424986479], [0.17216576980282916, -0.14008365541190262, 0.46164493945334317, -0.13625205953227082, -0.14224655853738705, -0.63135447884562823, 0.010689236515714469, 0.10031017112600532, 0.055662103443759867, -0.10900587980772192, -0.16291706898575081, -0.051399535544801395, -0.037678737112558452, -0.20502204245965328, -0.21500968546610558, 0.059583168488622475, 0.17701820581656244, -0.01453747957150946, 0.064170783861852695, -0.032200167744015809]], [[0.28535964953549303, 0.051532916188363989, -0.11858381074410625, 0.11206648778121577, 0.46188566233693168, 0.31715828316212319, -0.44730297975241801, -0.33045136450333729, 0.26219731971876414, -0.065219549965221515, -0.26206630095188388, 0.16026199079625442, 0.025577348444611226, 0.02487259842158562, -0.1805900530673917, -0.0032795869274564503, -0.018119130841192599, 0.3394542085752762, -0.38274734771246022, 0.13787473800630892], [0.024727918904336635, 0.11765208222941886, -0.063213250047801955, 0.09271498107610221, 0.089964256669600812, 0.1865698879009613, 0.04611195551313798, -0.26758300639979349, -0.060505074541244541, 0.27862501452505029, -0.26258939523172364, 0.0019440859016375015, 0.41069930348044514, -0.52244119118397159, 0.16870741144901316, 0.30200559751388073, -0.36340270847186945, 0.070038635382275935, 0.12807015031118318, -0.071154640472025521], [-0.19755870124339422, -0.27764903951637437, -0.50463960013847087, 0.087820740148376791, 0.37825951524384593, 0.19347385000064157, -0.25755496068428879, -0.12739982562007388, 0.086532957360981405, -0.064611486035669546, 0.025913559796252244, 0.1408247804533293, 0.20667638673938304, 0.00063861620926296053, -0.28383543399798744, -0.026283351898669271, 0.50831269592352157, -0.47206585706004228, 0.11463783429819524, 0.066867053550044769], [0.48725681824496847, 0.2112347631212344, 0.47635685863343391, 0.47144171085285147, 0.097667010621826103, -0.35789914363566866, -0.3950525620627291, -0.22129375744474825, -0.045746657393347642, 0.17401969871595752, 0.26234552707673597, -0.039193516545187268, -0.31953917309553814, -0.24935099209113207, 0.1362141359691772, 0.0050830631916513316, -0.35671358343017695, 0.21561087462781159, 0.16560710077105703, -0.015553321062687369], [0.05328986262240145, -0.18714182954090719, -0.097924821101537107, 0.3761387578515859, -0.10759805429529919, 0.35804095036989908, -0.23172575344307517, -0.038454139928872311, -0.0025290416530150688, 0.018183670967380192, 0.070889697208256011, 0.17562999825954945, 0.087340495273140084, -0.42173673944503171, -0.035888948939818095, -0.29305191608375153, -0.020800608692351914, 0.38553922248994971, -0.35514472200226632, -0.0011164823005435948], [-0.19327764204534256, 0.041210468743728079, 0.029696340020602074, 0.48792985013478013, 0.21844933472686778, 0.11290482668948744, 0.37082175156441188, 0.29126951966358239, -0.052730557148344515, -0.37432125803128907, -0.34235401107395269, -0.10385353882579715, -0.068653771875918146, -0.19784758053268689, -0.011819368270211851, 0.33809226230059586, 0.19273842455743878, -0.28553769328374395, 0.047189380065272908, 0.2604071768593707], [0.0013149901442139592, 0.1917239071187157, -0.11148226169269138, 0.37273977234167921, -0.21581272515258507, -0.28388024062352374, -0.060387880188506778, -0.076058781288581123, -0.037354384003280769, 0.16346351318364724, 0.21874819275151497, -0.13300818800097308, -0.42815626035042986, -0.09905429927229796, 0.36462239357615905, 0.11007432983619322, -0.26911212023560982, 0.24993568684843107, 0.19335814446224736, -0.048156010337462661]], [[0.37847005361080099, 0.06839305050223829, 0.4386085060470446, 0.17252652960847037, 0.014153611632118526, -0.07894861979975136, 0.1412318227318361, -0.16963909490252194, -0.30689877220398831, 0.012135875361638609, 0.17261756092976158, -0.26088859645050988, 0.051167900002925051, 0.069895511321974968, -0.080838283804776562, -0.34204152524605336, 0.29511889332546787, 0.23295214674406961, -0.37847122789841992, 0.17169822828060455], [-0.028757531045181046, 0.13831397669606307, 0.30589922110131529, 0.079384535430878589, -0.34429737463451082, -0.47875870535813358, -0.16468769156373991, 0.21290639431746128, 0.32928678238553072, 0.23576351957622321, 0.17202814379402692, -0.05678985733779824, -0.29030246194900944, -0.19376054826166564, -0.010202531206971635, 0.060073291215612057, -0.20564776412669403, -0.13821543301381589, 0.46462433286661053, -0.24955403728115189], [0.60542205498248181, -0.017333738720799165, -0.14273658085837035, 0.35497442293511883, 0.20341443788286911, -0.043386457774876583, -0.40529805784264489, -0.10936953406622445, 0.092067798014603547, -0.055951094580754861, 0.051479588381066962, -0.23601096896035584, -0.37866005949248377, 0.1654925720898095, 0.26276730442507457, -0.19257543011456085, -0.24129111702061995, -0.04893294484125215, 0.3600945693741423, -0.2695157721932116], [-0.16352027033536254, 0.090782696881491043, 0.28741897890921442, 0.38930361531007063, 0.0979140138725694, 0.078604185756120812, 0.013181588712960022, 0.22364995025074935, 0.16865002199979123, -0.18241200627710641, -0.15218435740191288, -0.10202376148473855, -0.056988733014642433, 0.11251290167896835, 0.29597327080415714, 0.13138264454460871, -0.159523325203134, -0.14102723200293624, 0.45955486823973135, -0.0096329408938090142], [0.01601375765186977, -0.18308308220973019, -0.3390685722012311, -0.45640993040392464, 0.41069258369269457, 0.15360281403207948, -0.26855931771335045, -0.23481358560746049, 0.087113178447413633, 0.021564809919762253, -0.1051031116190725, 0.24303114069203147, 0.35207755887706238, -0.35027963777159965, -0.11282628402553779, 0.1587770155394583, 0.08440881368878389, -0.22616166301583573, 0.07459894076743831, 0.0088363671806056636], [-0.28154002873230066, -0.03859007216264608, -0.19334059808434861, -0.015310119308723395, 0.69476291814509894, -0.2903268110690565, -0.22979612193157412, 0.19483720665916318, 0.28438109094863084, 0.17704744243588139, 0.15809530412060244, 0.2049141212234242, 0.16816851798493632, 0.00081614244238408743, 0.097094464155708923, 0.10391433015654369, -0.016386551031567647, 0.12609887198518646, 0.10133773352167508, -0.24423345151149398], [-0.10256964256190854, 0.088020258743384344, 0.26754100181949886, 0.16120077792968004, 0.22089097301483207, 0.13968928017339921, -0.41390600484947238, -0.18506018239974115, 0.18658124075290777, 0.25030701470481803, 0.014621356788048146, -0.056281324276015419, 0.19537123299645998, 0.35502806544042509, 0.23370754407617397, 0.25557201470427743, 0.46130368681471973, 0.19540750692863376, -0.12341223735299485, 0.11318418458484716]], [[0.24328414373038743, 0.093552075070560375, 0.11345703553146815, 0.35710238410383516, -0.17352143143430057, -0.42908864793245677, -0.15985396462427631, 0.49995105044216631, -0.050938296493941319, -0.30925616949171963, -0.019133488848668081, 0.1888534807322462, -0.11169535323359014, -0.016692729726119106, -0.16407194068114533, 0.054281983160959668, -0.0039409709415297338, 0.1719411245380697, -0.20293303852942207, 0.096599759170111932], [0.03785449398127403, 0.10707161161932274, -0.13339231193516821, -0.19993611770015035, -0.23405217564792125, -0.096283204360021538, -0.047597238062803922, -0.057132616149235788, 0.31674158231397398, 0.50471103780110749, -0.14188336015016081, -0.4569815569832677, 0.062078456260834441, 0.075364415846527213, -0.23319523473091974, -0.25980919866718194, 0.12101924849821953, 0.30958371342611785, -0.24628142206682638, 0.065435001951064872], [0.80347397035580048, -0.13355287960227824, -0.31821574235192807, -0.43432245182050722, 0.0049358484172777519, 0.34416685559059684, 0.34151667708622291, -0.087712290660035094, -0.29174369671518285, -0.18354397482857515, -0.24899083599544247, -0.18793832367829952, -0.00021439893013297788, -0.087339632867667674, -0.10953083328880284, 0.048361244727319397, 0.0030580659649224219, -0.16575070745906806, -0.038506422775348562, -0.021946756476708015], [-0.23455821565336088, -0.18243125963174783, -0.37317756774637667, 0.0063702147625734097, 0.58570439837100241, 0.40528822223323929, -0.34894999868928372, -0.3457878457215815, -0.027049624004249422, 0.10634982307525212, 0.447414278983988, 0.13233206964573074, -0.23461981820349181, -0.11947728364941379, -0.014667385349338799, 0.077415187084566739, -0.096873128658537075, -0.21671371317608026, 0.33344988957071142, -0.23739693566835179], [-0.28395703660137928, -0.14597062667991867, -0.46518298130147162, -0.11104560999166725, -0.061558744313345765, 0.063650148680449464, 0.10824120353875956, -0.26392348911989261, -0.29559536938403069, 0.015757577344678492, 0.22987482579196575, 0.042716155888389951, -0.0024997458715226998, 0.41575259823534017, 0.041569871657073813, 0.18217803990864692, 0.15058980803361882, -0.03564215992278405, -0.2537206947187246, 0.34573777560209051], [0.3466933200400345, -0.094909714155713948, 0.058718461132017764, -0.28046471842127491, 0.046425651326790968, 0.22119155583519901, 0.17351874321984193, 0.42676936615370586, 0.39634935718289233, 0.11857185522062549, -0.17515761057836193, -0.2639251111667888, -0.11905023262164002, 0.099763782689862662, 0.16374349358492005, 0.17949917900749129, 0.21594712992525106, 0.20201466881554386, -0.065765575903523638, -0.037972028882246503], [-0.26696349219517718, -0.054089152586710876, -0.069223238606167081, -0.28345648935434942, -0.040129714451070934, -0.034206516787206306, -0.34836067695496281, -0.019574413456201051, 0.28469112963064963, 0.24702676981359573, -0.059928304289797431, -0.29570522004240907, -0.31178116700480429, -0.23054236606738493, -0.12236942396982688, -0.0038755856983598552, -0.096214881879151212, -0.27634050649586805, 0.42363648572539636, 0.019690132721636701]]], prehc: 10000, lambdahc: 1, rcut: 5.5, dcut: 0.01, rcut_in: 0.69999999999999996, dcut_in: 0.10000000000000001, inner_cutoff_type: distance} +functions: + 0: + - {mu0: 0, rank: 1, ndensity: 2, num_ms_combs: 1, mus: [0], ns: [1], ls: [0], ms_combs: [0], ctildes: [0.54800501635844368, -1.8886106271030743]} + - {mu0: 0, rank: 1, ndensity: 2, num_ms_combs: 1, mus: [0], ns: [2], ls: [0], ms_combs: [0], ctildes: [-1.0224646249161511, 0.1279969849674466]} + - {mu0: 0, rank: 1, ndensity: 2, num_ms_combs: 1, mus: [0], ns: [3], ls: [0], ms_combs: [0], ctildes: [-2.7911548759436857, -0.10619111231117431]} + - {mu0: 0, rank: 1, ndensity: 2, num_ms_combs: 1, mus: [0], ns: [4], ls: [0], ms_combs: [0], ctildes: [-2.8773895420000111, -1.2518911085047666]} + - {mu0: 0, rank: 1, ndensity: 2, num_ms_combs: 1, mus: [0], ns: [5], ls: [0], ms_combs: [0], ctildes: [-0.70589042299263993, -2.2035428825460759]} + - {mu0: 0, rank: 1, ndensity: 2, num_ms_combs: 1, mus: [0], ns: [6], ls: [0], ms_combs: [0], ctildes: [1.4703224870099074, -1.3978471136451494]} + - {mu0: 0, rank: 1, ndensity: 2, num_ms_combs: 1, mus: [0], ns: [7], ls: [0], ms_combs: [0], ctildes: [1.4193488759678983, 0.70095222107944843]} + - {mu0: 0, rank: 1, ndensity: 2, num_ms_combs: 1, mus: [0], ns: [8], ls: [0], ms_combs: [0], ctildes: [-0.095434202852000605, 1.799637828623404]} + - {mu0: 0, rank: 1, ndensity: 2, num_ms_combs: 1, mus: [0], ns: [9], ls: [0], ms_combs: [0], ctildes: [-0.82606804805744738, 0.86062172275099247]} + - {mu0: 0, rank: 1, ndensity: 2, num_ms_combs: 1, mus: [0], ns: [10], ls: [0], ms_combs: [0], ctildes: [-0.37879803452606575, -0.22677355656374837]} + - {mu0: 0, rank: 1, ndensity: 2, num_ms_combs: 1, mus: [0], ns: [11], ls: [0], ms_combs: [0], ctildes: [0.37800556162802346, 0.52005079731459758]} + - {mu0: 0, rank: 1, ndensity: 2, num_ms_combs: 1, mus: [0], ns: [12], ls: [0], ms_combs: [0], ctildes: [0.27572963887092311, -0.033961331855953791]} + - {mu0: 0, rank: 1, ndensity: 2, num_ms_combs: 1, mus: [0], ns: [13], ls: [0], ms_combs: [0], ctildes: [-0.84157309420131954, 0.039696761327905311]} + - {mu0: 0, rank: 1, ndensity: 2, num_ms_combs: 1, mus: [0], ns: [14], ls: [0], ms_combs: [0], ctildes: [0.0087035526510658434, -0.52875515932102091]} + - {mu0: 0, rank: 1, ndensity: 2, num_ms_combs: 1, mus: [0], ns: [15], ls: [0], ms_combs: [0], ctildes: [0.38889318898162623, -0.37185202413776558]} + - {mu0: 0, rank: 1, ndensity: 2, num_ms_combs: 1, mus: [0], ns: [16], ls: [0], ms_combs: [0], ctildes: [0.39997760518174957, 0.27522993410644692]} + - {mu0: 0, rank: 1, ndensity: 2, num_ms_combs: 1, mus: [0], ns: [17], ls: [0], ms_combs: [0], ctildes: [-0.40560716175517142, 0.88999948688450869]} + - {mu0: 0, rank: 1, ndensity: 2, num_ms_combs: 1, mus: [0], ns: [18], ls: [0], ms_combs: [0], ctildes: [-0.5674695869403743, 0.52134854113374618]} + - {mu0: 0, rank: 1, ndensity: 2, num_ms_combs: 1, mus: [0], ns: [19], ls: [0], ms_combs: [0], ctildes: [0.56901395171601066, -0.43955374061852098]} + - {mu0: 0, rank: 1, ndensity: 2, num_ms_combs: 1, mus: [0], ns: [20], ls: [0], ms_combs: [0], ctildes: [-0.093349950157230063, -0.25906184180015246]} + - {mu0: 0, rank: 2, ndensity: 2, num_ms_combs: 1, mus: [0, 0], ns: [1, 1], ls: [0, 0], ms_combs: [0, 0], ctildes: [1.4219164729833627, 2.1695649338924619]} + - {mu0: 0, rank: 2, ndensity: 2, num_ms_combs: 2, mus: [0, 0], ns: [1, 1], ls: [1, 1], ms_combs: [0, 0, 1, -1], ctildes: [0.00011667193587998646, -0.7333296121574493, -0.00023334387175997286, 1.4666592243148984]} + - {mu0: 0, rank: 2, ndensity: 2, num_ms_combs: 3, mus: [0, 0], ns: [1, 1], ls: [2, 2], ms_combs: [0, 0, 1, -1, 2, -2], ctildes: [0.72978083653504799, -0.14852150045123022, -1.4595616730700962, 0.2970430009024605, 1.4595616730700967, -0.29704300090246055]} + - {mu0: 0, rank: 2, ndensity: 2, num_ms_combs: 4, mus: [0, 0], ns: [1, 1], ls: [3, 3], ms_combs: [0, 0, 1, -1, 2, -2, 3, -3], ctildes: [0.12593233861316111, 0.21034215858193256, -0.25186467722632222, -0.42068431716386512, 0.25186467722632216, 0.42068431716386506, -0.25186467722632216, -0.42068431716386506]} + - {mu0: 0, rank: 2, ndensity: 2, num_ms_combs: 5, mus: [0, 0], ns: [1, 1], ls: [4, 4], ms_combs: [0, 0, 1, -1, 2, -2, 3, -3, 4, -4], ctildes: [0.021526070114280385, -0.08224327877076984, -0.043052140228560749, 0.1644865575415396, 0.043052140228560749, -0.1644865575415396, -0.043052140228560749, 0.1644865575415396, 0.043052140228560756, -0.16448655754153962]} + - {mu0: 0, rank: 2, ndensity: 2, num_ms_combs: 6, mus: [0, 0], ns: [1, 1], ls: [5, 5], ms_combs: [0, 0, 1, -1, 2, -2, 3, -3, 4, -4, 5, -5], ctildes: [0.0068932448284070762, 0.020513741313869509, -0.013786489656814152, -0.041027482627739018, 0.013786489656814154, 0.041027482627739024, -0.013786489656814158, -0.041027482627739031, 0.013786489656814154, 0.041027482627739024, -0.01378648965681414, -0.041027482627738983]} + - {mu0: 0, rank: 2, ndensity: 2, num_ms_combs: 7, mus: [0, 0], ns: [1, 1], ls: [6, 6], ms_combs: [0, 0, 1, -1, 2, -2, 3, -3, 4, -4, 5, -5, 6, -6], ctildes: [0.021682051991893873, -0.083175239945493668, -0.043364103983787768, 0.16635047989098742, 0.043364103983787747, -0.16635047989098734, -0.043364103983787761, 0.16635047989098739, 0.043364103983787761, -0.16635047989098739, -0.043364103983787747, 0.16635047989098734, 0.043364103983787726, -0.16635047989098725]} + - {mu0: 0, rank: 2, ndensity: 2, num_ms_combs: 1, mus: [0, 0], ns: [1, 2], ls: [0, 0], ms_combs: [0, 0], ctildes: [-0.43318744497779094, -0.91213854456090637]} + - {mu0: 0, rank: 2, ndensity: 2, num_ms_combs: 2, mus: [0, 0], ns: [1, 2], ls: [1, 1], ms_combs: [0, 0, 1, -1], ctildes: [1.4537603276285165, -0.068513814824273253, -2.9075206552570325, 0.13702762964854648]} + - {mu0: 0, rank: 2, ndensity: 2, num_ms_combs: 3, mus: [0, 0], ns: [1, 2], ls: [2, 2], ms_combs: [0, 0, 1, -1, 2, -2], ctildes: [0.33089312021029504, 0.46445864722169578, -0.66178624042059009, -0.92891729444339166, 0.66178624042059031, 0.92891729444339188]} + - {mu0: 0, rank: 2, ndensity: 2, num_ms_combs: 4, mus: [0, 0], ns: [1, 2], ls: [3, 3], ms_combs: [0, 0, 1, -1, 2, -2, 3, -3], ctildes: [0.80993194140343017, 0.61680056161134955, -1.6198638828068603, -1.2336011232226991, 1.6198638828068601, 1.2336011232226989, -1.6198638828068601, -1.2336011232226989]} + - {mu0: 0, rank: 2, ndensity: 2, num_ms_combs: 5, mus: [0, 0], ns: [1, 2], ls: [4, 4], ms_combs: [0, 0, 1, -1, 2, -2, 3, -3, 4, -4], ctildes: [0.24362173864699546, 0.14724268882102981, -0.4872434772939907, -0.29448537764205945, 0.4872434772939907, 0.29448537764205945, -0.4872434772939907, -0.29448537764205945, 0.48724347729399076, 0.29448537764205951]} + - {mu0: 0, rank: 2, ndensity: 2, num_ms_combs: 6, mus: [0, 0], ns: [1, 2], ls: [5, 5], ms_combs: [0, 0, 1, -1, 2, -2, 3, -3, 4, -4, 5, -5], ctildes: [-0.0013835743926136814, 0.096200491899325419, 0.0027671487852273628, -0.19240098379865084, -0.0027671487852273633, 0.19240098379865087, 0.0027671487852273637, -0.19240098379865092, -0.0027671487852273633, 0.19240098379865087, 0.0027671487852273602, -0.19240098379865067]} + - {mu0: 0, rank: 2, ndensity: 2, num_ms_combs: 7, mus: [0, 0], ns: [1, 2], ls: [6, 6], ms_combs: [0, 0, 1, -1, 2, -2, 3, -3, 4, -4, 5, -5, 6, -6], ctildes: [-0.053542631159526212, 0.010687223714819706, 0.10708526231905247, -0.021374447429639423, -0.10708526231905242, 0.021374447429639412, 0.10708526231905244, -0.021374447429639416, -0.10708526231905244, 0.021374447429639416, 0.10708526231905242, -0.021374447429639412, -0.10708526231905235, 0.021374447429639398]} + - {mu0: 0, rank: 2, ndensity: 2, num_ms_combs: 1, mus: [0, 0], ns: [1, 3], ls: [0, 0], ms_combs: [0, 0], ctildes: [-1.2700560444266167, -2.3684832899534825]} + - {mu0: 0, rank: 2, ndensity: 2, num_ms_combs: 2, mus: [0, 0], ns: [1, 3], ls: [1, 1], ms_combs: [0, 0, 1, -1], ctildes: [1.4192223145176039, 0.19550034268946143, -2.8384446290352074, -0.3910006853789228]} + - {mu0: 0, rank: 2, ndensity: 2, num_ms_combs: 3, mus: [0, 0], ns: [1, 3], ls: [2, 2], ms_combs: [0, 0, 1, -1, 2, -2], ctildes: [0.15011475702951813, 0.25604389526605376, -0.30022951405903631, -0.51208779053210751, 0.30022951405903636, 0.51208779053210762]} + - {mu0: 0, rank: 2, ndensity: 2, num_ms_combs: 4, mus: [0, 0], ns: [1, 3], ls: [3, 3], ms_combs: [0, 0, 1, -1, 2, -2, 3, -3], ctildes: [-0.096866190381483239, -0.35563852391404271, 0.19373238076296648, 0.71127704782808543, -0.19373238076296645, -0.71127704782808532, 0.19373238076296645, 0.71127704782808532]} + - {mu0: 0, rank: 2, ndensity: 2, num_ms_combs: 5, mus: [0, 0], ns: [1, 3], ls: [4, 4], ms_combs: [0, 0, 1, -1, 2, -2, 3, -3, 4, -4], ctildes: [0.00046216751368274996, 0.76161788090645521, -0.00092433502736549949, -1.5232357618129095, 0.00092433502736549949, 1.5232357618129095, -0.00092433502736549949, -1.5232357618129095, 0.00092433502736549971, 1.5232357618129098]} + - {mu0: 0, rank: 2, ndensity: 2, num_ms_combs: 6, mus: [0, 0], ns: [1, 3], ls: [5, 5], ms_combs: [0, 0, 1, -1, 2, -2, 3, -3, 4, -4, 5, -5], ctildes: [-0.0035468660884478437, 0.055520373437220889, 0.0070937321768956875, -0.11104074687444178, -0.0070937321768956884, 0.11104074687444181, 0.0070937321768956901, -0.11104074687444182, -0.0070937321768956884, 0.11104074687444181, 0.0070937321768956806, -0.11104074687444168]} + - {mu0: 0, rank: 2, ndensity: 2, num_ms_combs: 7, mus: [0, 0], ns: [1, 3], ls: [6, 6], ms_combs: [0, 0, 1, -1, 2, -2, 3, -3, 4, -4, 5, -5, 6, -6], ctildes: [-0.014516376693873135, -0.043349937215293177, 0.029032753387746284, 0.086699874430586382, -0.029032753387746271, -0.086699874430586354, 0.029032753387746277, 0.086699874430586368, -0.029032753387746277, -0.086699874430586368, 0.029032753387746271, 0.086699874430586354, -0.029032753387746253, -0.086699874430586299]} + - {mu0: 0, rank: 2, ndensity: 2, num_ms_combs: 1, mus: [0, 0], ns: [1, 4], ls: [0, 0], ms_combs: [0, 0], ctildes: [-0.38166653284909635, 0.019258940079071113]} + - {mu0: 0, rank: 2, ndensity: 2, num_ms_combs: 2, mus: [0, 0], ns: [1, 4], ls: [1, 1], ms_combs: [0, 0, 1, -1], ctildes: [-0.15568464762014381, 0.16634367956773982, 0.3113692952402875, -0.33268735913547959]} + - {mu0: 0, rank: 2, ndensity: 2, num_ms_combs: 3, mus: [0, 0], ns: [1, 4], ls: [2, 2], ms_combs: [0, 0, 1, -1, 2, -2], ctildes: [-0.076735805723983225, -0.44625798187075244, 0.15347161144796648, 0.89251596374150499, -0.15347161144796651, -0.89251596374150521]} + - {mu0: 0, rank: 2, ndensity: 2, num_ms_combs: 4, mus: [0, 0], ns: [1, 4], ls: [3, 3], ms_combs: [0, 0, 1, -1, 2, -2, 3, -3], ctildes: [0.27082046977291108, -0.99043859045402871, -0.54164093954582215, 1.9808771809080574, 0.54164093954582215, -1.980877180908057, -0.54164093954582215, 1.980877180908057]} + - {mu0: 0, rank: 2, ndensity: 2, num_ms_combs: 5, mus: [0, 0], ns: [1, 4], ls: [4, 4], ms_combs: [0, 0, 1, -1, 2, -2, 3, -3, 4, -4], ctildes: [0.07558666777397742, -0.15140837809203422, -0.15117333554795476, 0.30281675618406828, 0.15117333554795476, -0.30281675618406828, -0.15117333554795476, 0.30281675618406828, 0.15117333554795478, -0.30281675618406834]} + - {mu0: 0, rank: 2, ndensity: 2, num_ms_combs: 6, mus: [0, 0], ns: [1, 4], ls: [5, 5], ms_combs: [0, 0, 1, -1, 2, -2, 3, -3, 4, -4, 5, -5], ctildes: [-0.055356012099554666, -0.023821236927555136, 0.11071202419910933, 0.047642473855110272, -0.11071202419910935, -0.047642473855110279, 0.11071202419910937, 0.047642473855110286, -0.11071202419910935, -0.047642473855110279, 0.11071202419910922, 0.047642473855110223]} + - {mu0: 0, rank: 2, ndensity: 2, num_ms_combs: 7, mus: [0, 0], ns: [1, 4], ls: [6, 6], ms_combs: [0, 0, 1, -1, 2, -2, 3, -3, 4, -4, 5, -5, 6, -6], ctildes: [0.041351398600620329, 0.092076431268007755, -0.082702797201240699, -0.18415286253601559, 0.082702797201240658, 0.18415286253601551, -0.082702797201240685, -0.18415286253601554, 0.082702797201240685, 0.18415286253601554, -0.082702797201240658, -0.18415286253601551, 0.082702797201240616, 0.1841528625360154]} + - {mu0: 0, rank: 2, ndensity: 2, num_ms_combs: 1, mus: [0, 0], ns: [1, 5], ls: [0, 0], ms_combs: [0, 0], ctildes: [-1.6225624330582755, 1.0743712944072534]} + - {mu0: 0, rank: 2, ndensity: 2, num_ms_combs: 2, mus: [0, 0], ns: [1, 5], ls: [1, 1], ms_combs: [0, 0, 1, -1], ctildes: [-0.18746996859988777, -0.36685568131214213, 0.37493993719977542, 0.73371136262428416]} + - {mu0: 0, rank: 2, ndensity: 2, num_ms_combs: 3, mus: [0, 0], ns: [1, 5], ls: [2, 2], ms_combs: [0, 0, 1, -1, 2, -2], ctildes: [-0.36726410935067633, 0.89013247713195309, 0.73452821870135276, -1.7802649542639064, -0.73452821870135288, 1.7802649542639069]} + - {mu0: 0, rank: 2, ndensity: 2, num_ms_combs: 4, mus: [0, 0], ns: [1, 5], ls: [3, 3], ms_combs: [0, 0, 1, -1, 2, -2, 3, -3], ctildes: [0.020797274574061393, -0.77538912376271985, -0.041594549148122786, 1.5507782475254397, 0.041594549148122786, -1.5507782475254395, -0.041594549148122786, 1.5507782475254395]} + - {mu0: 0, rank: 2, ndensity: 2, num_ms_combs: 5, mus: [0, 0], ns: [1, 5], ls: [4, 4], ms_combs: [0, 0, 1, -1, 2, -2, 3, -3, 4, -4], ctildes: [0.16494868035671303, 0.015029211488242506, -0.3298973607134259, -0.030058422976484998, 0.3298973607134259, 0.030058422976484998, -0.3298973607134259, -0.030058422976484998, 0.32989736071342596, 0.030058422976485005]} + - {mu0: 0, rank: 2, ndensity: 2, num_ms_combs: 6, mus: [0, 0], ns: [1, 5], ls: [5, 5], ms_combs: [0, 0, 1, -1, 2, -2, 3, -3, 4, -4, 5, -5], ctildes: [-0.087810838257199972, -0.067258707748941465, 0.17562167651439994, 0.13451741549788293, -0.17562167651439997, -0.13451741549788296, 0.1756216765144, 0.13451741549788299, -0.17562167651439997, -0.13451741549788296, 0.17562167651439978, 0.13451741549788282]} + - {mu0: 0, rank: 2, ndensity: 2, num_ms_combs: 7, mus: [0, 0], ns: [1, 5], ls: [6, 6], ms_combs: [0, 0, 1, -1, 2, -2, 3, -3, 4, -4, 5, -5, 6, -6], ctildes: [0.001724481681354592, -0.14679148503755093, -0.0034489633627091854, 0.29358297007510198, 0.0034489633627091841, -0.29358297007510187, -0.0034489633627091845, 0.29358297007510192, 0.0034489633627091845, -0.29358297007510192, -0.0034489633627091841, 0.29358297007510187, 0.0034489633627091819, -0.2935829700751017]} + - {mu0: 0, rank: 2, ndensity: 2, num_ms_combs: 1, mus: [0, 0], ns: [1, 6], ls: [0, 0], ms_combs: [0, 0], ctildes: [2.3588063720472192, -0.19306660335818052]} + - {mu0: 0, rank: 2, ndensity: 2, num_ms_combs: 2, mus: [0, 0], ns: [1, 6], ls: [1, 1], ms_combs: [0, 0, 1, -1], ctildes: [0.31278883532551527, -0.42950035682304383, -0.62557767065103043, 0.85900071364608743]} + - {mu0: 0, rank: 2, ndensity: 2, num_ms_combs: 3, mus: [0, 0], ns: [1, 6], ls: [2, 2], ms_combs: [0, 0, 1, -1, 2, -2], ctildes: [-0.18236995391529998, -0.35188270238838137, 0.36473990783060001, 0.70376540477676286, -0.36473990783060012, -0.70376540477676297]} + - {mu0: 0, rank: 2, ndensity: 2, num_ms_combs: 4, mus: [0, 0], ns: [1, 6], ls: [3, 3], ms_combs: [0, 0, 1, -1, 2, -2, 3, -3], ctildes: [0.11921529795898887, 0.044745256711350524, -0.23843059591797774, -0.089490513422701048, 0.23843059591797772, 0.089490513422701035, -0.23843059591797772, -0.089490513422701035]} + - {mu0: 0, rank: 2, ndensity: 2, num_ms_combs: 5, mus: [0, 0], ns: [1, 6], ls: [4, 4], ms_combs: [0, 0, 1, -1, 2, -2, 3, -3, 4, -4], ctildes: [0.11469909434938771, -0.045072875025308504, -0.22939818869877532, 0.090145750050616966, 0.22939818869877532, -0.090145750050616966, -0.22939818869877532, 0.090145750050616966, 0.22939818869877537, -0.09014575005061698]} + - {mu0: 0, rank: 2, ndensity: 2, num_ms_combs: 6, mus: [0, 0], ns: [1, 6], ls: [5, 5], ms_combs: [0, 0, 1, -1, 2, -2, 3, -3, 4, -4, 5, -5], ctildes: [0.049934389854070081, -0.0041853292391767095, -0.099868779708140162, 0.0083706584783534189, 0.099868779708140176, -0.0083706584783534207, -0.09986877970814019, 0.0083706584783534224, 0.099868779708140176, -0.0083706584783534207, -0.099868779708140065, 0.0083706584783534103]} + - {mu0: 0, rank: 2, ndensity: 2, num_ms_combs: 7, mus: [0, 0], ns: [1, 6], ls: [6, 6], ms_combs: [0, 0, 1, -1, 2, -2, 3, -3, 4, -4, 5, -5, 6, -6], ctildes: [-0.050495814215476637, 0.0037465711223344049, 0.10099162843095331, -0.0074931422446688134, -0.10099162843095327, 0.0074931422446688099, 0.1009916284309533, -0.0074931422446688116, -0.1009916284309533, 0.0074931422446688116, 0.10099162843095327, -0.0074931422446688099, -0.10099162843095322, 0.0074931422446688056]} + - {mu0: 0, rank: 2, ndensity: 2, num_ms_combs: 1, mus: [0, 0], ns: [2, 2], ls: [0, 0], ms_combs: [0, 0], ctildes: [-0.028116772006554509, -0.28609697481073121]} + - {mu0: 0, rank: 2, ndensity: 2, num_ms_combs: 2, mus: [0, 0], ns: [2, 2], ls: [1, 1], ms_combs: [0, 0, 1, -1], ctildes: [0.18814924135250757, 0.064366309069881958, -0.37629848270501509, -0.12873261813976389]} + - {mu0: 0, rank: 2, ndensity: 2, num_ms_combs: 3, mus: [0, 0], ns: [2, 2], ls: [2, 2], ms_combs: [0, 0, 1, -1, 2, -2], ctildes: [-0.014979314825900375, -0.88671357027343978, 0.029958629651800753, 1.7734271405468798, -0.02995862965180076, -1.7734271405468802]} + - {mu0: 0, rank: 2, ndensity: 2, num_ms_combs: 4, mus: [0, 0], ns: [2, 2], ls: [3, 3], ms_combs: [0, 0, 1, -1, 2, -2, 3, -3], ctildes: [-0.034151912231867906, -0.068800932616767105, 0.068303824463735813, 0.13760186523353421, -0.068303824463735799, -0.13760186523353421, 0.068303824463735799, 0.13760186523353421]} + - {mu0: 0, rank: 2, ndensity: 2, num_ms_combs: 5, mus: [0, 0], ns: [2, 2], ls: [4, 4], ms_combs: [0, 0, 1, -1, 2, -2, 3, -3, 4, -4], ctildes: [-0.08086444663637965, -0.34994463306626644, 0.16172889327275922, 0.69988926613253255, -0.16172889327275922, -0.69988926613253255, 0.16172889327275922, 0.69988926613253255, -0.16172889327275924, -0.69988926613253266]} + - {mu0: 0, rank: 2, ndensity: 2, num_ms_combs: 6, mus: [0, 0], ns: [2, 2], ls: [5, 5], ms_combs: [0, 0, 1, -1, 2, -2, 3, -3, 4, -4, 5, -5], ctildes: [0.033811225792431759, -0.33165902558207228, -0.067622451584863519, 0.66331805116414455, 0.067622451584863533, -0.66331805116414466, -0.067622451584863547, 0.66331805116414488, 0.067622451584863533, -0.66331805116414466, -0.067622451584863463, 0.663318051164144]} + - {mu0: 0, rank: 2, ndensity: 2, num_ms_combs: 7, mus: [0, 0], ns: [2, 2], ls: [6, 6], ms_combs: [0, 0, 1, -1, 2, -2, 3, -3, 4, -4, 5, -5, 6, -6], ctildes: [0.014625441029799895, 0.61936436676776463, -0.029250882059599803, -1.2387287335355297, 0.029250882059599789, 1.2387287335355293, -0.029250882059599796, -1.2387287335355295, 0.029250882059599796, 1.2387287335355295, -0.029250882059599789, -1.2387287335355293, 0.029250882059599772, 1.2387287335355286]} + - {mu0: 0, rank: 2, ndensity: 2, num_ms_combs: 1, mus: [0, 0], ns: [2, 3], ls: [0, 0], ms_combs: [0, 0], ctildes: [-0.093301813810979115, -0.98594040076905276]} + - {mu0: 0, rank: 2, ndensity: 2, num_ms_combs: 2, mus: [0, 0], ns: [2, 3], ls: [1, 1], ms_combs: [0, 0, 1, -1], ctildes: [0.36740747531064405, 0.35410333205913891, -0.73481495062128799, -0.7082066641182776]} + - {mu0: 0, rank: 2, ndensity: 2, num_ms_combs: 3, mus: [0, 0], ns: [2, 3], ls: [2, 2], ms_combs: [0, 0, 1, -1, 2, -2], ctildes: [1.2524772379394373, -0.034567757016887711, -2.5049544758788751, 0.069135514033775422, 2.504954475878876, -0.06913551403377545]} + - {mu0: 0, rank: 2, ndensity: 2, num_ms_combs: 4, mus: [0, 0], ns: [2, 3], ls: [3, 3], ms_combs: [0, 0, 1, -1, 2, -2, 3, -3], ctildes: [0.17489272213459733, -0.7545319318868281, -0.34978544426919467, 1.5090638637736562, 0.34978544426919461, -1.509063863773656, -0.34978544426919461, 1.509063863773656]} + - {mu0: 0, rank: 2, ndensity: 2, num_ms_combs: 5, mus: [0, 0], ns: [2, 3], ls: [4, 4], ms_combs: [0, 0, 1, -1, 2, -2, 3, -3, 4, -4], ctildes: [-0.40986981547290113, 0.41402768641878035, 0.81973963094580182, -0.82805537283756037, -0.81973963094580182, 0.82805537283756037, 0.81973963094580182, -0.82805537283756037, -0.81973963094580204, 0.82805537283756048]} + - {mu0: 0, rank: 2, ndensity: 2, num_ms_combs: 6, mus: [0, 0], ns: [2, 3], ls: [5, 5], ms_combs: [0, 0, 1, -1, 2, -2, 3, -3, 4, -4, 5, -5], ctildes: [-0.0068919001002576965, 0.27189672510765284, 0.013783800200515393, -0.54379345021530567, -0.013783800200515397, 0.54379345021530578, 0.013783800200515398, -0.54379345021530578, -0.013783800200515397, 0.54379345021530578, 0.013783800200515381, -0.54379345021530512]} + - {mu0: 0, rank: 2, ndensity: 2, num_ms_combs: 7, mus: [0, 0], ns: [2, 3], ls: [6, 6], ms_combs: [0, 0, 1, -1, 2, -2, 3, -3, 4, -4, 5, -5, 6, -6], ctildes: [0.051930049422014117, 0.21347044504222906, -0.10386009884402828, -0.42694089008445829, 0.10386009884402823, 0.42694089008445812, -0.10386009884402826, -0.42694089008445824, 0.10386009884402826, 0.42694089008445824, -0.10386009884402823, -0.42694089008445812, 0.10386009884402818, 0.42694089008445785]} + - {mu0: 0, rank: 2, ndensity: 2, num_ms_combs: 1, mus: [0, 0], ns: [2, 4], ls: [0, 0], ms_combs: [0, 0], ctildes: [0.70898101387598889, 0.12531490121331465]} + - {mu0: 0, rank: 2, ndensity: 2, num_ms_combs: 2, mus: [0, 0], ns: [2, 4], ls: [1, 1], ms_combs: [0, 0, 1, -1], ctildes: [0.17649505121359044, 0.0056869321230603462, -0.35299010242718082, -0.011373864246120691]} + - {mu0: 0, rank: 2, ndensity: 2, num_ms_combs: 3, mus: [0, 0], ns: [2, 4], ls: [2, 2], ms_combs: [0, 0, 1, -1, 2, -2], ctildes: [-0.88020227081945857, 0.13685387132626822, 1.7604045416389174, -0.2737077426525365, -1.7604045416389178, 0.27370774265253656]} + - {mu0: 0, rank: 2, ndensity: 2, num_ms_combs: 4, mus: [0, 0], ns: [2, 4], ls: [3, 3], ms_combs: [0, 0, 1, -1, 2, -2, 3, -3], ctildes: [0.39321923110140222, 1.3471725537105592, -0.78643846220280444, -2.6943451074211184, 0.78643846220280433, 2.694345107421118, -0.78643846220280433, -2.694345107421118]} + - {mu0: 0, rank: 2, ndensity: 2, num_ms_combs: 5, mus: [0, 0], ns: [2, 4], ls: [4, 4], ms_combs: [0, 0, 1, -1, 2, -2, 3, -3, 4, -4], ctildes: [-0.070553360902234255, -0.69054440883028601, 0.14110672180446843, 1.3810888176605713, -0.14110672180446843, -1.3810888176605713, 0.14110672180446843, 1.3810888176605713, -0.14110672180446845, -1.3810888176605716]} + - {mu0: 0, rank: 2, ndensity: 2, num_ms_combs: 6, mus: [0, 0], ns: [2, 4], ls: [5, 5], ms_combs: [0, 0, 1, -1, 2, -2, 3, -3, 4, -4, 5, -5], ctildes: [-0.039343589666738954, 0.012758383427574849, 0.078687179333477908, -0.025516766855149697, -0.078687179333477922, 0.025516766855149701, 0.078687179333477936, -0.025516766855149708, -0.078687179333477922, 0.025516766855149701, 0.078687179333477839, -0.025516766855149673]} + - {mu0: 0, rank: 2, ndensity: 2, num_ms_combs: 7, mus: [0, 0], ns: [2, 4], ls: [6, 6], ms_combs: [0, 0, 1, -1, 2, -2, 3, -3, 4, -4, 5, -5, 6, -6], ctildes: [-0.17803071464755035, 0.1368145371185254, 0.35606142929510082, -0.27362907423705091, -0.35606142929510071, 0.2736290742370508, 0.35606142929510076, -0.27362907423705085, -0.35606142929510076, 0.27362907423705085, 0.35606142929510071, -0.2736290742370508, -0.35606142929510048, 0.27362907423705063]} + - {mu0: 0, rank: 2, ndensity: 2, num_ms_combs: 1, mus: [0, 0], ns: [2, 5], ls: [0, 0], ms_combs: [0, 0], ctildes: [1.0070849931771757, 0.50817061641931782]} + - {mu0: 0, rank: 2, ndensity: 2, num_ms_combs: 2, mus: [0, 0], ns: [2, 5], ls: [1, 1], ms_combs: [0, 0, 1, -1], ctildes: [0.19395148381426122, 1.3907700160050773, -0.38790296762852239, -2.7815400320101542]} + - {mu0: 0, rank: 2, ndensity: 2, num_ms_combs: 3, mus: [0, 0], ns: [2, 5], ls: [2, 2], ms_combs: [0, 0, 1, -1, 2, -2], ctildes: [-0.1929776881405755, 0.75643896134510191, 0.38595537628115106, -1.512877922690204, -0.38595537628115117, 1.5128779226902045]} + - {mu0: 0, rank: 2, ndensity: 2, num_ms_combs: 4, mus: [0, 0], ns: [2, 5], ls: [3, 3], ms_combs: [0, 0, 1, -1, 2, -2, 3, -3], ctildes: [-0.56815630710901532, -0.15761609715573718, 1.1363126142180306, 0.31523219431147437, -1.1363126142180304, -0.31523219431147431, 1.1363126142180304, 0.31523219431147431]} + - {mu0: 0, rank: 2, ndensity: 2, num_ms_combs: 5, mus: [0, 0], ns: [2, 5], ls: [4, 4], ms_combs: [0, 0, 1, -1, 2, -2, 3, -3, 4, -4], ctildes: [0.3177397618439079, 0.25886696808442117, -0.63547952368781546, -0.517733936168842, 0.63547952368781546, 0.517733936168842, -0.63547952368781546, -0.517733936168842, 0.63547952368781557, 0.51773393616884211]} + - {mu0: 0, rank: 2, ndensity: 2, num_ms_combs: 6, mus: [0, 0], ns: [2, 5], ls: [5, 5], ms_combs: [0, 0, 1, -1, 2, -2, 3, -3, 4, -4, 5, -5], ctildes: [-0.047422691189342683, 0.059256433246119011, 0.094845382378685367, -0.11851286649223802, -0.094845382378685381, 0.11851286649223804, 0.094845382378685394, -0.11851286649223806, -0.094845382378685381, 0.11851286649223804, 0.09484538237868527, -0.11851286649223791]} + - {mu0: 0, rank: 2, ndensity: 2, num_ms_combs: 7, mus: [0, 0], ns: [2, 5], ls: [6, 6], ms_combs: [0, 0, 1, -1, 2, -2, 3, -3, 4, -4, 5, -5, 6, -6], ctildes: [0.067570484874723891, -0.086865467825053583, -0.13514096974944784, 0.17373093565010725, 0.13514096974944778, -0.17373093565010717, -0.13514096974944781, 0.17373093565010719, 0.13514096974944781, -0.17373093565010719, -0.13514096974944778, 0.17373093565010717, 0.1351409697494477, -0.17373093565010705]} + - {mu0: 0, rank: 2, ndensity: 2, num_ms_combs: 1, mus: [0, 0], ns: [2, 6], ls: [0, 0], ms_combs: [0, 0], ctildes: [-0.075167007983605039, -0.42283123197470784]} + - {mu0: 0, rank: 2, ndensity: 2, num_ms_combs: 2, mus: [0, 0], ns: [2, 6], ls: [1, 1], ms_combs: [0, 0, 1, -1], ctildes: [0.099102521821574988, -0.16920000551249864, -0.19820504364314995, 0.33840001102499723]} + - {mu0: 0, rank: 2, ndensity: 2, num_ms_combs: 3, mus: [0, 0], ns: [2, 6], ls: [2, 2], ms_combs: [0, 0, 1, -1, 2, -2], ctildes: [-0.1596606083756843, -0.54846422652024729, 0.31932121675136865, 1.0969284530404946, -0.31932121675136871, -1.0969284530404948]} + - {mu0: 0, rank: 2, ndensity: 2, num_ms_combs: 4, mus: [0, 0], ns: [2, 6], ls: [3, 3], ms_combs: [0, 0, 1, -1, 2, -2, 3, -3], ctildes: [-0.042488293069970311, 0.53120979855293737, 0.084976586139940621, -1.0624195971058747, -0.084976586139940608, 1.0624195971058745, 0.084976586139940608, -1.0624195971058745]} + - {mu0: 0, rank: 2, ndensity: 2, num_ms_combs: 5, mus: [0, 0], ns: [2, 6], ls: [4, 4], ms_combs: [0, 0, 1, -1, 2, -2, 3, -3, 4, -4], ctildes: [-0.12276490400115744, -0.18735609623268329, 0.24552980800231478, 0.37471219246536641, -0.24552980800231478, -0.37471219246536641, 0.24552980800231478, 0.37471219246536641, -0.24552980800231483, -0.37471219246536647]} + - {mu0: 0, rank: 2, ndensity: 2, num_ms_combs: 6, mus: [0, 0], ns: [2, 6], ls: [5, 5], ms_combs: [0, 0, 1, -1, 2, -2, 3, -3, 4, -4, 5, -5], ctildes: [0.020524716710489142, -0.076796193169792859, -0.041049433420978283, 0.15359238633958572, 0.04104943342097829, -0.15359238633958575, -0.041049433420978297, 0.15359238633958577, 0.04104943342097829, -0.15359238633958575, -0.041049433420978249, 0.15359238633958558]} + - {mu0: 0, rank: 2, ndensity: 2, num_ms_combs: 7, mus: [0, 0], ns: [2, 6], ls: [6, 6], ms_combs: [0, 0, 1, -1, 2, -2, 3, -3, 4, -4, 5, -5, 6, -6], ctildes: [0.13512840240964671, 0.077022931927633903, -0.27025680481929354, -0.15404586385526786, 0.27025680481929343, 0.15404586385526781, -0.27025680481929348, -0.15404586385526783, 0.27025680481929348, 0.15404586385526783, -0.27025680481929343, -0.15404586385526781, 0.27025680481929326, 0.15404586385526772]} + - {mu0: 0, rank: 2, ndensity: 2, num_ms_combs: 1, mus: [0, 0], ns: [3, 3], ls: [0, 0], ms_combs: [0, 0], ctildes: [0.19404192130845199, -3.5515012942692046]} + - {mu0: 0, rank: 2, ndensity: 2, num_ms_combs: 2, mus: [0, 0], ns: [3, 3], ls: [1, 1], ms_combs: [0, 0, 1, -1], ctildes: [0.4720601623699236, 1.1996029181088324, -0.94412032473984708, -2.3992058362176638]} + - {mu0: 0, rank: 2, ndensity: 2, num_ms_combs: 3, mus: [0, 0], ns: [3, 3], ls: [2, 2], ms_combs: [0, 0, 1, -1, 2, -2], ctildes: [0.067523744152659354, -0.32160334972782706, -0.13504748830531871, 0.64320669945565423, 0.13504748830531874, -0.64320669945565434]} + - {mu0: 0, rank: 2, ndensity: 2, num_ms_combs: 4, mus: [0, 0], ns: [3, 3], ls: [3, 3], ms_combs: [0, 0, 1, -1, 2, -2, 3, -3], ctildes: [-0.13719694040526006, -0.05591999422257328, 0.27439388081052013, 0.11183998844514656, -0.27439388081052007, -0.11183998844514653, 0.27439388081052007, 0.11183998844514653]} + - {mu0: 0, rank: 2, ndensity: 2, num_ms_combs: 5, mus: [0, 0], ns: [3, 3], ls: [4, 4], ms_combs: [0, 0, 1, -1, 2, -2, 3, -3, 4, -4], ctildes: [-0.19487964430004176, -0.15166650777998536, 0.38975928860008335, 0.30333301555997055, -0.38975928860008335, -0.30333301555997055, 0.38975928860008335, 0.30333301555997055, -0.38975928860008341, -0.30333301555997061]} + - {mu0: 0, rank: 2, ndensity: 2, num_ms_combs: 6, mus: [0, 0], ns: [3, 3], ls: [5, 5], ms_combs: [0, 0, 1, -1, 2, -2, 3, -3, 4, -4, 5, -5], ctildes: [0.013965086902164987, -0.072411108053370352, -0.027930173804329973, 0.1448222161067407, 0.027930173804329977, -0.14482221610674073, -0.027930173804329984, 0.14482221610674076, 0.027930173804329977, -0.14482221610674073, -0.027930173804329946, 0.14482221610674056]} + - {mu0: 0, rank: 2, ndensity: 2, num_ms_combs: 7, mus: [0, 0], ns: [3, 3], ls: [6, 6], ms_combs: [0, 0, 1, -1, 2, -2, 3, -3, 4, -4, 5, -5, 6, -6], ctildes: [0.0072537427452838881, 0.1434237970247613, -0.014507485490567781, -0.2868475940495227, 0.014507485490567776, 0.28684759404952259, -0.014507485490567778, -0.28684759404952265, 0.014507485490567778, 0.28684759404952265, -0.014507485490567776, -0.28684759404952259, 0.014507485490567767, 0.28684759404952243]} + - {mu0: 0, rank: 2, ndensity: 2, num_ms_combs: 1, mus: [0, 0], ns: [3, 4], ls: [0, 0], ms_combs: [0, 0], ctildes: [1.1243144896025201, 0.83570643041329595]} + - {mu0: 0, rank: 2, ndensity: 2, num_ms_combs: 2, mus: [0, 0], ns: [3, 4], ls: [1, 1], ms_combs: [0, 0, 1, -1], ctildes: [-1.9909670935972132, 0.7506016975029709, 3.9819341871944256, -1.5012033950059416]} + - {mu0: 0, rank: 2, ndensity: 2, num_ms_combs: 3, mus: [0, 0], ns: [3, 4], ls: [2, 2], ms_combs: [0, 0, 1, -1, 2, -2], ctildes: [-0.43437972248161277, 0.7615570042505293, 0.86875944496322555, -1.5231140085010586, -0.86875944496322577, 1.523114008501059]} + - {mu0: 0, rank: 2, ndensity: 2, num_ms_combs: 4, mus: [0, 0], ns: [3, 4], ls: [3, 3], ms_combs: [0, 0, 1, -1, 2, -2, 3, -3], ctildes: [-0.39792366163794718, -0.3062952731459338, 0.79584732327589436, 0.6125905462918676, -0.79584732327589425, -0.61259054629186749, 0.79584732327589425, 0.61259054629186749]} + - {mu0: 0, rank: 2, ndensity: 2, num_ms_combs: 5, mus: [0, 0], ns: [3, 4], ls: [4, 4], ms_combs: [0, 0, 1, -1, 2, -2, 3, -3, 4, -4], ctildes: [-0.088243051264490197, 0.37613297619025754, 0.17648610252898031, -0.75226595238051464, -0.17648610252898031, 0.75226595238051464, 0.17648610252898031, -0.75226595238051464, -0.17648610252898034, 0.75226595238051486]} + - {mu0: 0, rank: 2, ndensity: 2, num_ms_combs: 6, mus: [0, 0], ns: [3, 4], ls: [5, 5], ms_combs: [0, 0, 1, -1, 2, -2, 3, -3, 4, -4, 5, -5], ctildes: [-0.048992922492662967, -0.011226786919308201, 0.097985844985325934, 0.022453573838616403, -0.097985844985325948, -0.022453573838616406, 0.097985844985325962, 0.02245357383861641, -0.097985844985325948, -0.022453573838616406, 0.097985844985325837, 0.022453573838616382]} + - {mu0: 0, rank: 2, ndensity: 2, num_ms_combs: 7, mus: [0, 0], ns: [3, 4], ls: [6, 6], ms_combs: [0, 0, 1, -1, 2, -2, 3, -3, 4, -4, 5, -5, 6, -6], ctildes: [-0.11164683480608291, 0.013742745695606283, 0.2232936696121659, -0.027485491391212577, -0.22329366961216582, 0.027485491391212567, 0.22329366961216587, -0.027485491391212574, -0.22329366961216587, 0.027485491391212574, 0.22329366961216582, -0.027485491391212567, -0.22329366961216568, 0.027485491391212549]} + - {mu0: 0, rank: 2, ndensity: 2, num_ms_combs: 1, mus: [0, 0], ns: [3, 5], ls: [0, 0], ms_combs: [0, 0], ctildes: [0.95692601340707439, 1.6435636435407537]} + - {mu0: 0, rank: 2, ndensity: 2, num_ms_combs: 2, mus: [0, 0], ns: [3, 5], ls: [1, 1], ms_combs: [0, 0, 1, -1], ctildes: [0.30252764942329086, -0.80305418843389487, -0.6050552988465816, 1.6061083768677895]} + - {mu0: 0, rank: 2, ndensity: 2, num_ms_combs: 3, mus: [0, 0], ns: [3, 5], ls: [2, 2], ms_combs: [0, 0, 1, -1, 2, -2], ctildes: [-0.59616386513966479, 0.69079616952950917, 1.1923277302793298, -1.3815923390590183, -1.19232773027933, 1.3815923390590188]} + - {mu0: 0, rank: 2, ndensity: 2, num_ms_combs: 4, mus: [0, 0], ns: [3, 5], ls: [3, 3], ms_combs: [0, 0, 1, -1, 2, -2, 3, -3], ctildes: [0.11680212306420745, 0.28988271719019054, -0.23360424612841491, -0.57976543438038108, 0.23360424612841488, 0.57976543438038108, -0.23360424612841488, -0.57976543438038108]} + - {mu0: 0, rank: 2, ndensity: 2, num_ms_combs: 5, mus: [0, 0], ns: [3, 5], ls: [4, 4], ms_combs: [0, 0, 1, -1, 2, -2, 3, -3, 4, -4], ctildes: [-0.12677523038520958, 0.49346740554534763, 0.25355046077041904, -0.9869348110906947, -0.25355046077041904, 0.9869348110906947, 0.25355046077041904, -0.9869348110906947, -0.25355046077041909, 0.98693481109069492]} + - {mu0: 0, rank: 2, ndensity: 2, num_ms_combs: 6, mus: [0, 0], ns: [3, 5], ls: [5, 5], ms_combs: [0, 0, 1, -1, 2, -2, 3, -3, 4, -4, 5, -5], ctildes: [-0.055090137138501367, -0.19152931837608786, 0.11018027427700273, 0.38305863675217572, -0.11018027427700275, -0.38305863675217577, 0.11018027427700278, 0.38305863675217589, -0.11018027427700275, -0.38305863675217577, 0.11018027427700262, 0.38305863675217539]} + - {mu0: 0, rank: 2, ndensity: 2, num_ms_combs: 7, mus: [0, 0], ns: [3, 5], ls: [6, 6], ms_combs: [0, 0, 1, -1, 2, -2, 3, -3, 4, -4, 5, -5, 6, -6], ctildes: [0.036218615783718107, 0.01134221956681682, -0.072437231567436242, -0.02268443913363365, 0.072437231567436214, 0.02268443913363364, -0.072437231567436228, -0.022684439133633647, 0.072437231567436228, 0.022684439133633647, -0.072437231567436214, -0.02268443913363364, 0.072437231567436172, 0.022684439133633626]} + - {mu0: 0, rank: 2, ndensity: 2, num_ms_combs: 1, mus: [0, 0], ns: [3, 6], ls: [0, 0], ms_combs: [0, 0], ctildes: [-1.8737876280167678, -0.87838187316941418]} + - {mu0: 0, rank: 2, ndensity: 2, num_ms_combs: 2, mus: [0, 0], ns: [3, 6], ls: [1, 1], ms_combs: [0, 0, 1, -1], ctildes: [-0.22596923794941043, 0.23223007277273977, 0.45193847589882075, -0.46446014554547943]} + - {mu0: 0, rank: 2, ndensity: 2, num_ms_combs: 3, mus: [0, 0], ns: [3, 6], ls: [2, 2], ms_combs: [0, 0, 1, -1, 2, -2], ctildes: [0.63908274969020196, 0.46991974693880872, -1.2781654993804039, -0.93983949387761767, 1.2781654993804044, 0.93983949387761789]} + - {mu0: 0, rank: 2, ndensity: 2, num_ms_combs: 4, mus: [0, 0], ns: [3, 6], ls: [3, 3], ms_combs: [0, 0, 1, -1, 2, -2, 3, -3], ctildes: [-0.1057035994099399, -0.25435495379928136, 0.21140719881987979, 0.50870990759856272, -0.21140719881987977, -0.50870990759856272, 0.21140719881987977, 0.50870990759856272]} + - {mu0: 0, rank: 2, ndensity: 2, num_ms_combs: 5, mus: [0, 0], ns: [3, 6], ls: [4, 4], ms_combs: [0, 0, 1, -1, 2, -2, 3, -3, 4, -4], ctildes: [-0.10678713728440517, 0.25238110592900509, 0.21357427456881023, -0.50476221185800996, -0.21357427456881023, 0.50476221185800996, 0.21357427456881023, -0.50476221185800996, -0.21357427456881029, 0.50476221185800996]} + - {mu0: 0, rank: 2, ndensity: 2, num_ms_combs: 6, mus: [0, 0], ns: [3, 6], ls: [5, 5], ms_combs: [0, 0, 1, -1, 2, -2, 3, -3, 4, -4, 5, -5], ctildes: [0.040025091372195364, 0.038161862941572679, -0.080050182744390727, -0.076323725883145357, 0.080050182744390741, 0.076323725883145371, -0.080050182744390755, -0.076323725883145385, 0.080050182744390741, 0.076323725883145371, -0.080050182744390644, -0.076323725883145288]} + - {mu0: 0, rank: 2, ndensity: 2, num_ms_combs: 7, mus: [0, 0], ns: [3, 6], ls: [6, 6], ms_combs: [0, 0, 1, -1, 2, -2, 3, -3, 4, -4, 5, -5, 6, -6], ctildes: [0.074914002116634223, -0.033198781090821021, -0.1498280042332685, 0.06639756218164207, 0.14982800423326845, -0.066397562181642042, -0.14982800423326847, 0.066397562181642056, 0.14982800423326847, -0.066397562181642056, -0.14982800423326845, 0.066397562181642042, 0.14982800423326836, -0.066397562181642]} + - {mu0: 0, rank: 2, ndensity: 2, num_ms_combs: 1, mus: [0, 0], ns: [4, 4], ls: [0, 0], ms_combs: [0, 0], ctildes: [-0.26170622209215838, 1.4193439261287595]} + - {mu0: 0, rank: 2, ndensity: 2, num_ms_combs: 2, mus: [0, 0], ns: [4, 4], ls: [1, 1], ms_combs: [0, 0, 1, -1], ctildes: [0.42380478590445808, 0.028277776383595007, -0.84760957180891605, -0.056555552767190001]} + - {mu0: 0, rank: 2, ndensity: 2, num_ms_combs: 3, mus: [0, 0], ns: [4, 4], ls: [2, 2], ms_combs: [0, 0, 1, -1, 2, -2], ctildes: [0.11925206169147308, 0.32471265776033481, -0.23850412338294621, -0.64942531552066962, 0.23850412338294627, 0.64942531552066984]} + - {mu0: 0, rank: 2, ndensity: 2, num_ms_combs: 4, mus: [0, 0], ns: [4, 4], ls: [3, 3], ms_combs: [0, 0, 1, -1, 2, -2, 3, -3], ctildes: [-0.19782158663421806, -0.0931577759320142, 0.39564317326843612, 0.1863155518640284, -0.39564317326843607, -0.18631555186402837, 0.39564317326843607, 0.18631555186402837]} + - {mu0: 0, rank: 2, ndensity: 2, num_ms_combs: 5, mus: [0, 0], ns: [4, 4], ls: [4, 4], ms_combs: [0, 0, 1, -1, 2, -2, 3, -3, 4, -4], ctildes: [-0.14960715497261459, 0.040905647683395566, 0.29921430994522902, -0.08181129536679109, -0.29921430994522902, 0.08181129536679109, 0.29921430994522902, -0.08181129536679109, -0.29921430994522907, 0.081811295366791104]} + - {mu0: 0, rank: 2, ndensity: 2, num_ms_combs: 6, mus: [0, 0], ns: [4, 4], ls: [5, 5], ms_combs: [0, 0, 1, -1, 2, -2, 3, -3, 4, -4, 5, -5], ctildes: [0.091980186745003301, -0.069652214802700979, -0.1839603734900066, 0.13930442960540196, 0.18396037349000663, -0.13930442960540199, -0.18396037349000666, 0.13930442960540201, 0.18396037349000663, -0.13930442960540199, -0.18396037349000643, 0.13930442960540185]} + - {mu0: 0, rank: 2, ndensity: 2, num_ms_combs: 7, mus: [0, 0], ns: [4, 4], ls: [6, 6], ms_combs: [0, 0, 1, -1, 2, -2, 3, -3, 4, -4, 5, -5, 6, -6], ctildes: [0.11162746569924076, -0.41027160777901672, -0.2232549313984816, 0.82054321555803378, 0.22325493139848152, -0.82054321555803345, -0.22325493139848157, 0.82054321555803367, 0.22325493139848157, -0.82054321555803367, -0.22325493139848152, 0.82054321555803345, 0.22325493139848138, -0.820543215558033]} + - {mu0: 0, rank: 2, ndensity: 2, num_ms_combs: 1, mus: [0, 0], ns: [4, 5], ls: [0, 0], ms_combs: [0, 0], ctildes: [0.31712701649388253, 0.73590877701621382]} + - {mu0: 0, rank: 2, ndensity: 2, num_ms_combs: 2, mus: [0, 0], ns: [4, 5], ls: [1, 1], ms_combs: [0, 0, 1, -1], ctildes: [0.033783993053337853, 2.0825045797800863, -0.067567986106675693, -4.1650091595601717]} + - {mu0: 0, rank: 2, ndensity: 2, num_ms_combs: 3, mus: [0, 0], ns: [4, 5], ls: [2, 2], ms_combs: [0, 0, 1, -1, 2, -2], ctildes: [0.34725841917651501, -0.92672059416901986, -0.69451683835303013, 1.8534411883380399, 0.69451683835303024, -1.8534411883380404]} + - {mu0: 0, rank: 2, ndensity: 2, num_ms_combs: 4, mus: [0, 0], ns: [4, 5], ls: [3, 3], ms_combs: [0, 0, 1, -1, 2, -2, 3, -3], ctildes: [0.011336366709675234, 0.41653100544127236, -0.022672733419350468, -0.83306201088254472, 0.022672733419350465, 0.83306201088254461, -0.022672733419350465, -0.83306201088254461]} + - {mu0: 0, rank: 2, ndensity: 2, num_ms_combs: 5, mus: [0, 0], ns: [4, 5], ls: [4, 4], ms_combs: [0, 0, 1, -1, 2, -2, 3, -3, 4, -4], ctildes: [0.25736950174304662, 0.18998072843909167, -0.51473900348609303, -0.37996145687818311, 0.51473900348609303, 0.37996145687818311, -0.51473900348609303, -0.37996145687818311, 0.51473900348609303, 0.37996145687818317]} + - {mu0: 0, rank: 2, ndensity: 2, num_ms_combs: 6, mus: [0, 0], ns: [4, 5], ls: [5, 5], ms_combs: [0, 0, 1, -1, 2, -2, 3, -3, 4, -4, 5, -5], ctildes: [0.26011921149041684, 0.051837150895556343, -0.52023842298083367, -0.10367430179111269, 0.52023842298083378, 0.1036743017911127, -0.52023842298083389, -0.10367430179111271, 0.52023842298083378, 0.1036743017911127, -0.52023842298083323, -0.10367430179111259]} + - {mu0: 0, rank: 2, ndensity: 2, num_ms_combs: 7, mus: [0, 0], ns: [4, 5], ls: [6, 6], ms_combs: [0, 0, 1, -1, 2, -2, 3, -3, 4, -4, 5, -5, 6, -6], ctildes: [0.00030878910168388453, -0.064117743784754169, -0.00061757820336776927, 0.12823548756950839, 0.00061757820336776905, -0.12823548756950834, -0.00061757820336776916, 0.12823548756950837, 0.00061757820336776916, -0.12823548756950837, -0.00061757820336776905, 0.12823548756950834, 0.00061757820336776873, -0.12823548756950826]} + - {mu0: 0, rank: 2, ndensity: 2, num_ms_combs: 1, mus: [0, 0], ns: [4, 6], ls: [0, 0], ms_combs: [0, 0], ctildes: [-2.5237969566223573, 1.2797140092791259]} + - {mu0: 0, rank: 2, ndensity: 2, num_ms_combs: 2, mus: [0, 0], ns: [4, 6], ls: [1, 1], ms_combs: [0, 0, 1, -1], ctildes: [-0.54556435016182991, -0.79335662230150228, 1.0911287003236598, 1.5867132446030041]} + - {mu0: 0, rank: 2, ndensity: 2, num_ms_combs: 3, mus: [0, 0], ns: [4, 6], ls: [2, 2], ms_combs: [0, 0, 1, -1, 2, -2], ctildes: [-0.2946311104832936, -0.46183833969501031, 0.5892622209665872, 0.92367667939002074, -0.58926222096658742, -0.92367667939002096]} + - {mu0: 0, rank: 2, ndensity: 2, num_ms_combs: 4, mus: [0, 0], ns: [4, 6], ls: [3, 3], ms_combs: [0, 0, 1, -1, 2, -2, 3, -3], ctildes: [-0.1467557943811785, 0.24295386604587921, 0.293511588762357, -0.48590773209175842, -0.29351158876235695, 0.48590773209175836, 0.29351158876235695, -0.48590773209175836]} + - {mu0: 0, rank: 2, ndensity: 2, num_ms_combs: 5, mus: [0, 0], ns: [4, 6], ls: [4, 4], ms_combs: [0, 0, 1, -1, 2, -2, 3, -3, 4, -4], ctildes: [-0.3278121219030985, 0.47830646673699784, 0.65562424380619666, -0.95661293347399512, -0.65562424380619666, 0.95661293347399512, 0.65562424380619666, -0.95661293347399512, -0.65562424380619677, 0.95661293347399534]} + - {mu0: 0, rank: 2, ndensity: 2, num_ms_combs: 6, mus: [0, 0], ns: [4, 6], ls: [5, 5], ms_combs: [0, 0, 1, -1, 2, -2, 3, -3, 4, -4, 5, -5], ctildes: [-0.12301526230267197, 0.036191856474107019, 0.24603052460534394, -0.072383712948214038, -0.24603052460534397, 0.072383712948214052, 0.24603052460534403, -0.072383712948214052, -0.24603052460534397, 0.072383712948214052, 0.24603052460534372, -0.072383712948213969]} + - {mu0: 0, rank: 2, ndensity: 2, num_ms_combs: 7, mus: [0, 0], ns: [4, 6], ls: [6, 6], ms_combs: [0, 0, 1, -1, 2, -2, 3, -3, 4, -4, 5, -5, 6, -6], ctildes: [-0.063736094883519681, 0.049337335141767531, 0.12747218976703939, -0.098674670283535104, -0.12747218976703936, 0.098674670283535063, 0.12747218976703939, -0.09867467028353509, -0.12747218976703939, 0.09867467028353509, 0.12747218976703936, -0.098674670283535063, -0.12747218976703928, 0.098674670283535007]} + - {mu0: 0, rank: 2, ndensity: 2, num_ms_combs: 1, mus: [0, 0], ns: [5, 5], ls: [0, 0], ms_combs: [0, 0], ctildes: [1.3203684598764522, -1.271530748135441]} + - {mu0: 0, rank: 2, ndensity: 2, num_ms_combs: 2, mus: [0, 0], ns: [5, 5], ls: [1, 1], ms_combs: [0, 0, 1, -1], ctildes: [0.28956199367223412, -0.67043525104918977, -0.57912398734446813, 1.3408705020983791]} + - {mu0: 0, rank: 2, ndensity: 2, num_ms_combs: 3, mus: [0, 0], ns: [5, 5], ls: [2, 2], ms_combs: [0, 0, 1, -1, 2, -2], ctildes: [0.12726382821095528, 0.17505816133998198, -0.25452765642191061, -0.35011632267996401, 0.25452765642191066, 0.35011632267996412]} + - {mu0: 0, rank: 2, ndensity: 2, num_ms_combs: 4, mus: [0, 0], ns: [5, 5], ls: [3, 3], ms_combs: [0, 0, 1, -1, 2, -2, 3, -3], ctildes: [-0.024562311483278892, 0.12112919763792822, 0.049124622966557785, -0.24225839527585644, -0.049124622966557778, 0.24225839527585641, 0.049124622966557778, -0.24225839527585641]} + - {mu0: 0, rank: 2, ndensity: 2, num_ms_combs: 5, mus: [0, 0], ns: [5, 5], ls: [4, 4], ms_combs: [0, 0, 1, -1, 2, -2, 3, -3, 4, -4], ctildes: [0.24018196363397118, -0.81456382726225851, -0.48036392726794214, 1.6291276545245161, 0.48036392726794214, -1.6291276545245161, -0.48036392726794214, 1.6291276545245161, 0.48036392726794219, -1.6291276545245164]} + - {mu0: 0, rank: 2, ndensity: 2, num_ms_combs: 6, mus: [0, 0], ns: [5, 5], ls: [5, 5], ms_combs: [0, 0, 1, -1, 2, -2, 3, -3, 4, -4, 5, -5], ctildes: [0.20103711231454144, 0.11764103607520673, -0.40207422462908288, -0.23528207215041347, 0.40207422462908299, 0.23528207215041352, -0.40207422462908304, -0.23528207215041355, 0.40207422462908299, 0.23528207215041352, -0.40207422462908254, -0.23528207215041325]} + - {mu0: 0, rank: 2, ndensity: 2, num_ms_combs: 7, mus: [0, 0], ns: [5, 5], ls: [6, 6], ms_combs: [0, 0, 1, -1, 2, -2, 3, -3, 4, -4, 5, -5, 6, -6], ctildes: [-0.020609505258132651, 0.098430059836913386, 0.041219010516265316, -0.19686011967382686, -0.041219010516265302, 0.19686011967382677, 0.041219010516265309, -0.1968601196738268, -0.041219010516265309, 0.1968601196738268, 0.041219010516265302, -0.19686011967382677, -0.041219010516265274, 0.19686011967382666]} + - {mu0: 0, rank: 2, ndensity: 2, num_ms_combs: 1, mus: [0, 0], ns: [5, 6], ls: [0, 0], ms_combs: [0, 0], ctildes: [-1.0693859200297273, -0.74765058561447162]} + - {mu0: 0, rank: 2, ndensity: 2, num_ms_combs: 2, mus: [0, 0], ns: [5, 6], ls: [1, 1], ms_combs: [0, 0, 1, -1], ctildes: [-0.57968251173743246, 0.44608716522614478, 1.1593650234748647, -0.89217433045228933]} + - {mu0: 0, rank: 2, ndensity: 2, num_ms_combs: 3, mus: [0, 0], ns: [5, 6], ls: [2, 2], ms_combs: [0, 0, 1, -1, 2, -2], ctildes: [-0.045670616514188807, -1.0053870520320305, 0.091341233028377627, 2.0107741040640614, -0.091341233028377655, -2.0107741040640619]} + - {mu0: 0, rank: 2, ndensity: 2, num_ms_combs: 4, mus: [0, 0], ns: [5, 6], ls: [3, 3], ms_combs: [0, 0, 1, -1, 2, -2, 3, -3], ctildes: [0.0024738995211484654, -0.20542657027815012, -0.0049477990422969308, 0.41085314055630023, 0.00494779904229693, -0.41085314055630018, -0.00494779904229693, 0.41085314055630018]} + - {mu0: 0, rank: 2, ndensity: 2, num_ms_combs: 5, mus: [0, 0], ns: [5, 6], ls: [4, 4], ms_combs: [0, 0, 1, -1, 2, -2, 3, -3, 4, -4], ctildes: [0.29152366093716703, 0.13903010299853416, -0.58304732187433372, -0.27806020599706821, 0.58304732187433372, 0.27806020599706821, -0.58304732187433372, -0.27806020599706821, 0.58304732187433383, 0.27806020599706827]} + - {mu0: 0, rank: 2, ndensity: 2, num_ms_combs: 6, mus: [0, 0], ns: [5, 6], ls: [5, 5], ms_combs: [0, 0, 1, -1, 2, -2, 3, -3, 4, -4, 5, -5], ctildes: [-0.18762517289100256, -0.048677520399145392, 0.37525034578200511, 0.097355040798290784, -0.37525034578200522, -0.097355040798290798, 0.37525034578200528, 0.097355040798290812, -0.37525034578200522, -0.097355040798290798, 0.37525034578200478, 0.097355040798290687]} + - {mu0: 0, rank: 2, ndensity: 2, num_ms_combs: 7, mus: [0, 0], ns: [5, 6], ls: [6, 6], ms_combs: [0, 0, 1, -1, 2, -2, 3, -3, 4, -4, 5, -5, 6, -6], ctildes: [-0.042787457024574392, 0.28769806539677339, 0.085574914049148812, -0.57539613079354712, -0.085574914049148784, 0.57539613079354679, 0.085574914049148798, -0.57539613079354701, -0.085574914049148798, 0.57539613079354701, 0.085574914049148784, -0.57539613079354679, -0.085574914049148729, 0.57539613079354646]} + - {mu0: 0, rank: 2, ndensity: 2, num_ms_combs: 1, mus: [0, 0], ns: [6, 6], ls: [0, 0], ms_combs: [0, 0], ctildes: [1.1991236148274658, -1.8743020319689565]} + - {mu0: 0, rank: 2, ndensity: 2, num_ms_combs: 2, mus: [0, 0], ns: [6, 6], ls: [1, 1], ms_combs: [0, 0, 1, -1], ctildes: [0.20538831472964592, 0.49634893087140386, -0.41077662945929178, -0.99269786174280761]} + - {mu0: 0, rank: 2, ndensity: 2, num_ms_combs: 3, mus: [0, 0], ns: [6, 6], ls: [2, 2], ms_combs: [0, 0, 1, -1, 2, -2], ctildes: [0.0093457171659961386, -0.065775382445376632, -0.018691434331992277, 0.13155076489075326, 0.018691434331992284, -0.13155076489075332]} + - {mu0: 0, rank: 2, ndensity: 2, num_ms_combs: 4, mus: [0, 0], ns: [6, 6], ls: [3, 3], ms_combs: [0, 0, 1, -1, 2, -2, 3, -3], ctildes: [0.055244377766581435, -0.4721792277606921, -0.11048875553316287, 0.9443584555213842, 0.11048875553316284, -0.94435845552138409, -0.11048875553316284, 0.94435845552138409]} + - {mu0: 0, rank: 2, ndensity: 2, num_ms_combs: 5, mus: [0, 0], ns: [6, 6], ls: [4, 4], ms_combs: [0, 0, 1, -1, 2, -2, 3, -3, 4, -4], ctildes: [-0.16073034007712958, 0.39499144407495013, 0.32146068015425899, -0.78998288814989981, -0.32146068015425899, 0.78998288814989981, 0.32146068015425899, -0.78998288814989981, -0.32146068015425905, 0.78998288814990003]} + - {mu0: 0, rank: 2, ndensity: 2, num_ms_combs: 6, mus: [0, 0], ns: [6, 6], ls: [5, 5], ms_combs: [0, 0, 1, -1, 2, -2, 3, -3, 4, -4, 5, -5], ctildes: [0.040546356546338355, 0.0054608196303276664, -0.08109271309267671, -0.010921639260655333, 0.081092713092676724, 0.010921639260655336, -0.081092713092676738, -0.010921639260655338, 0.081092713092676724, 0.010921639260655336, -0.081092713092676627, -0.010921639260655324]} + - {mu0: 0, rank: 2, ndensity: 2, num_ms_combs: 7, mus: [0, 0], ns: [6, 6], ls: [6, 6], ms_combs: [0, 0, 1, -1, 2, -2, 3, -3, 4, -4, 5, -5, 6, -6], ctildes: [0.0069128145976557788, 0.14239646679464524, -0.013825629195311563, -0.28479293358929064, 0.013825629195311558, 0.28479293358929048, -0.013825629195311561, -0.28479293358929053, 0.013825629195311561, 0.28479293358929053, -0.013825629195311558, -0.28479293358929048, 0.013825629195311549, 0.28479293358929031]} + - {mu0: 0, rank: 3, ndensity: 2, num_ms_combs: 1, mus: [0, 0, 0], ns: [1, 1, 1], ls: [0, 0, 0], ms_combs: [0, 0, 0], ctildes: [-0.0054333765345306404, -1.091355395952734]} + - {mu0: 0, rank: 3, ndensity: 2, num_ms_combs: 2, mus: [0, 0, 0], ns: [1, 1, 1], ls: [0, 1, 1], ms_combs: [0, 0, 0, 0, 1, -1], ctildes: [0.11023379639597022, -0.88466183395068954, -0.22046759279194042, 1.7693236679013788]} + - {mu0: 0, rank: 3, ndensity: 2, num_ms_combs: 3, mus: [0, 0, 0], ns: [1, 1, 1], ls: [0, 2, 2], ms_combs: [0, 0, 0, 0, 1, -1, 0, 2, -2], ctildes: [-0.055007053567298375, -1.1414016594613499, 0.11001410713459676, 2.2828033189226997, -0.11001410713459679, -2.2828033189227006]} + - {mu0: 0, rank: 3, ndensity: 2, num_ms_combs: 4, mus: [0, 0, 0], ns: [1, 1, 1], ls: [0, 3, 3], ms_combs: [0, 0, 0, 0, 1, -1, 0, 2, -2, 0, 3, -3], ctildes: [-0.048008180007174814, 0.23916868115333242, 0.096016360014349628, -0.47833736230666485, -0.096016360014349614, 0.47833736230666479, 0.096016360014349614, -0.47833736230666479]} + - {mu0: 0, rank: 3, ndensity: 2, num_ms_combs: 5, mus: [0, 0, 0], ns: [1, 1, 1], ls: [0, 4, 4], ms_combs: [0, 0, 0, 0, 1, -1, 0, 2, -2, 0, 3, -3, 0, 4, -4], ctildes: [-0.0032253196316928512, 0.011773626632183582, 0.0064506392633856997, -0.02354725326436715, -0.0064506392633856997, 0.02354725326436715, 0.0064506392633856997, -0.02354725326436715, -0.0064506392633857006, 0.023547253264367157]} + - {mu0: 0, rank: 3, ndensity: 2, num_ms_combs: 4, mus: [0, 0, 0], ns: [1, 1, 1], ls: [1, 1, 2], ms_combs: [-1, 1, 0, 0, 0, 0, 0, 1, -1, 1, 1, -2], ctildes: [0.05389763395342366, -1.1452467330933471, 0.053897633953423653, -1.1452467330933469, -0.18670688083015841, 3.967251057839901, 0.13202170152919365, -2.8052701256680983]} + - {mu0: 0, rank: 3, ndensity: 2, num_ms_combs: 8, mus: [0, 0, 0], ns: [1, 1, 1], ls: [1, 2, 3], ms_combs: [0, 0, 0, 0, 1, -1, 0, 2, -2, 1, -2, 1, 1, -1, 0, 1, 0, -1, 1, 1, -2, 1, 2, -3], ctildes: [0.16708253023261796, -0.092547161971829953, -0.31505384038744133, 0.17450860215960193, 0.24907193030186495, -0.1379611635291301, -0.11138835348841196, 0.061698107981219955, 0.1929302876133715, -0.10686425775434381, -0.27284462933537196, 0.15112888264912724, 0.35224090183934365, -0.19510654854366821, -0.4314052380220843, 0.23895574470377176]} + - {mu0: 0, rank: 3, ndensity: 2, num_ms_combs: 11, mus: [0, 0, 0], ns: [1, 1, 1], ls: [1, 3, 4], ms_combs: [0, 0, 0, 0, 1, -1, 0, 2, -2, 0, 3, -3, 1, -3, 2, 1, -2, 1, 1, -1, 0, 1, 0, -1, 1, 1, -2, 1, 2, -3, 1, 3, -4], ctildes: [-0.026541782255918491, 0.35074109539417897, 0.051397940327917897, -0.67920721064610068, -0.045971715390680901, 0.60750139752508003, 0.035111477600793597, -0.46398685649168953, -0.013270891127959237, 0.17537054769708935, 0.02298585769534045, -0.30375069876254002, -0.032506911695528447, 0.42956835777028873, 0.041966242544472239, -0.55457036523399894, -0.05139794032791789, 0.67920721064610057, 0.060814863133391101, -0.80364880948777562, -0.070222955201587209, 0.92797371298337927]} + - {mu0: 0, rank: 3, ndensity: 2, num_ms_combs: 5, mus: [0, 0, 0], ns: [1, 1, 1], ls: [2, 2, 2], ms_combs: [-2, 0, 2, -2, 1, 1, -1, -1, 2, -1, 0, 1, 0, 0, 0], ctildes: [0.51364223579557589, 0.47715252900136113, -0.41938712934716049, -0.38959340851062896, -0.20969356467358025, -0.19479670425531448, 0.25682111789778789, 0.23857626450068048, -0.085607039299262611, -0.079525421500226817]} + - {mu0: 0, rank: 3, ndensity: 2, num_ms_combs: 10, mus: [0, 0, 0], ns: [1, 1, 1], ls: [2, 2, 4], ms_combs: [-2, 1, 1, -2, 2, 0, -1, 1, 0, -1, 2, -1, 0, 0, 0, 0, 1, -1, 0, 2, -2, 1, 1, -2, 1, 2, -3, 2, 2, -4], ctildes: [0.03663096610921749, -0.32881781084918771, -0.016381866060340259, 0.14705179545429034, -0.065527464241361064, 0.58820718181716147, 0.03663096610921749, -0.32881781084918771, -0.049145598181020801, 0.44115538636287116, 0.17945435150553293, -1.6108717098390091, -0.12689338886299661, 1.1390583096487317, -0.10360801814896914, 0.93003721530550432, 0.19383285321804952, -1.7399403083112539, -0.13706052492721954, 1.2303235908666998]} + - {mu0: 0, rank: 3, ndensity: 2, num_ms_combs: 9, mus: [0, 0, 0], ns: [1, 1, 1], ls: [2, 3, 3], ms_combs: [0, -3, 3, 0, -1, 1, 0, 0, 0, 1, -3, 2, 1, -2, 1, 1, -1, 0, 2, -3, 1, 2, -2, 0, 2, -1, -1], ctildes: [-0.068317839280451514, -0.15077979199196659, 0.040990703568270906, 0.090467875195179931, -0.027327135712180609, -0.060311916796786637, 0.13663567856090303, 0.30155958398393318, -0.10583754151282547, -0.23358704933180205, 0.03864640594497596, 0.085293930706733329, -0.086415990779018237, -0.19072302712841238, 0.12221066616559595, 0.2697230916218526, -0.06693753862663028, -0.14773342156132133]} + - {mu0: 0, rank: 3, ndensity: 2, num_ms_combs: 13, mus: [0, 0, 0], ns: [1, 1, 1], ls: [2, 4, 4], ms_combs: [0, -4, 4, 0, -3, 3, 0, -2, 2, 0, -1, 1, 0, 0, 0, 1, -4, 3, 1, -3, 2, 1, -2, 1, 1, -1, 0, 2, -4, 2, 2, -3, 1, 2, -2, 0, 2, -1, -1], ctildes: [0.010714959601891521, 0.011855642128166357, -0.0026787399004728806, -0.0029639105320415898, -0.0030614170291118621, -0.0033873263223332438, 0.0065055111868627105, 0.0071980684349581469, -0.0038267712863898308, -0.0042341579029165589, -0.018558854431524101, -0.020534574522338138, 0.01753646908716492, 0.019403349094510594, -0.011930692134551212, -0.01320079790721738, 0.0041920179119375447, 0.0046382875909323365, 0.0099201249676820653, 0.01097619177796664, -0.014880187451523096, -0.016464287666949958, 0.01778522575431905, 0.019678587651850012, -0.0093736370139890727, -0.010371526352518435]} + - {mu0: 0, rank: 3, ndensity: 2, num_ms_combs: 17, mus: [0, 0, 0], ns: [1, 1, 1], ls: [3, 3, 4], ms_combs: [-3, 1, 2, -3, 2, 1, -3, 3, 0, -2, 1, 1, -2, 2, 0, -2, 3, -1, -1, 1, 0, -1, 2, -1, -1, 3, -2, 0, 0, 0, 0, 1, -1, 0, 2, -2, 0, 3, -3, 1, 1, -2, 1, 2, -3, 1, 3, -4, 2, 2, -4], ctildes: [0.030307413524744482, -0.3178169430554757, -0.022589812287841723, 0.23688676302440775, 0.012372949759858304, -0.12974822368284894, -0.023330657809270038, 0.24465559683480373, 0.028870216106336038, -0.30274585525998077, -0.022589812287841723, 0.23688676302440775, 0.0041243165866194347, -0.043249407894282978, -0.023330657809270038, 0.24465559683480373, 0.030307413524744482, -0.3178169430554757, -0.012372949759858304, 0.12974822368284894, 0.031946818908928164, -0.33500847301577896, 0.014287051749047815, -0.14982034374033723, -0.065471496097762802, 0.68656306583439819, -0.026084468410657077, 0.27353327279920192, 0.030863559243437772, -0.32364893304248599, 0.053457252712046308, -0.5605763958450436, -0.034506508247959436, 0.36185050756413845]} + - {mu0: 0, rank: 3, ndensity: 2, num_ms_combs: 13, mus: [0, 0, 0], ns: [1, 1, 1], ls: [4, 4, 4], ms_combs: [-4, 0, 4, -4, 1, 3, -4, 2, 2, -3, -1, 4, -3, 0, 3, -3, 1, 2, -2, -2, 4, -2, -1, 3, -2, 0, 2, -2, 1, 1, -1, -1, 2, -1, 0, 1, 0, 0, 0], ctildes: [-0.0057243101026909346, 0.05870621191581029, 0.012067905305077136, -0.12376356163631838, -0.0068418592034381054, 0.070167344027424139, 0.0060339526525385653, -0.06188178081815917, -0.0085864651540364014, 0.088059317873715431, 0.0045612394689587363, -0.046778229351616088, -0.0034209296017190523, 0.035083672013712063, 0.0022806197344793682, -0.023389114675808044, 0.0044976722235428764, -0.04612630936242236, -0.0051719594164616293, 0.053041526415565025, -0.0025859797082308147, 0.026520763207782513, 0.0036799136374441725, -0.037739707660163761, -0.001226637879148058, 0.012579902553387924]} + - {mu0: 0, rank: 3, ndensity: 2, num_ms_combs: 1, mus: [0, 0, 0], ns: [1, 1, 2], ls: [0, 0, 0], ms_combs: [0, 0, 0], ctildes: [-0.53374308296652839, 6.6932468212772411]} + - {mu0: 0, rank: 3, ndensity: 2, num_ms_combs: 2, mus: [0, 0, 0], ns: [1, 1, 2], ls: [0, 1, 1], ms_combs: [0, 0, 0, 0, 1, -1], ctildes: [-0.7761902490355902, 0.18611292623692396, 1.5523804980711802, -0.37222585247384787]} + - {mu0: 0, rank: 3, ndensity: 2, num_ms_combs: 3, mus: [0, 0, 0], ns: [1, 1, 2], ls: [0, 2, 2], ms_combs: [0, 0, 0, 0, 1, -1, 0, 2, -2], ctildes: [-0.26374971224304777, -0.88668004828514924, 0.52749942448609566, 1.7733600965702987, -0.52749942448609577, -1.7733600965702991]} + - {mu0: 0, rank: 3, ndensity: 2, num_ms_combs: 4, mus: [0, 0, 0], ns: [1, 1, 2], ls: [0, 3, 3], ms_combs: [0, 0, 0, 0, 1, -1, 0, 2, -2, 0, 3, -3], ctildes: [0.10025317685348403, -0.35382911263209366, -0.20050635370696807, 0.70765822526418731, 0.20050635370696804, -0.7076582252641872, -0.20050635370696804, 0.7076582252641872]} + - {mu0: 0, rank: 3, ndensity: 2, num_ms_combs: 5, mus: [0, 0, 0], ns: [1, 1, 2], ls: [0, 4, 4], ms_combs: [0, 0, 0, 0, 1, -1, 0, 2, -2, 0, 3, -3, 0, 4, -4], ctildes: [0.0060028361701652917, 0.027959376616650811, -0.012005672340330578, -0.055918753233301595, 0.012005672340330578, 0.055918753233301595, -0.012005672340330578, -0.055918753233301595, 0.01200567234033058, 0.055918753233301602]} + - {mu0: 0, rank: 3, ndensity: 2, num_ms_combs: 2, mus: [0, 0, 0], ns: [1, 1, 2], ls: [1, 1, 0], ms_combs: [0, 0, 0, 1, -1, 0], ctildes: [-0.71283367197452729, -2.6540702049770886, 1.4256673439490544, 5.3081404099541754]} + - {mu0: 0, rank: 3, ndensity: 2, num_ms_combs: 4, mus: [0, 0, 0], ns: [1, 1, 2], ls: [1, 1, 2], ms_combs: [-1, 1, 0, 0, 0, 0, 0, 1, -1, 1, 1, -2], ctildes: [-0.15655534147473699, 0.93974948237190137, -0.15655534147473696, 0.93974948237190126, 0.54232361126107909, -3.2553876997093729, -0.3834807031202862, 2.3019067178557742]} + - {mu0: 0, rank: 3, ndensity: 2, num_ms_combs: 5, mus: [0, 0, 0], ns: [1, 1, 2], ls: [1, 2, 1], ms_combs: [0, 0, 0, 0, 1, -1, 1, -2, 1, 1, -1, 0, 1, 0, -1], ctildes: [-0.24934587965883112, 0.92522168800678395, 0.43187973222705056, -1.6025309718923901, -0.61077017462955552, 2.266321034573155, 0.43187973222705056, -1.6025309718923901, -0.24934587965883109, 0.92522168800678384]} + - {mu0: 0, rank: 3, ndensity: 2, num_ms_combs: 8, mus: [0, 0, 0], ns: [1, 1, 2], ls: [1, 2, 3], ms_combs: [0, 0, 0, 0, 1, -1, 0, 2, -2, 1, -2, 1, 1, -1, 0, 1, 0, -1, 1, 1, -2, 1, 2, -3], ctildes: [-0.012637249888729293, 0.18421750651936872, 0.023829026911651802, -0.34736386152832704, -0.018838499866566896, 0.27461524481521266, 0.0084248332591528603, -0.12281167101291246, -0.014592239250148848, 0.2127160539567983, 0.020636542652953508, -0.30082592844019129, -0.02664166200606264, 0.38836460365208147, 0.032629238907273379, -0.47564755655291402]} + - {mu0: 0, rank: 3, ndensity: 2, num_ms_combs: 8, mus: [0, 0, 0], ns: [1, 1, 2], ls: [1, 3, 2], ms_combs: [0, 0, 0, 0, 1, -1, 0, 2, -2, 1, -3, 2, 1, -2, 1, 1, -1, 0, 1, 0, -1, 1, 1, -2], ctildes: [0.2891506300058474, 0.3235370369833025, -0.54522765669732576, -0.61006728750905626, 0.43104030961997708, 0.48230053862235223, -0.74658371637202015, -0.8353690374117495, 0.60958305179406946, 0.68207596285957905, -0.47218100154574488, -0.52833376900070772, 0.3338823881404519, 0.37358839079024736, -0.19276708667056502, -0.21569135798886846]} + - {mu0: 0, rank: 3, ndensity: 2, num_ms_combs: 11, mus: [0, 0, 0], ns: [1, 1, 2], ls: [1, 3, 4], ms_combs: [0, 0, 0, 0, 1, -1, 0, 2, -2, 0, 3, -3, 1, -3, 2, 1, -2, 1, 1, -1, 0, 1, 0, -1, 1, 1, -2, 1, 2, -3, 1, 3, -4], ctildes: [0.022093750204418925, -0.3786268725357318, -0.042784363298490585, 0.73320778587874347, 0.038267497883788831, -0.65580098034279244, -0.029227284284837455, 0.5008762722078488, 0.011046875102209456, -0.18931343626786579, -0.019133748941894416, 0.32790049017139622, 0.02705920725266895, -0.4637213203091744, -0.034933286350387248, 0.59866165027958229, 0.042784363298490578, -0.73320778587874336, -0.050623141348597869, 0.86754315176969321, 0.058454568569674917, -1.0017525444156976]} + - {mu0: 0, rank: 3, ndensity: 2, num_ms_combs: 11, mus: [0, 0, 0], ns: [1, 1, 2], ls: [1, 4, 3], ms_combs: [0, 0, 0, 0, 1, -1, 0, 2, -2, 0, 3, -3, 1, -4, 3, 1, -3, 2, 1, -2, 1, 1, -1, 0, 1, 0, -1, 1, 1, -2, 1, 2, -3], ctildes: [0.023219221282882015, -0.37805128961919787, -0.044963828670253427, 0.73209317435369503, 0.040216870974136246, -0.65480404148738702, -0.030716142575541956, 0.50011484757982583, 0.061432285151083912, -1.0002296951596517, -0.053201919553368233, 0.86622432562782337, 0.044963828670253427, -0.73209317435369503, -0.036712812374681991, 0.59775157378031774, 0.028437622183916173, -0.46301637808408874, -0.020108435487068123, 0.32740202074369351, 0.011609610641441004, -0.18902564480959888]} + - {mu0: 0, rank: 3, ndensity: 2, num_ms_combs: 3, mus: [0, 0, 0], ns: [1, 1, 2], ls: [2, 2, 0], ms_combs: [0, 0, 0, 1, -1, 0, 2, -2, 0], ctildes: [0.66188258032213032, -1.6035862271242129, -1.3237651606442606, 3.2071724542484263, 1.3237651606442611, -3.2071724542484272]} + - {mu0: 0, rank: 3, ndensity: 2, num_ms_combs: 8, mus: [0, 0, 0], ns: [1, 1, 2], ls: [2, 2, 2], ms_combs: [-2, 1, 1, -2, 2, 0, -1, 1, 0, -1, 2, -1, 0, 0, 0, 0, 1, -1, 0, 2, -2, 1, 1, -2], ctildes: [-0.29994043012665994, -0.4410011705743595, 0.24490033568040928, 0.36007594795908937, 0.12245016784020464, 0.18003797397954469, -0.29994043012665994, -0.4410011705743595, -0.12245016784020464, -0.18003797397954469, 0.24490033568040928, 0.36007594795908937, 0.48980067136081884, 0.72015189591817907, -0.29994043012665994, -0.4410011705743595]} + - {mu0: 0, rank: 3, ndensity: 2, num_ms_combs: 10, mus: [0, 0, 0], ns: [1, 1, 2], ls: [2, 2, 4], ms_combs: [-2, 1, 1, -2, 2, 0, -1, 1, 0, -1, 2, -1, 0, 0, 0, 0, 1, -1, 0, 2, -2, 1, 1, -2, 1, 2, -3, 2, 2, -4], ctildes: [-0.044085586214434681, 0.09902757220059985, 0.019715673520680717, -0.044286476617461948, 0.078862694082722895, -0.17714590646984785, -0.044085586214434681, 0.09902757220059985, 0.059147020562042171, -0.13285942985238589, -0.21597438247368247, 0.48513404471617988, 0.15271695040971789, -0.34304157280326852, 0.12469286785924374, -0.28009227130993836, -0.23327899505178321, 0.5240046579625609, 0.164953159309499, -0.3705272470186643]} + - {mu0: 0, rank: 3, ndensity: 2, num_ms_combs: 8, mus: [0, 0, 0], ns: [1, 1, 2], ls: [2, 3, 1], ms_combs: [0, 0, 0, 0, 1, -1, 1, -2, 1, 1, -1, 0, 1, 0, -1, 2, -3, 1, 2, -2, 0, 2, -1, -1], ctildes: [-0.12230856939060937, 0.21052301835788501, 0.19972905745118807, -0.34378264939159586, -0.25784910442071868, 0.44382149193623815, 0.23062725016886743, -0.39696601033792139, -0.14122977092373262, 0.24309104263907477, 0.31579936823218946, -0.54356809606227119, -0.18232685025876835, 0.31382918658444464, 0.081539046260406214, -0.14034867890525662]} + - {mu0: 0, rank: 3, ndensity: 2, num_ms_combs: 14, mus: [0, 0, 0], ns: [1, 1, 2], ls: [2, 3, 3], ms_combs: [0, 0, 0, 0, 1, -1, 0, 3, -3, 1, -3, 2, 1, -2, 1, 1, -1, 0, 1, 0, -1, 1, 1, -2, 1, 2, -3, 2, -3, 1, 2, -2, 0, 2, -1, -1, 2, 0, -2, 2, 1, -3], ctildes: [0.0068857172496481333, -0.094618964002105177, -0.010328575874472197, 0.14192844600315774, 0.017214293124120331, -0.23654741000526291, -0.017214293124120331, 0.23654741000526291, 0.013334134117290179, -0.18322883590777619, -0.004868937360559376, 0.066905711074734364, -0.0048689373605593795, 0.06690571107473442, 0.01333413411729018, -0.18322883590777622, -0.017214293124120331, 0.23654741000526291, 0.010887274916399172, -0.14960571804606662, -0.015396931844056118, 0.21157443546931273, 0.0168664937747183, -0.2317681817959274, -0.015396931844056114, 0.2115744354693127, 0.010887274916399171, -0.14960571804606659]} + - {mu0: 0, rank: 3, ndensity: 2, num_ms_combs: 13, mus: [0, 0, 0], ns: [1, 1, 2], ls: [2, 4, 2], ms_combs: [0, 0, 0, 0, 1, -1, 0, 2, -2, 1, -3, 2, 1, -2, 1, 1, -1, 0, 1, 0, -1, 1, 1, -2, 2, -4, 2, 2, -3, 1, 2, -2, 0, 2, -1, -1, 2, 0, -2], ctildes: [0.0019181320898100042, 0.29044345622908696, -0.0035020140462115493, -0.53027477552145108, 0.0024762978798867257, 0.3749608896633923, -0.0037826074926131958, -0.57276221984349351, 0.0040437775045721767, 0.61230856878355633, -0.0035020140462115488, -0.53027477552145108, 0.0025575094530800066, 0.38725794163878274, -0.0014296912475463006, -0.2164837705827411, 0.0053494148171876676, 0.81000809931758855, -0.0037826074926131962, -0.57276221984349351, 0.0024762978798867248, 0.37496088966339214, -0.0014296912475463006, -0.2164837705827411, 0.00063937736327000143, 0.096814485409695672]} + - {mu0: 0, rank: 3, ndensity: 2, num_ms_combs: 20, mus: [0, 0, 0], ns: [1, 1, 2], ls: [2, 4, 4], ms_combs: [0, 0, 0, 0, 1, -1, 0, 2, -2, 0, 3, -3, 0, 4, -4, 1, -4, 3, 1, -3, 2, 1, -2, 1, 1, -1, 0, 1, 0, -1, 1, 1, -2, 1, 2, -3, 1, 3, -4, 2, -4, 2, 2, -3, 1, 2, -2, 0, 2, -1, -1, 2, 0, -2, 2, 1, -3, 2, 2, -4], ctildes: [-0.00025569532116833575, 0.14634043763272608, 0.00043468204598617058, -0.24877874397563426, -0.00020455625693466839, 0.11707235010618076, -0.00017898672481783494, 0.10243830634290822, 0.00071594689927133965, -0.40975322537163283, -0.00062002820252967858, 0.35485670245444428, 0.00058587158204997101, -0.33530806634246552, -0.00039858955876907916, 0.2281221658635714, 0.00014005009525242573, -0.080153958766622013, 0.00014005009525242562, -0.080153958766621944, -0.00039858955876907927, 0.22812216586357142, 0.0005858715820499709, -0.33530806634246546, -0.00062002820252967869, 0.35485670245444439, 0.00033141901485762024, -0.18967888599784491, -0.00049712852228643025, 0.28451832899676732, 0.00059418223235287258, -0.34006444669695191, -0.0006263230664794885, 0.35845940093576373, 0.00059418223235287225, -0.34006444669695168, -0.00049712852228643025, 0.28451832899676732, 0.00033141901485762029, -0.18967888599784494]} + - {mu0: 0, rank: 3, ndensity: 2, num_ms_combs: 4, mus: [0, 0, 0], ns: [1, 1, 2], ls: [3, 3, 0], ms_combs: [0, 0, 0, 1, -1, 0, 2, -2, 0, 3, -3, 0], ctildes: [0.45906226681991319, 0.13421183834036657, -0.91812453363982638, -0.26842367668073314, 0.91812453363982627, 0.26842367668073308, -0.91812453363982627, -0.26842367668073308]} + - {mu0: 0, rank: 3, ndensity: 2, num_ms_combs: 12, mus: [0, 0, 0], ns: [1, 1, 2], ls: [3, 3, 2], ms_combs: [-3, 1, 2, -3, 2, 1, -3, 3, 0, -2, 1, 1, -2, 3, -1, -1, 1, 0, -1, 2, -1, -1, 3, -2, 0, 0, 0, 0, 1, -1, 0, 2, -2, 1, 1, -2], ctildes: [-0.051866732116976115, -0.11010539827197312, 0.082008504139725671, 0.1740919206097013, -0.082008504139725658, -0.17409192060970127, -0.063523514156107908, -0.13485102184612738, 0.082008504139725671, 0.1740919206097013, 0.049205102483835382, 0.10445515236582074, -0.063523514156107908, -0.13485102184612738, -0.051866732116976115, -0.11010539827197312, -0.032803401655890262, -0.069636768243880515, 0.046391015513732078, 0.098481262090327926, 0.14670127199159969, 0.31142509505343119, -0.080351595884499957, -0.17057454953395476]} + - {mu0: 0, rank: 3, ndensity: 2, num_ms_combs: 17, mus: [0, 0, 0], ns: [1, 1, 2], ls: [3, 3, 4], ms_combs: [-3, 1, 2, -3, 2, 1, -3, 3, 0, -2, 1, 1, -2, 2, 0, -2, 3, -1, -1, 1, 0, -1, 2, -1, -1, 3, -2, 0, 0, 0, 0, 1, -1, 0, 2, -2, 0, 3, -3, 1, 1, -2, 1, 2, -3, 1, 3, -4, 2, 2, -4], ctildes: [-0.013601370176438395, 0.099870809137640917, 0.010137862767218186, -0.074439306066557409, -0.0055527361224971495, 0.040772087097684685, 0.010470339643619271, -0.076880584719736952, -0.01295638428582668, 0.095134869894597573, 0.010137862767218186, -0.074439306066557409, -0.0018509120408323832, 0.013590695699228229, 0.010470339643619271, -0.076880584719736952, -0.013601370176438395, 0.099870809137640917, 0.0055527361224971495, -0.040772087097684685, -0.014337103018877197, 0.10527307621296733, -0.0064117473901253762, 0.047079550922542242, 0.029382317752185069, -0.21574560578707772, 0.011706195595321812, -0.08595510679163175, -0.013850957419698625, 0.10170345390882819, -0.023990561984391149, 0.17615554947533005, 0.015485841171950612, -0.11370791824332832]} + - {mu0: 0, rank: 3, ndensity: 2, num_ms_combs: 11, mus: [0, 0, 0], ns: [1, 1, 2], ls: [3, 4, 1], ms_combs: [0, 0, 0, 0, 1, -1, 1, -2, 1, 1, -1, 0, 1, 0, -1, 2, -3, 1, 2, -2, 0, 2, -1, -1, 3, -4, 1, 3, -3, 0, 3, -2, -1], ctildes: [-0.076417829476500332, 0.72915557132463715, 0.12082719749604683, -1.1528961869936054, -0.14798249045790207, 1.4120036922673374, 0.14798249045790204, -1.4120036922673371, -0.093592344734220789, 0.89302954642645338, 0.17509524401014526, -1.6707052994969613, -0.13235956325743323, 1.2629344961561844, 0.066179781628716586, -0.63146724807809196, -0.20218257252616115, 1.9291643088022099, 0.10109128626308055, -0.96458215440110462, -0.03820891473825018, 0.36457778566231863]} + - {mu0: 0, rank: 3, ndensity: 2, num_ms_combs: 22, mus: [0, 0, 0], ns: [1, 1, 2], ls: [3, 4, 3], ms_combs: [0, 0, 0, 0, 1, -1, 0, 2, -2, 0, 3, -3, 1, -4, 3, 1, -3, 2, 1, -2, 1, 1, -1, 0, 1, 0, -1, 1, 1, -2, 1, 2, -3, 2, -4, 2, 2, -3, 1, 2, -2, 0, 2, -1, -1, 2, 0, -2, 2, 1, -3, 3, -4, 1, 3, -3, 0, 3, -2, -1, 3, -1, -2, 3, 0, -3], ctildes: [0.018408730733763382, -0.17365214262673853, -0.023765569185560734, 0.22418395214219777, -0.010628285644577622, 0.10025811129090276, 0.048704923473889494, -0.45944038402386883, -0.039767403490777314, 0.3751315026956103, -0.022959721110372745, 0.21658227406281944, 0.038809011967623333, -0.36609086084593895, -0.023765569185560744, 0.22418395214219786, -0.0061362435779211341, 0.057884047542246242, 0.034711835559683456, -0.32744162031717428, -0.045091997110010849, 0.42535914217651755, 0.051339497147230399, -0.48429268752595367, -0.022959721110372748, 0.21658227406281946, -0.01062828564457762, 0.10025811129090274, 0.034711835559683463, -0.32744162031717433, -0.04295370504544789, 0.40518833279572325, 0.033609590259736109, -0.31704398558589703, -0.039767403490777314, 0.3751315026956103, 0.048704923473889487, -0.45944038402386878, -0.045091997110010856, 0.4253591421765176, 0.033609590259736102, -0.31704398558589697, -0.018408730733763382, 0.17365214262673853]} + - {mu0: 0, rank: 3, ndensity: 2, num_ms_combs: 5, mus: [0, 0, 0], ns: [1, 1, 2], ls: [4, 4, 0], ms_combs: [0, 0, 0, 1, -1, 0, 2, -2, 0, 3, -3, 0, 4, -4, 0], ctildes: [0.14167759407239719, -0.27761099528356287, -0.28335518814479421, 0.55522199056712551, 0.28335518814479421, -0.55522199056712551, -0.28335518814479421, 0.55522199056712551, 0.28335518814479427, -0.55522199056712562]} + - {mu0: 0, rank: 3, ndensity: 2, num_ms_combs: 18, mus: [0, 0, 0], ns: [1, 1, 2], ls: [4, 4, 2], ms_combs: [-4, 2, 2, -4, 3, 1, -4, 4, 0, -3, 1, 2, -3, 2, 1, -3, 3, 0, -3, 4, -1, -2, 1, 1, -2, 2, 0, -2, 3, -1, -2, 4, -2, -1, 1, 0, -1, 2, -1, -1, 3, -2, 0, 0, 0, 0, 1, -1, 0, 2, -2, 1, 1, -2], ctildes: [0.0026156392948006313, 0.00031515760416888283, -0.0048934130442634827, -0.00058960588881823916, 0.0056504266767230934, 0.00068081823724999756, -0.0039234589422009478, -0.00047273640625332424, 0.004623840706228813, 0.00055712519762580678, -0.0014126066691807729, -0.00017020455931249934, -0.0048934130442634827, -0.00058960588881823916, -0.0031457655284550952, -0.00037903235709743936, -0.0016144076219208844, -0.00019451949635714222, 0.004623840706228813, 0.00055712519762580678, 0.0026156392948006313, 0.00031515760416888283, 0.0034306161965818776, 0.00041335392975892706, -0.0031457655284550952, -0.00037903235709743936, -0.0039234589422009478, -0.00047273640625332424, -0.0020180095274011053, -0.00024314937044642775, 0.0022106186788358496, 0.00026635679007337688, 0.0093788607505348558, 0.0011300561548557991, -0.0049430936382077358, -0.000595591888872712]} + - {mu0: 0, rank: 3, ndensity: 2, num_ms_combs: 25, mus: [0, 0, 0], ns: [1, 1, 2], ls: [4, 4, 4], ms_combs: [-4, 1, 3, -4, 2, 2, -4, 3, 1, -4, 4, 0, -3, 1, 2, -3, 2, 1, -3, 3, 0, -3, 4, -1, -2, 1, 1, -2, 2, 0, -2, 3, -1, -2, 4, -2, -1, 1, 0, -1, 2, -1, -1, 3, -2, -1, 4, -3, 0, 0, 0, 0, 1, -1, 0, 2, -2, 0, 3, -3, 0, 4, -4, 1, 1, -2, 1, 2, -3, 1, 3, -4, 2, 2, -4], ctildes: [-0.01468152892375137, 0.13908766064516764, 0.016647289027906235, -0.15771058307351105, -0.014681528923751361, 0.13908766064516759, 0.009285414186538974, -0.087966760412658884, -0.005549096342635413, 0.052570194357837025, -0.005549096342635413, 0.052570194357837025, 0.013928121279808464, -0.13195014061898835, -0.014681528923751361, 0.13908766064516759, 0.012584167648929745, -0.11921799483871515, -0.0072956825751377694, 0.069116740324232012, -0.005549096342635413, 0.052570194357837025, 0.016647289027906235, -0.15771058307351105, -0.0059691948342036278, 0.056550060265280727, 0.012584167648929745, -0.11921799483871515, -0.005549096342635413, 0.052570194357837025, -0.01468152892375137, 0.13908766064516764, 0.0059691948342036278, -0.056550060265280727, -0.011938389668407252, 0.11310012053056141, -0.014591365150275523, 0.13823348064846389, 0.027856242559616915, -0.2639002812379766, 0.018570828373077948, -0.17593352082531777, 0.012584167648929744, -0.11921799483871512, -0.011098192685270826, 0.10514038871567405, -0.029363057847502743, 0.27817532129033534, 0.016647289027906239, -0.15771058307351107]} + - {mu0: 0, rank: 3, ndensity: 2, num_ms_combs: 1, mus: [0, 0, 0], ns: [1, 1, 3], ls: [0, 0, 0], ms_combs: [0, 0, 0], ctildes: [-0.12248992877744602, -0.22714604374939074]} + - {mu0: 0, rank: 3, ndensity: 2, num_ms_combs: 2, mus: [0, 0, 0], ns: [1, 1, 3], ls: [0, 1, 1], ms_combs: [0, 0, 0, 0, 1, -1], ctildes: [-0.18720345535904734, -0.18761915161343976, 0.37440691071809462, 0.37523830322687945]} + - {mu0: 0, rank: 3, ndensity: 2, num_ms_combs: 3, mus: [0, 0, 0], ns: [1, 1, 3], ls: [0, 2, 2], ms_combs: [0, 0, 0, 0, 1, -1, 0, 2, -2], ctildes: [0.01359427877662295, 1.3504151553478212, -0.027188557553245903, -2.7008303106956428, 0.02718855755324591, 2.7008303106956433]} + - {mu0: 0, rank: 3, ndensity: 2, num_ms_combs: 4, mus: [0, 0, 0], ns: [1, 1, 3], ls: [0, 3, 3], ms_combs: [0, 0, 0, 0, 1, -1, 0, 2, -2, 0, 3, -3], ctildes: [-0.0098804793294207591, 0.16789689607614258, 0.019760958658841518, -0.33579379215228516, -0.019760958658841515, 0.33579379215228511, 0.019760958658841515, -0.33579379215228511]} + - {mu0: 0, rank: 3, ndensity: 2, num_ms_combs: 5, mus: [0, 0, 0], ns: [1, 1, 3], ls: [0, 4, 4], ms_combs: [0, 0, 0, 0, 1, -1, 0, 2, -2, 0, 3, -3, 0, 4, -4], ctildes: [0.0033483500029951501, -0.069674370049927006, -0.0066967000059902967, 0.13934874009985393, 0.0066967000059902967, -0.13934874009985393, -0.0066967000059902967, 0.13934874009985393, 0.0066967000059902984, -0.13934874009985396]} + - {mu0: 0, rank: 3, ndensity: 2, num_ms_combs: 2, mus: [0, 0, 0], ns: [1, 1, 3], ls: [1, 1, 0], ms_combs: [0, 0, 0, 1, -1, 0], ctildes: [0.12849159217097461, 1.5126735214895837, -0.25698318434194917, -3.0253470429791669]} + - {mu0: 0, rank: 3, ndensity: 2, num_ms_combs: 4, mus: [0, 0, 0], ns: [1, 1, 3], ls: [1, 1, 2], ms_combs: [-1, 1, 0, 0, 0, 0, 0, 1, -1, 1, 1, -2], ctildes: [0.022111920264818615, -1.1191896921104503, 0.022111920264818612, -1.1191896921104501, -0.076597938703155427, 3.8769868200853375, 0.054162921881912705, -2.7414436710532115]} + - {mu0: 0, rank: 3, ndensity: 2, num_ms_combs: 5, mus: [0, 0, 0], ns: [1, 1, 3], ls: [1, 2, 1], ms_combs: [0, 0, 0, 0, 1, -1, 1, -2, 1, 1, -1, 0, 1, 0, -1], ctildes: [0.29055464897742167, -0.81804470166643894, -0.50325541440423494, 1.4168949861487969, 0.71171063238816124, -2.0037921058700672, -0.50325541440423494, 1.4168949861487969, 0.29055464897742167, -0.81804470166643883]} + - {mu0: 0, rank: 3, ndensity: 2, num_ms_combs: 8, mus: [0, 0, 0], ns: [1, 1, 3], ls: [1, 2, 3], ms_combs: [0, 0, 0, 0, 1, -1, 0, 2, -2, 1, -2, 1, 1, -1, 0, 1, 0, -1, 1, 1, -2, 1, 2, -3], ctildes: [-0.0088326252482506893, -0.046154811064235822, 0.016654957889913533, 0.087030346367746772, -0.013166900316579661, -0.068803530018860443, 0.0058884168321671256, 0.030769874042823876, -0.010199037129457239, -0.05329498518466573, 0.01442361663162519, 0.07537049085462745, -0.018620809002121552, -0.097302885291816812, 0.022805740326510705, 0.11917120973275673]} + - {mu0: 0, rank: 3, ndensity: 2, num_ms_combs: 8, mus: [0, 0, 0], ns: [1, 1, 3], ls: [1, 3, 2], ms_combs: [0, 0, 0, 0, 1, -1, 0, 2, -2, 1, -3, 2, 1, -2, 1, 1, -1, 0, 1, 0, -1, 1, 1, -2], ctildes: [-0.36040895456826522, 0.4578916950848213, 0.67959364206819928, -0.86340886038261389, -0.53726594807618322, 0.68258463769484501, 0.93057191924461158, -1.1822712729534657, -0.75980879037057714, 0.96532045209557482, 0.5885453582814496, -0.74773400694391523, -0.41616441387667918, 0.52872778683383148, 0.24027263637884358, -0.3052611300565477]} + - {mu0: 0, rank: 3, ndensity: 2, num_ms_combs: 11, mus: [0, 0, 0], ns: [1, 1, 3], ls: [1, 3, 4], ms_combs: [0, 0, 0, 0, 1, -1, 0, 2, -2, 0, 3, -3, 1, -3, 2, 1, -2, 1, 1, -1, 0, 1, 0, -1, 1, 1, -2, 1, 2, -3, 1, 3, -4], ctildes: [-0.015073494390291745, -0.12738850796657666, 0.029189696371375411, 0.24668678492638102, -0.026108058131589749, -0.22064336809850338, 0.019940358772719557, 0.16851915598356604, -0.0075367471951458672, -0.06369425398328829, 0.013054029065794875, 0.11032168404925169, -0.018461184948459702, -0.15601842180629133, 0.023833287285546473, 0.20141891645244339, -0.029189696371375407, -0.24668678492638099, 0.034537714515502055, 0.29188374021216112, -0.039880717545439121, -0.33703831196713213]} + - {mu0: 0, rank: 3, ndensity: 2, num_ms_combs: 11, mus: [0, 0, 0], ns: [1, 1, 3], ls: [1, 4, 3], ms_combs: [0, 0, 0, 0, 1, -1, 0, 2, -2, 0, 3, -3, 1, -4, 3, 1, -3, 2, 1, -2, 1, 1, -1, 0, 1, 0, -1, 1, 1, -2, 1, 2, -3], ctildes: [-0.040672806919226145, 0.39617723593375309, 0.078762551920836324, -0.76719391845895613, -0.070447368070538641, 0.6861991014394625, 0.053805066115609743, -0.52409322069283626, -0.10761013223121949, 1.0481864413856725, 0.09319310821683871, -0.90775608614240111, -0.078762551920836324, 0.76719391845895613, 0.064309354348505363, -0.62641121133028244, -0.049813811679422546, 0.48521603787195955, 0.03522368403526932, -0.34309955071973125, -0.020336403459613069, 0.19808861796687652]} + - {mu0: 0, rank: 3, ndensity: 2, num_ms_combs: 3, mus: [0, 0, 0], ns: [1, 1, 3], ls: [2, 2, 0], ms_combs: [0, 0, 0, 1, -1, 0, 2, -2, 0], ctildes: [-0.21628725020328551, -0.090157506476657129, 0.43257450040657108, 0.18031501295331429, -0.43257450040657119, -0.18031501295331431]} + - {mu0: 0, rank: 3, ndensity: 2, num_ms_combs: 8, mus: [0, 0, 0], ns: [1, 1, 3], ls: [2, 2, 2], ms_combs: [-2, 1, 1, -2, 2, 0, -1, 1, 0, -1, 2, -1, 0, 0, 0, 0, 1, -1, 0, 2, -2, 1, 1, -2], ctildes: [0.39654759807221229, -0.68557153017490924, -0.3237797580010634, 0.55976681036920262, -0.1618898790005317, 0.27988340518460131, 0.39654759807221229, -0.68557153017490924, 0.1618898790005317, -0.27988340518460131, -0.3237797580010634, 0.55976681036920262, -0.64755951600212713, 1.1195336207384059, 0.39654759807221229, -0.68557153017490924]} + - {mu0: 0, rank: 3, ndensity: 2, num_ms_combs: 10, mus: [0, 0, 0], ns: [1, 1, 3], ls: [2, 2, 4], ms_combs: [-2, 1, 1, -2, 2, 0, -1, 1, 0, -1, 2, -1, 0, 0, 0, 0, 1, -1, 0, 2, -2, 1, 1, -2, 1, 2, -3, 2, 2, -4], ctildes: [0.021267260943035802, -0.26422303933118813, -0.0095110082327708682, 0.11816413543322746, -0.038044032931083487, 0.47265654173291, 0.021267260943035802, -0.26422303933118813, -0.028533024698312617, 0.35449240629968248, 0.104187875074119, -1.2944232492974832, -0.0736719529823264, 0.91529545730377504, -0.060152897720337689, 0.74733561142721172, 0.11253576704557947, -1.3981369054479238, -0.07957480400395886, 0.98863208686940185]} + - {mu0: 0, rank: 3, ndensity: 2, num_ms_combs: 8, mus: [0, 0, 0], ns: [1, 1, 3], ls: [2, 3, 1], ms_combs: [0, 0, 0, 0, 1, -1, 1, -2, 1, 1, -1, 0, 1, 0, -1, 2, -3, 1, 2, -2, 0, 2, -1, -1], ctildes: [-0.37511098688249417, -0.21773941658497425, 0.61255367651596282, 0.35556697834965845, -0.79080339593481663, -0.45903499520317348, 0.70731606005917247, 0.41057338133023441, -0.43314085850518819, -0.25142382155705378, 0.96853240345021885, 0.56220075616434939, -0.55918244385085913, -0.32458675791009833, 0.25007399125499602, 0.14515961105664946]} + - {mu0: 0, rank: 3, ndensity: 2, num_ms_combs: 14, mus: [0, 0, 0], ns: [1, 1, 3], ls: [2, 3, 3], ms_combs: [0, 0, 0, 0, 1, -1, 0, 3, -3, 1, -3, 2, 1, -2, 1, 1, -1, 0, 1, 0, -1, 1, 1, -2, 1, 2, -3, 2, -3, 1, 2, -2, 0, 2, -1, -1, 2, 0, -2, 2, 1, -3], ctildes: [0.005638409364188619, 0.15520838847771842, -0.0084576140462829259, -0.23281258271657757, 0.014096023410471547, 0.38802097119429602, -0.014096023410471547, -0.38802097119429602, 0.010918732783301235, 0.30055975188294726, -0.0039869574965235004, -0.10974890398963065, -0.003986957496523503, -0.10974890398963073, 0.010918732783301237, 0.30055975188294731, -0.014096023410471547, -0.38802097119429602, 0.0089151079856289311, 0.2454060097769121, -0.012607866623297121, -0.34705650731437354, 0.013811225903192642, 0.38018135557007809, -0.012607866623297119, -0.34705650731437349, 0.0089151079856289294, 0.24540600977691207]} + - {mu0: 0, rank: 3, ndensity: 2, num_ms_combs: 13, mus: [0, 0, 0], ns: [1, 1, 3], ls: [2, 4, 2], ms_combs: [0, 0, 0, 0, 1, -1, 0, 2, -2, 1, -3, 2, 1, -2, 1, 1, -1, 0, 1, 0, -1, 1, 1, -2, 2, -4, 2, 2, -3, 1, 2, -2, 0, 2, -1, -1, 2, 0, -2], ctildes: [0.047715925605148334, -0.82721324965669207, -0.08711696268726031, 1.5102778556804102, 0.061600995072537197, -1.0679277132274958, -0.094097074268167838, 1.6312865275353647, 0.10059400371694466, -1.743918653056431, -0.08711696268726031, 1.51027785568041, 0.063621234140197797, -1.1029509995422566, -0.035565351087478159, 0.6165683527069562, 0.13307335860967132, -2.3069875313570236, -0.094097074268167852, 1.631286527535365, 0.061600995072537176, -1.0679277132274954, -0.035565351087478159, 0.6165683527069562, 0.015905308535049449, -0.2757377498855641]} + - {mu0: 0, rank: 3, ndensity: 2, num_ms_combs: 20, mus: [0, 0, 0], ns: [1, 1, 3], ls: [2, 4, 4], ms_combs: [0, 0, 0, 0, 1, -1, 0, 2, -2, 0, 3, -3, 0, 4, -4, 1, -4, 3, 1, -3, 2, 1, -2, 1, 1, -1, 0, 1, 0, -1, 1, 1, -2, 1, 2, -3, 1, 3, -4, 2, -4, 2, 2, -3, 1, 2, -2, 0, 2, -1, -1, 2, 0, -2, 2, 1, -3, 2, 2, -4], ctildes: [0.0062362150103960345, 0.0081136260409385985, -0.010601565517673255, -0.013793164269595613, 0.0049889720083168233, 0.0064909008327508726, 0.0043653505072772224, 0.0056795382286570165, -0.01746140202910889, -0.022718152914628063, 0.015122017742901436, 0.019674497551127382, -0.014288963667579816, -0.018590652746582986, 0.0097212971204366401, 0.012647891282867607, -0.0034157156346462233, -0.0044440160057834062, -0.0034157156346462203, -0.0044440160057834027, 0.0097212971204366418, 0.012647891282867609, -0.014288963667579813, -0.018590652746582986, 0.01512201774290144, 0.019674497551127389, -0.0080830584843791103, -0.010516461299034517, 0.012124587726568664, 0.015774691948551772, -0.01449165412717953, -0.018854363120346003, 0.015275544701755569, 0.01987424376405757, -0.014491654127179522, -0.018854363120345993, 0.012124587726568664, 0.015774691948551772, -0.0080830584843791121, -0.010516461299034517]} + - {mu0: 0, rank: 3, ndensity: 2, num_ms_combs: 4, mus: [0, 0, 0], ns: [1, 1, 3], ls: [3, 3, 0], ms_combs: [0, 0, 0, 1, -1, 0, 2, -2, 0, 3, -3, 0], ctildes: [-0.015124376998041641, -1.3354907252695525, 0.030248753996083282, 2.670981450539105, -0.030248753996083278, -2.6709814505391045, 0.030248753996083278, 2.6709814505391045]} + - {mu0: 0, rank: 3, ndensity: 2, num_ms_combs: 12, mus: [0, 0, 0], ns: [1, 1, 3], ls: [3, 3, 2], ms_combs: [-3, 1, 2, -3, 2, 1, -3, 3, 0, -2, 1, 1, -2, 3, -1, -1, 1, 0, -1, 2, -1, -1, 3, -2, 0, 0, 0, 0, 1, -1, 0, 2, -2, 1, 1, -2], ctildes: [0.027591178854195937, 0.41903472082559001, -0.043625484254166992, -0.66255206825082835, 0.043625484254166985, 0.66255206825082824, 0.033792154797324531, 0.51321062526614758, -0.043625484254166992, -0.66255206825082835, -0.026175290552500185, -0.39753124095049686, 0.033792154797324531, 0.51321062526614758, 0.027591178854195937, 0.41903472082559001, 0.017450193701666797, 0.26502082730033133, -0.024678300598934756, -0.3747960482794665, -0.078039638674931328, -1.1852091705935464, 0.042744070481812453, 0.64916579809607389]} + - {mu0: 0, rank: 3, ndensity: 2, num_ms_combs: 17, mus: [0, 0, 0], ns: [1, 1, 3], ls: [3, 3, 4], ms_combs: [-3, 1, 2, -3, 2, 1, -3, 3, 0, -2, 1, 1, -2, 2, 0, -2, 3, -1, -1, 1, 0, -1, 2, -1, -1, 3, -2, 0, 0, 0, 0, 1, -1, 0, 2, -2, 0, 3, -3, 1, 1, -2, 1, 2, -3, 1, 3, -4, 2, 2, -4], ctildes: [0.05277155057736322, -0.44767454018737951, -0.039333591456350812, 0.333676901218314, 0.021543895308335833, -0.18276236571569376, -0.0406235585751928, 0.34462002171536726, 0.0502690890527836, -0.42644552000328534, -0.039333591456350812, 0.333676901218314, 0.0071812984361119443, -0.060920788571897921, -0.0406235585751928, 0.34462002171536726, 0.05277155057736322, -0.44767454018737951, -0.021543895308335833, 0.18276236571569376, 0.055626098494413828, -0.47189039915356729, 0.024876747511321618, -0.21103580208737724, -0.11399957851493554, 0.9670875374111253, -0.045418519231037825, 0.38529689748150764, 0.053739916680212781, -0.45588943713627039, 0.093080266084646771, -0.78962366775399839, -0.060083053401065331, 0.50969988583040904]} + - {mu0: 0, rank: 3, ndensity: 2, num_ms_combs: 11, mus: [0, 0, 0], ns: [1, 1, 3], ls: [3, 4, 1], ms_combs: [0, 0, 0, 0, 1, -1, 1, -2, 1, 1, -1, 0, 1, 0, -1, 2, -3, 1, 2, -2, 0, 2, -1, -1, 3, -4, 1, 3, -3, 0, 3, -2, -1], ctildes: [-0.0021039850361472623, -0.092683814511699811, 0.0033266924385685231, 0.14654597804476904, -0.0040743495028339799, -0.17948143503339539, 0.004074349502833979, 0.17948143503339536, -0.0025768448825060066, -0.11351402648421369, 0.0048208353445996161, 0.21236529784855554, -0.0036442089809717009, -0.16053307577355375, 0.0018221044904858496, 0.080266537886776845, -0.0055666211678469012, -0.24521832375879718, 0.0027833105839234497, 0.12260916187939855, -0.0010519925180736314, -0.046341907255849912]} + - {mu0: 0, rank: 3, ndensity: 2, num_ms_combs: 22, mus: [0, 0, 0], ns: [1, 1, 3], ls: [3, 4, 3], ms_combs: [0, 0, 0, 0, 1, -1, 0, 2, -2, 0, 3, -3, 1, -4, 3, 1, -3, 2, 1, -2, 1, 1, -1, 0, 1, 0, -1, 1, 1, -2, 1, 2, -3, 2, -4, 2, 2, -3, 1, 2, -2, 0, 2, -1, -1, 2, 0, -2, 2, 1, -3, 3, -4, 1, 3, -3, 0, 3, -2, -1, 3, -1, -2, 3, 0, -3], ctildes: [0.0010447944050088393, 0.0029812097081844359, -0.0013488237769366403, -0.0038487251837833972, -0.00060321233097966815, -0.0017212022275310092, 0.002764266166845086, 0.0078875394939874575, -0.0022570138740032036, -0.0064401490287731653, -0.0013030875677204697, -0.0037182217753834934, 0.0022026200042855777, 0.0062849419069791572, -0.001348823776936641, -0.0038487251837833985, -0.00034826480166961354, -0.00099373656939481304, 0.0019700832232733721, 0.0056214229354570218, -0.0025592131783864058, -0.0073024426012834077, 0.0029137923820578299, 0.0083141966451774438, -0.0013030875677204699, -0.0037182217753834942, -0.00060321233097966804, -0.001721202227531009, 0.0019700832232733721, 0.0056214229354570227, -0.0024378536116872918, -0.0069561559857636837, 0.0019075248785950995, 0.0054429193527533639, -0.0022570138740032036, -0.0064401490287731653, 0.0027642661668450855, 0.0078875394939874575, -0.0025592131783864062, -0.0073024426012834086, 0.0019075248785950991, 0.005442919352753363, -0.0010447944050088393, -0.0029812097081844359]} + - {mu0: 0, rank: 3, ndensity: 2, num_ms_combs: 5, mus: [0, 0, 0], ns: [1, 1, 3], ls: [4, 4, 0], ms_combs: [0, 0, 0, 1, -1, 0, 2, -2, 0, 3, -3, 0, 4, -4, 0], ctildes: [-0.043383193891576619, 0.020554637525101317, 0.086766387783153195, -0.041109275050202614, -0.086766387783153195, 0.041109275050202614, 0.086766387783153195, -0.041109275050202614, -0.086766387783153209, 0.041109275050202621]} + - {mu0: 0, rank: 3, ndensity: 2, num_ms_combs: 18, mus: [0, 0, 0], ns: [1, 1, 3], ls: [4, 4, 2], ms_combs: [-4, 2, 2, -4, 3, 1, -4, 4, 0, -3, 1, 2, -3, 2, 1, -3, 3, 0, -3, 4, -1, -2, 1, 1, -2, 2, 0, -2, 3, -1, -2, 4, -2, -1, 1, 0, -1, 2, -1, -1, 3, -2, 0, 0, 0, 0, 1, -1, 0, 2, -2, 1, 1, -2], ctildes: [-0.005483853539698995, -0.015292787372204924, 0.010259350552400586, 0.028610185417786906, -0.011846477606278423, -0.033036196505048736, 0.008225780309548493, 0.022939181058307386, -0.009694175062387525, -0.027034084135325254, 0.0029616194015696044, 0.0082590491262621805, 0.010259350552400586, 0.028610185417786906, 0.0065952967836860887, 0.018392262054291578, 0.0033847078875081215, 0.0094389132871567846, -0.009694175062387525, -0.027034084135325254, -0.005483853539698995, -0.015292787372204924, -0.0071925042609547549, -0.020057690735208158, 0.0065952967836860887, 0.018392262054291578, 0.008225780309548493, 0.022939181058307386, 0.0042308848593851514, 0.011798641608945979, -0.0046347021513846402, -0.012924764314277517, -0.01966337592014376, -0.054835130951181192, 0.010363509065960577, 0.028900651599887989]} + - {mu0: 0, rank: 3, ndensity: 2, num_ms_combs: 25, mus: [0, 0, 0], ns: [1, 1, 3], ls: [4, 4, 4], ms_combs: [-4, 1, 3, -4, 2, 2, -4, 3, 1, -4, 4, 0, -3, 1, 2, -3, 2, 1, -3, 3, 0, -3, 4, -1, -2, 1, 1, -2, 2, 0, -2, 3, -1, -2, 4, -2, -1, 1, 0, -1, 2, -1, -1, 3, -2, -1, 4, -3, 0, 0, 0, 0, 1, -1, 0, 2, -2, 0, 3, -3, 0, 4, -4, 1, 1, -2, 1, 2, -3, 1, 3, -4, 2, 2, -4], ctildes: [-0.0091111179070086888, 0.10343947150568977, 0.010331036634742415, -0.11728933600800301, -0.0091111179070086836, 0.10343947150568972, 0.0057623769232987322, -0.065420865984413296, -0.0034436788782474721, 0.039096445336001012, -0.0034436788782474721, 0.039096445336001012, 0.0086435653849481001, -0.098131298976619971, -0.0091111179070086836, 0.10343947150568972, 0.0078095296345788764, -0.088662404147734103, -0.0045275818683061485, 0.051402108987753325, -0.0034436788782474721, 0.039096445336001012, 0.010331036634742415, -0.11728933600800301, -0.0037043851649777573, 0.042056270989979987, 0.0078095296345788764, -0.088662404147734103, -0.0034436788782474721, 0.039096445336001012, -0.0091111179070086888, 0.10343947150568977, 0.0037043851649777573, -0.042056270989979987, -0.007408770329955512, 0.084112541979959946, -0.0090551637366122883, 0.10280421797550655, 0.017287130769896193, -0.19626259795323983, 0.011524753846597464, -0.13084173196882659, 0.0078095296345788746, -0.088662404147734075, -0.0068873577564949441, 0.078192890672002024, -0.018222235814017378, 0.20687894301137957, 0.010331036634742417, -0.11728933600800304]} + - {mu0: 0, rank: 3, ndensity: 2, num_ms_combs: 1, mus: [0, 0, 0], ns: [1, 2, 2], ls: [0, 0, 0], ms_combs: [0, 0, 0], ctildes: [0.7423877235499704, 0.55584236760649253]} + - {mu0: 0, rank: 3, ndensity: 2, num_ms_combs: 2, mus: [0, 0, 0], ns: [1, 2, 2], ls: [0, 1, 1], ms_combs: [0, 0, 0, 0, 1, -1], ctildes: [-0.15495508916452846, 0.36947944167003505, 0.30991017832905687, -0.73895888334006998]} + - {mu0: 0, rank: 3, ndensity: 2, num_ms_combs: 3, mus: [0, 0, 0], ns: [1, 2, 2], ls: [0, 2, 2], ms_combs: [0, 0, 0, 0, 1, -1, 0, 2, -2], ctildes: [-0.40912862138485789, 0.30520012327494744, 0.8182572427697159, -0.61040024654989489, -0.81825724276971612, 0.61040024654989511]} + - {mu0: 0, rank: 3, ndensity: 2, num_ms_combs: 4, mus: [0, 0, 0], ns: [1, 2, 2], ls: [0, 3, 3], ms_combs: [0, 0, 0, 0, 1, -1, 0, 2, -2, 0, 3, -3], ctildes: [0.11639484388052569, -0.67693330729249157, -0.23278968776105138, 1.3538666145849831, 0.23278968776105136, -1.3538666145849829, -0.23278968776105136, 1.3538666145849829]} + - {mu0: 0, rank: 3, ndensity: 2, num_ms_combs: 5, mus: [0, 0, 0], ns: [1, 2, 2], ls: [0, 4, 4], ms_combs: [0, 0, 0, 0, 1, -1, 0, 2, -2, 0, 3, -3, 0, 4, -4], ctildes: [0.0021273603486274858, -0.084864327782664339, -0.0042547206972549691, 0.1697286555653286, 0.0042547206972549691, -0.1697286555653286, -0.0042547206972549691, 0.1697286555653286, 0.0042547206972549699, -0.16972865556532862]} + - {mu0: 0, rank: 3, ndensity: 2, num_ms_combs: 2, mus: [0, 0, 0], ns: [1, 2, 2], ls: [1, 0, 1], ms_combs: [0, 0, 0, 1, 0, -1], ctildes: [1.1434129941491276, -0.24666486202327567, -2.2868259882982547, 0.49332972404655123]} + - {mu0: 0, rank: 3, ndensity: 2, num_ms_combs: 5, mus: [0, 0, 0], ns: [1, 2, 2], ls: [1, 1, 2], ms_combs: [0, 0, 0, 0, 1, -1, 1, -1, 0, 1, 0, -1, 1, 1, -2], ctildes: [-0.033315580047004023, -0.21184888199957155, 0.057704277325038911, 0.36693302714992182, -0.03331558004700403, -0.21184888199957158, 0.057704277325038911, 0.36693302714992182, -0.081606171600008298, -0.51892166347803459]} + - {mu0: 0, rank: 3, ndensity: 2, num_ms_combs: 8, mus: [0, 0, 0], ns: [1, 2, 2], ls: [1, 2, 3], ms_combs: [0, 0, 0, 0, 1, -1, 0, 2, -2, 1, -2, 1, 1, -1, 0, 1, 0, -1, 1, 1, -2, 1, 2, -3], ctildes: [-0.15803664756182675, -0.800145744692352, 0.29799676044521639, 1.5087692853587253, -0.23558712458961392, -1.1927868513595266, 0.10535776504121783, 0.53343049646156793, -0.18248500202329335, -0.92392872217812583, 0.2580727647910232, 1.3066325295703491, -0.33317050671511206, -1.6868553422129433, 0.40804886939827051, 2.0659674291548065]} + - {mu0: 0, rank: 3, ndensity: 2, num_ms_combs: 11, mus: [0, 0, 0], ns: [1, 2, 2], ls: [1, 3, 4], ms_combs: [0, 0, 0, 0, 1, -1, 0, 2, -2, 0, 3, -3, 1, -3, 2, 1, -2, 1, 1, -1, 0, 1, 0, -1, 1, 1, -2, 1, 2, -3, 1, 3, -4], ctildes: [-0.055418948086233558, 0.42010965376556181, 0.10731833150115794, -0.81353884630749207, -0.095988433787378494, 0.72765126507212252, 0.073312377278486426, -0.55575283362056294, -0.027709474043116761, 0.21005482688278077, 0.047994216893689247, -0.36382563253606126, -0.067874072446531264, 0.51452714387146792, 0.087625050741563723, -0.66425168646195398, -0.10731833150115792, 0.81353884630749196, 0.12698076226999661, -0.96259214428118789, -0.14662475455697288, 1.1115056672411261]} + - {mu0: 0, rank: 3, ndensity: 2, num_ms_combs: 3, mus: [0, 0, 0], ns: [1, 2, 2], ls: [2, 0, 2], ms_combs: [0, 0, 0, 1, 0, -1, 2, 0, -2], ctildes: [0.59828595972685794, -0.37480987548551037, -1.1965719194537159, 0.74961975097102085, 1.1965719194537163, -0.74961975097102096]} + - {mu0: 0, rank: 3, ndensity: 2, num_ms_combs: 4, mus: [0, 0, 0], ns: [1, 2, 2], ls: [2, 1, 1], ms_combs: [0, -1, 1, 0, 0, 0, 1, -1, 0, 2, -1, -1], ctildes: [-0.48431979958528193, -0.062884861791553281, -0.48431979958528204, -0.062884861791553281, 1.6777329999865691, 0.21783955129983423, -1.1863363813109526, -0.15403582393474755]} + - {mu0: 0, rank: 3, ndensity: 2, num_ms_combs: 8, mus: [0, 0, 0], ns: [1, 2, 2], ls: [2, 1, 3], ms_combs: [0, 0, 0, 0, 1, -1, 1, -1, 0, 1, 0, -1, 1, 1, -2, 2, -1, -1, 2, 0, -2, 2, 1, -3], ctildes: [0.92704095670412145, 0.57699256573233682, -1.5138515430577664, -0.94222491428233845, 1.0704546918058653, 0.66625362629195495, -1.7480451917950637, -1.0879876157961603, 1.9543739382977092, 1.2164071337990687, -0.61802730446941423, -0.38466171048822451, 1.3819510647445694, 0.86012973299301365, -2.3936094577115012, -1.4897883986445521]} + - {mu0: 0, rank: 3, ndensity: 2, num_ms_combs: 7, mus: [0, 0, 0], ns: [1, 2, 2], ls: [2, 2, 2], ms_combs: [0, -2, 2, 0, -1, 1, 0, 0, 0, 1, -2, 1, 1, -1, 0, 2, -2, 0, 2, -1, -1], ctildes: [0.093851393332326088, -0.12103798133085801, 0.046925696666163016, -0.060518990665428971, -0.046925696666163016, 0.060518990665428971, -0.22988802531344221, 0.2964812937571184, 0.093851393332326033, -0.12103798133085794, 0.18770278666465209, -0.24207596266171594, -0.11494401265672111, 0.1482406468785592]} + - {mu0: 0, rank: 3, ndensity: 2, num_ms_combs: 13, mus: [0, 0, 0], ns: [1, 2, 2], ls: [2, 2, 4], ms_combs: [0, 0, 0, 0, 1, -1, 0, 2, -2, 1, -2, 1, 1, -1, 0, 1, 0, -1, 1, 1, -2, 1, 2, -3, 2, -2, 0, 2, -1, -1, 2, 0, -2, 2, 1, -3, 2, 2, -4], ctildes: [0.18512796742815335, 0.0024667306089269504, -0.33799587928493735, -0.0045036133259924813, 0.23899917825548886, 0.003184535522651384, -0.13798623990190587, -0.0018385924412467026, 0.24683728990420448, 0.0032889741452359336, -0.33799587928493735, -0.0045036133259924813, 0.39028402378028554, 0.0052003247321754887, -0.36507727512934057, -0.0048644583619419099, 0.061709322476051098, 0.00082224353630898318, -0.13798623990190587, -0.0018385924412467026, 0.23899917825548886, 0.003184535522651384, -0.36507727512934057, -0.0048644583619419099, 0.51629723380212733, 0.0068793829890574591]} + - {mu0: 0, rank: 3, ndensity: 2, num_ms_combs: 9, mus: [0, 0, 0], ns: [1, 2, 2], ls: [2, 3, 3], ms_combs: [0, -3, 3, 0, -1, 1, 0, 0, 0, 1, -3, 2, 1, -2, 1, 1, -1, 0, 2, -3, 1, 2, -2, 0, 2, -1, -1], ctildes: [0.21998203559490651, 0.11353968690387535, -0.13198922135694388, -0.068123812142325199, 0.08799281423796261, 0.045415874761550144, -0.43996407118981301, -0.2270793738077507, 0.3407947041295521, 0.17589492660492545, -0.12444063128670309, -0.064227746034822161, 0.27825771072005517, 0.14361760617545491, -0.3935158283351915, -0.20310596644888634, 0.21553749591451499, 0.11124571938794249]} + - {mu0: 0, rank: 3, ndensity: 2, num_ms_combs: 13, mus: [0, 0, 0], ns: [1, 2, 2], ls: [2, 4, 4], ms_combs: [0, -4, 4, 0, -3, 3, 0, -2, 2, 0, -1, 1, 0, 0, 0, 1, -4, 3, 1, -3, 2, 1, -2, 1, 1, -1, 0, 2, -4, 2, 2, -3, 1, 2, -2, 0, 2, -1, -1], ctildes: [-0.048519120650054544, 0.77811790981451845, 0.012129780162513638, -0.19452947745362961, 0.013862605900015579, -0.22231940280414803, -0.029458037537533122, 0.47242873095881482, 0.017328257375019491, -0.2778992535051853, 0.084037582104458747, -1.3477397540780431, -0.079408051082703532, 1.2734943647592325, 0.054024159924294927, -0.86640412762159946, -0.01898215489310686, 0.30442337971727312, -0.044919977121109962, 0.7203972008992866, 0.067379965681664933, -1.0805958013489296, -0.080534462678695559, 1.2915590168988662, 0.042445388700417176, -0.68071137098805312]} + - {mu0: 0, rank: 3, ndensity: 2, num_ms_combs: 4, mus: [0, 0, 0], ns: [1, 2, 2], ls: [3, 0, 3], ms_combs: [0, 0, 0, 1, 0, -1, 2, 0, -2, 3, 0, -3], ctildes: [-0.54234196513500177, 0.21579720344522221, 1.0846839302700035, -0.43159440689044443, -1.0846839302700033, 0.43159440689044437, 1.0846839302700033, -0.43159440689044437]} + - {mu0: 0, rank: 3, ndensity: 2, num_ms_combs: 8, mus: [0, 0, 0], ns: [1, 2, 2], ls: [3, 1, 2], ms_combs: [0, 0, 0, 0, 1, -1, 1, -1, 0, 1, 0, -1, 1, 1, -2, 2, -1, -1, 2, 0, -2, 3, -1, -2], ctildes: [-0.042134369738408568, 0.4901205577716235, -0.048652579421210838, 0.56594247192963298, 0.068805137661910515, -0.80036351932584149, 0.07944932950146659, -0.92418018666466162, 0.02808957982560573, -0.32674703851441583, -0.088827050766029383, 1.0332648604203141, -0.062810209949461113, 0.73062858956497578, 0.10879047486653483, -1.265485838588926]} + - {mu0: 0, rank: 3, ndensity: 2, num_ms_combs: 11, mus: [0, 0, 0], ns: [1, 2, 2], ls: [3, 1, 4], ms_combs: [0, 0, 0, 0, 1, -1, 1, -1, 0, 1, 0, -1, 1, 1, -2, 2, -1, -1, 2, 0, -2, 2, 1, -3, 3, -1, -2, 3, 0, -3, 3, 1, -4], ctildes: [-0.15743312940830348, 0.51353363310523847, 0.24892363404913764, -0.81196796785689973, -0.19281541782994802, 0.62894768343273066, 0.30486794416983809, -0.99445360436698871, -0.30486794416983803, 0.9944536043669886, 0.13634108946487375, -0.44473317196685375, -0.2726821789297475, 0.88946634393370749, 0.36072461620366431, -1.1766533728052171, -0.078716564704151684, 0.25676681655261907, 0.20826445426851004, -0.67934114153197322, -0.41652890853702013, 1.3586822830639467]} + - {mu0: 0, rank: 3, ndensity: 2, num_ms_combs: 14, mus: [0, 0, 0], ns: [1, 2, 2], ls: [3, 2, 3], ms_combs: [0, 0, 0, 0, 1, -1, 0, 2, -2, 1, -2, 1, 1, -1, 0, 1, 0, -1, 1, 1, -2, 1, 2, -3, 2, -2, 0, 2, -1, -1, 2, 1, -3, 3, -2, -1, 3, -1, -2, 3, 0, -3], ctildes: [-0.079354955695139534, -0.16936932664273263, 0.056112427292791228, 0.11976219939407566, 0.17744307528581607, 0.37872132767651634, -0.19437915001425785, -0.41486842835346721, 0.05611242729279118, 0.11976219939407556, 0.11903243354270929, 0.25405398996409889, -0.15367021092315145, -0.3279822907228338, -0.1254712018091956, -0.26779641898003714, 0.1774430752858161, 0.37872132767651639, -0.15367021092315142, -0.3279822907228338, 0.19838738923784885, 0.42342331660683152, -0.12547120180919563, -0.2677964189800372, 0.19838738923784885, 0.42342331660683152, -0.19838738923784885, -0.42342331660683152]} + - {mu0: 0, rank: 3, ndensity: 2, num_ms_combs: 22, mus: [0, 0, 0], ns: [1, 2, 2], ls: [3, 3, 4], ms_combs: [0, 0, 0, 0, 1, -1, 0, 2, -2, 0, 3, -3, 1, -3, 2, 1, -2, 1, 1, -1, 0, 1, 0, -1, 1, 1, -2, 1, 2, -3, 1, 3, -4, 2, -3, 1, 2, -2, 0, 2, -1, -1, 2, 0, -2, 2, 1, -3, 2, 2, -4, 3, -3, 0, 3, -2, -1, 3, -1, -2, 3, 0, -3, 3, 1, -4], ctildes: [-0.029447201660031678, -0.049852381110703795, 0.038016173873904678, 0.0643591472701803, 0.017001349805300485, 0.028782285654008651, -0.077909972399212143, -0.1318970026833361, 0.072130618419915318, 0.12211289618398671, -0.055526175948736378, -0.09400255131113272, 0.0098157338866772265, 0.016617460370234597, 0.038016173873904678, 0.0643591472701803, -0.062080151975994237, -0.10509804739505242, 0.03672711320369311, 0.062176843343711471, 0.063613226084130253, 0.10769345172555908, -0.05376298868200971, -0.091017578932256316, 0.068710137206740574, 0.11632222259164215, -0.055526175948736378, -0.09400255131113272, 0.017001349805300485, 0.028782285654008651, 0.03672711320369311, 0.062176843343711471, -0.082124321740787903, -0.13903164834289422, 0.029447201660031678, 0.049852381110703795, -0.05376298868200971, -0.091017578932256316, 0.072130618419915318, 0.12211289618398671, -0.077909972399212143, -0.1318970026833361, 0.063613226084130253, 0.10769345172555908]} + - {mu0: 0, rank: 3, ndensity: 2, num_ms_combs: 5, mus: [0, 0, 0], ns: [1, 2, 2], ls: [4, 0, 4], ms_combs: [0, 0, 0, 1, 0, -1, 2, 0, -2, 3, 0, -3, 4, 0, -4], ctildes: [-0.27123727774769962, -0.23882357388242462, 0.54247455549539902, 0.47764714776484901, -0.54247455549539902, -0.47764714776484901, 0.54247455549539902, 0.47764714776484901, -0.54247455549539914, -0.47764714776484907]} + - {mu0: 0, rank: 3, ndensity: 2, num_ms_combs: 11, mus: [0, 0, 0], ns: [1, 2, 2], ls: [4, 1, 3], ms_combs: [0, 0, 0, 0, 1, -1, 1, -1, 0, 1, 0, -1, 1, 1, -2, 2, -1, -1, 2, 0, -2, 2, 1, -3, 3, -1, -2, 3, 0, -3, 4, -1, -3], ctildes: [0.094400086854595541, 0.15242371690448128, 0.11561602223408643, 0.18668016555720685, -0.14925964288912111, -0.24100305742343528, -0.18280498213419111, -0.29516725856904491, -0.081752873335537171, -0.13200281097852834, 0.18280498213419111, 0.29516725856904491, 0.16350574667107434, 0.26400562195705668, 0.047200043427297764, 0.076211858452240613, -0.2162977718107949, -0.34924661021065267, -0.12487957678007866, -0.2016376244186846, 0.24975915356015732, 0.4032752488373692]} + - {mu0: 0, rank: 3, ndensity: 2, num_ms_combs: 9, mus: [0, 0, 0], ns: [1, 2, 2], ls: [4, 2, 2], ms_combs: [0, -2, 2, 0, -1, 1, 0, 0, 0, 1, -2, 1, 1, -1, 0, 2, -2, 0, 2, -1, -1, 3, -2, -1, 4, -2, -2], ctildes: [0.02319225983729575, -0.13942537777055669, 0.092769039349183013, -0.55770151108222688, 0.069576779511887232, -0.41827613331167002, -0.10371893909606301, 0.62352924496710571, -0.25405847744815951, 1.527328489872265, 0.17964647222152311, -1.0799843322880875, 0.14668073034460138, -0.88180351476873686, -0.27441451909563724, 1.6497033173588347, 0.19404036730857038, -1.1665164026503747]} + - {mu0: 0, rank: 3, ndensity: 2, num_ms_combs: 20, mus: [0, 0, 0], ns: [1, 2, 2], ls: [4, 2, 4], ms_combs: [0, 0, 0, 0, 1, -1, 0, 2, -2, 1, -2, 1, 1, -1, 0, 1, 0, -1, 1, 1, -2, 1, 2, -3, 2, -2, 0, 2, -1, -1, 2, 0, -2, 2, 1, -3, 2, 2, -4, 3, -2, -1, 3, -1, -2, 3, 0, -3, 3, 1, -4, 4, -2, -2, 4, -1, -3, 4, 0, -4], ctildes: [0.00448346524911147, 0.084481063990562558, -0.0024556950527288698, -0.046272184429668496, -0.010418631745865039, -0.19631625234319899, 0.010982202139823366, 0.2069354997042922, -0.002455695052728872, -0.046272184429668538, -0.0076218909234894968, -0.1436178087839563, 0.0069890306448874649, 0.13169294559821865, 0.008716852712944206, 0.16424996089453114, -0.010418631745865044, -0.1963162523431991, 0.0069890306448874631, 0.13169294559821862, 0.0035867721992891731, 0.067584851192449988, -0.010272909439878668, -0.19357043526358036, -0.0058112351419628058, -0.10949997392968747, 0.008716852712944206, 0.16424996089453114, -0.01027290943987867, -0.19357043526358039, 0.0031384256743780281, 0.059136744793393772, 0.01087182544760272, 0.20485569315278451, -0.0058112351419628049, -0.10949997392968745, 0.010871825447602719, 0.20485569315278446, -0.01255370269751211, -0.23654697917357506]} + - {mu0: 0, rank: 3, ndensity: 2, num_ms_combs: 14, mus: [0, 0, 0], ns: [1, 2, 2], ls: [4, 3, 3], ms_combs: [0, -3, 3, 0, -2, 2, 0, -1, 1, 0, 0, 0, 1, -3, 2, 1, -2, 1, 1, -1, 0, 2, -3, 1, 2, -2, 0, 2, -1, -1, 3, -3, 0, 3, -2, -1, 4, -3, -1, 4, -2, -2], ctildes: [0.039963883092777673, -0.28605861736497923, 0.093249060549814577, -0.66747010718495148, 0.01332129436425924, -0.095352872454993173, -0.039963883092777673, 0.28605861736497923, -0.14592746836942433, 1.044538383330254, -0.15071324126639737, 1.0787946034966649, 0.10318630244540539, -0.73860017406245615, 0.19578224343509001, -1.4013952981405093, 0.046146317322955835, -0.33031203947936572, -0.084251263145247765, 0.60306451679461881, -0.21146899217589712, 1.5136799238694356, 0.099687438918841215, -0.71355555914267665, 0.17266370908385206, -1.235914482458335, -0.11145394496269352, 0.79777936798294813]} + - {mu0: 0, rank: 3, ndensity: 2, num_ms_combs: 19, mus: [0, 0, 0], ns: [1, 2, 2], ls: [4, 4, 4], ms_combs: [0, -4, 4, 0, -3, 3, 0, -2, 2, 0, -1, 1, 0, 0, 0, 1, -4, 3, 1, -3, 2, 1, -2, 1, 1, -1, 0, 2, -4, 2, 2, -3, 1, 2, -2, 0, 2, -1, -1, 3, -4, 1, 3, -3, 0, 3, -2, -1, 4, -4, 0, 4, -3, -1, 4, -2, -2], ctildes: [-0.0067542073784844356, 0.070682983742950595, -0.010131311067726651, 0.10602447561442585, 0.0053068772259520537, -0.05553663008374686, 0.0043419904575971368, -0.045439060977611086, -0.0043419904575971386, 0.045439060977611107, 0.021358679105125761, -0.22351922044437733, 0.0080728218921420503, -0.08448232436269236, -0.018307439232964941, 0.19158790323803773, 0.0086839809151942754, -0.090878121955222199, -0.024218465676426153, 0.25344697308807712, 0.0080728218921420503, -0.08448232436269236, 0.010613754451904113, -0.11107326016749378, -0.0091537196164824704, 0.095793951619018866, 0.021358679105125757, -0.2235192204443773, -0.020262622135453306, 0.21204895122885176, 0.0080728218921420503, -0.08448232436269236, -0.013508414756968871, 0.14136596748590119, 0.021358679105125754, -0.22351922044437728, -0.012109232838213075, 0.12672348654403853]} + - {mu0: 0, rank: 3, ndensity: 2, num_ms_combs: 1, mus: [0, 0, 0], ns: [1, 2, 3], ls: [0, 0, 0], ms_combs: [0, 0, 0], ctildes: [3.1368088553291966, 1.703607094729318]} + - {mu0: 0, rank: 3, ndensity: 2, num_ms_combs: 2, mus: [0, 0, 0], ns: [1, 2, 3], ls: [0, 1, 1], ms_combs: [0, 0, 0, 0, 1, -1], ctildes: [-0.37414493544370137, 0.7847289878942596, 0.74828987088740262, -1.5694579757885188]} + - {mu0: 0, rank: 3, ndensity: 2, num_ms_combs: 3, mus: [0, 0, 0], ns: [1, 2, 3], ls: [0, 2, 2], ms_combs: [0, 0, 0, 0, 1, -1, 0, 2, -2], ctildes: [0.2602802577281515, 0.1645241787517564, -0.52056051545630311, -0.3290483575035128, 0.52056051545630322, 0.32904835750351291]} + - {mu0: 0, rank: 3, ndensity: 2, num_ms_combs: 4, mus: [0, 0, 0], ns: [1, 2, 3], ls: [0, 3, 3], ms_combs: [0, 0, 0, 0, 1, -1, 0, 2, -2, 0, 3, -3], ctildes: [-0.058748485317100911, 0.31640410514552347, 0.11749697063420182, -0.63280821029104695, -0.11749697063420181, 0.63280821029104684, 0.11749697063420181, -0.63280821029104684]} + - {mu0: 0, rank: 3, ndensity: 2, num_ms_combs: 5, mus: [0, 0, 0], ns: [1, 2, 3], ls: [0, 4, 4], ms_combs: [0, 0, 0, 0, 1, -1, 0, 2, -2, 0, 3, -3, 0, 4, -4], ctildes: [0.0060124982041382374, 0.031273806580488082, -0.012024996408276468, -0.062547613160976137, 0.012024996408276468, 0.062547613160976137, -0.012024996408276468, -0.062547613160976137, 0.012024996408276471, 0.062547613160976151]} + - {mu0: 0, rank: 3, ndensity: 2, num_ms_combs: 2, mus: [0, 0, 0], ns: [1, 2, 3], ls: [1, 0, 1], ms_combs: [0, 0, 0, 1, 0, -1], ctildes: [1.7139432192127111, -0.00037525198151375564, -3.4278864384254217, 0.00075050396302751116]} + - {mu0: 0, rank: 3, ndensity: 2, num_ms_combs: 2, mus: [0, 0, 0], ns: [1, 2, 3], ls: [1, 1, 0], ms_combs: [0, 0, 0, 1, -1, 0], ctildes: [-0.33709438393905822, -0.66655944130596034, 0.67418876787811632, 1.3331188826119205]} + - {mu0: 0, rank: 3, ndensity: 2, num_ms_combs: 5, mus: [0, 0, 0], ns: [1, 2, 3], ls: [1, 1, 2], ms_combs: [0, 0, 0, 0, 1, -1, 1, -1, 0, 1, 0, -1, 1, 1, -2], ctildes: [0.23267453463359611, 0.0089513258105209919, -0.40300411561283295, -0.015504151098925025, 0.23267453463359614, 0.0089513258105209936, -0.40300411561283295, -0.015504151098925025, 0.56993388599184325, 0.021926180757181498]} + - {mu0: 0, rank: 3, ndensity: 2, num_ms_combs: 5, mus: [0, 0, 0], ns: [1, 2, 3], ls: [1, 2, 1], ms_combs: [0, 0, 0, 0, 1, -1, 1, -2, 1, 1, -1, 0, 1, 0, -1], ctildes: [0.29709433267560875, 0.52462810375342661, -0.51458247883492481, -0.90868253077945149, 0.72772952052791673, 1.2850711589198074, -0.51458247883492481, -0.90868253077945149, 0.2970943326756087, 0.52462810375342661]} + - {mu0: 0, rank: 3, ndensity: 2, num_ms_combs: 8, mus: [0, 0, 0], ns: [1, 2, 3], ls: [1, 2, 3], ms_combs: [0, 0, 0, 0, 1, -1, 0, 2, -2, 1, -2, 1, 1, -1, 0, 1, 0, -1, 1, 1, -2, 1, 2, -3], ctildes: [0.10394684045700418, -0.35886437580857783, -0.19600404205350352, 0.67668115642806126, 0.15495480087212438, -0.53496342600734059, -0.069297893638002775, 0.23924291720571852, 0.12002747263852487, -0.41438088795129879, -0.16974447966276748, 0.58602307172893264, 0.21913918094818058, -0.75655253243315668, -0.26838958798723761, 0.92658383403582756]} + - {mu0: 0, rank: 3, ndensity: 2, num_ms_combs: 8, mus: [0, 0, 0], ns: [1, 2, 3], ls: [1, 3, 2], ms_combs: [0, 0, 0, 0, 1, -1, 0, 2, -2, 1, -3, 2, 1, -2, 1, 1, -1, 0, 1, 0, -1, 1, 1, -2], ctildes: [0.20374156938315149, 0.31547249613104789, -0.38417878752110918, -0.59486064344562906, 0.30371999932214466, 0.47027863092037531, -0.52605847010073936, -0.81454648246802186, 0.42952494220532406, 0.66507441794184563, -0.33270858958838467, -0.51516442893547187, 0.23526049985695877, 0.36427626112636746, -0.13582771292210108, -0.2103149974206987]} + - {mu0: 0, rank: 3, ndensity: 2, num_ms_combs: 11, mus: [0, 0, 0], ns: [1, 2, 3], ls: [1, 3, 4], ms_combs: [0, 0, 0, 0, 1, -1, 0, 2, -2, 0, 3, -3, 1, -3, 2, 1, -2, 1, 1, -1, 0, 1, 0, -1, 1, 1, -2, 1, 2, -3, 1, 3, -4], ctildes: [0.040597327149104315, -0.24713381574223806, -0.078616385974507583, 0.47857257632718991, 0.070316633273744017, -0.42804832513392138, -0.053705215765230394, 0.32692730850421048, 0.020298663574552147, -0.12356690787111894, -0.035158316636872008, 0.21402416256696069, 0.049721368218072028, -0.30267587337773999, -0.064190010353079877, 0.39075287229692374, 0.078616385974507583, -0.47857257632718986, -0.093020162336828111, 0.56625470871103722, 0.10741043153046081, -0.65385461700842107]} + - {mu0: 0, rank: 3, ndensity: 2, num_ms_combs: 11, mus: [0, 0, 0], ns: [1, 2, 3], ls: [1, 4, 3], ms_combs: [0, 0, 0, 0, 1, -1, 0, 2, -2, 0, 3, -3, 1, -4, 3, 1, -3, 2, 1, -2, 1, 1, -1, 0, 1, 0, -1, 1, 1, -2, 1, 2, -3], ctildes: [0.056403575331714112, -0.50175416044474019, -0.10922505396314212, 0.97164275364638142, 0.097693858203067419, -0.86906369879935608, -0.074614916691306488, 0.66375836391439191, 0.14922983338261298, -1.3275167278287838, -0.12923682671186193, 1.1496632102485194, 0.10922505396314212, -0.97164275364638142, -0.08918188311255193, 0.79334298623547139, 0.069079989615665985, -0.61452083470408791, -0.048846929101533709, 0.43453184939967804, 0.028201787665857053, -0.25087708022237004]} + - {mu0: 0, rank: 3, ndensity: 2, num_ms_combs: 3, mus: [0, 0, 0], ns: [1, 2, 3], ls: [2, 0, 2], ms_combs: [0, 0, 0, 1, 0, -1, 2, 0, -2], ctildes: [-0.69675403413360282, -1.8515139258555053, 1.3935080682672059, 3.7030278517110111, -1.3935080682672063, -3.7030278517110125]} + - {mu0: 0, rank: 3, ndensity: 2, num_ms_combs: 5, mus: [0, 0, 0], ns: [1, 2, 3], ls: [2, 1, 1], ms_combs: [0, 0, 0, 0, 1, -1, 1, -1, 0, 1, 0, -1, 2, -1, -1], ctildes: [-0.74466223657437314, 0.25439835594593235, -0.74466223657437303, 0.2543983559459323, 1.2897928282246893, -0.44063087786034683, 1.2897928282246893, -0.44063087786034683, -1.8240425103269073, 0.62314616347046514]} + - {mu0: 0, rank: 3, ndensity: 2, num_ms_combs: 8, mus: [0, 0, 0], ns: [1, 2, 3], ls: [2, 1, 3], ms_combs: [0, 0, 0, 0, 1, -1, 1, -1, 0, 1, 0, -1, 1, 1, -2, 2, -1, -1, 2, 0, -2, 2, 1, -3], ctildes: [-0.32539435323653987, 0.72744242109103729, 0.5313667537416471, -1.187908499285238, -0.37573303486780085, 0.83997815527372677, 0.61356947662231498, -1.3716785836699532, -0.68599152932323193, 1.5335832781833236, 0.21692956882435988, -0.48496161406069144, -0.48506926222098773, 1.0844071355177236, 0.84016460735670129, -1.8782482548069264]} + - {mu0: 0, rank: 3, ndensity: 2, num_ms_combs: 3, mus: [0, 0, 0], ns: [1, 2, 3], ls: [2, 2, 0], ms_combs: [0, 0, 0, 1, -1, 0, 2, -2, 0], ctildes: [-0.1650618810831099, 0.16569896142629745, 0.33012376216621986, -0.3313979228525949, -0.33012376216621991, 0.33139792285259501]} + - {mu0: 0, rank: 3, ndensity: 2, num_ms_combs: 10, mus: [0, 0, 0], ns: [1, 2, 3], ls: [2, 2, 2], ms_combs: [0, 0, 0, 0, 1, -1, 0, 2, -2, 1, -2, 1, 1, -1, 0, 1, 0, -1, 1, 1, -2, 2, -2, 0, 2, -1, -1, 2, 0, -2], ctildes: [0.12380706884752062, 0.53768919859597963, -0.12380706884752062, -0.53768919859597963, -0.24761413769504137, -1.0753783971919599, 0.30326414522605261, 1.3170641767661597, -0.12380706884752062, -0.53768919859597963, -0.12380706884752062, -0.53768919859597963, 0.30326414522605261, 1.3170641767661597, -0.24761413769504123, -1.0753783971919593, 0.30326414522605261, 1.3170641767661597, -0.24761413769504137, -1.0753783971919599]} + - {mu0: 0, rank: 3, ndensity: 2, num_ms_combs: 13, mus: [0, 0, 0], ns: [1, 2, 3], ls: [2, 2, 4], ms_combs: [0, 0, 0, 0, 1, -1, 0, 2, -2, 1, -2, 1, 1, -1, 0, 1, 0, -1, 1, 1, -2, 1, 2, -3, 2, -2, 0, 2, -1, -1, 2, 0, -2, 2, 1, -3, 2, 2, -4], ctildes: [0.070773677094601889, -0.021156383650525693, -0.1292144647410004, 0.038626095202088033, 0.091368424245751409, -0.027312773848153608, -0.052751584333716517, 0.01576903733354686, 0.094364902792802519, -0.028208511534034256, -0.1292144647410004, 0.038626095202088033, 0.14920401200281994, -0.044601572925339263, -0.13956757341166479, 0.041720951199458081, 0.023591225698200619, -0.0070521278835085615, -0.052751584333716517, 0.01576903733354686, 0.091368424245751409, -0.027312773848153608, -0.13956757341166479, 0.041720951199458081, 0.19737835518627894, -0.059002335021379672]} + - {mu0: 0, rank: 3, ndensity: 2, num_ms_combs: 8, mus: [0, 0, 0], ns: [1, 2, 3], ls: [2, 3, 1], ms_combs: [0, 0, 0, 0, 1, -1, 1, -2, 1, 1, -1, 0, 1, 0, -1, 2, -3, 1, 2, -2, 0, 2, -1, -1], ctildes: [-0.35774775910205042, -0.13260029939428292, 0.58419964428275972, 0.21653538217084956, -0.75419849772247527, -0.27954597633745298, 0.67457564377427004, 0.25003352237083709, -0.41309153003910826, -0.15311363709982836, 0.9237007420968425, 0.34237250083745002, -0.53329887210026916, -0.19766885552162719, 0.23849850606803355, 0.088400199596188583]} + - {mu0: 0, rank: 3, ndensity: 2, num_ms_combs: 14, mus: [0, 0, 0], ns: [1, 2, 3], ls: [2, 3, 3], ms_combs: [0, 0, 0, 0, 1, -1, 0, 3, -3, 1, -3, 2, 1, -2, 1, 1, -1, 0, 1, 0, -1, 1, 1, -2, 1, 2, -3, 2, -3, 1, 2, -2, 0, 2, -1, -1, 2, 0, -2, 2, 1, -3], ctildes: [0.01361038368125936, -0.029283255560943272, -0.020415575521889034, 0.043924883341414894, 0.034025959203148395, -0.073208138902358166, -0.034025959203148395, 0.073208138902358166, 0.026356394666505346, -0.056706780555134506, -0.009623994595569214, 0.020706388582361655, -0.0096239945955692227, 0.020706388582361672, 0.026356394666505353, -0.056706780555134513, -0.034025959203148395, 0.073208138902358166, 0.021519906130783369, -0.046300892438686181, -0.030433743111149762, 0.0654793500367679, 0.033338495222588353, -0.071729034131829003, -0.030433743111149759, 0.065479350036767886, 0.021519906130783365, -0.046300892438686174]} + - {mu0: 0, rank: 3, ndensity: 2, num_ms_combs: 13, mus: [0, 0, 0], ns: [1, 2, 3], ls: [2, 4, 2], ms_combs: [0, 0, 0, 0, 1, -1, 0, 2, -2, 1, -3, 2, 1, -2, 1, 1, -1, 0, 1, 0, -1, 1, 1, -2, 2, -4, 2, 2, -3, 1, 2, -2, 0, 2, -1, -1, 2, 0, -2], ctildes: [-0.16111897033757405, 0.1297822431801699, 0.2941616483196503, -0.23694887384466692, -0.2080036962918371, 0.16754815549007973, 0.31773089436264967, -0.25593403502784046, -0.33966861368522827, 0.27360499219679424, 0.29416164831965025, -0.2369488738446669, -0.21482529378343215, 0.17304299090689323, 0.1200909900465293, -0.096733972674422797, -0.44933933999259229, 0.3619453834092427, 0.31773089436264973, -0.25593403502784051, -0.20800369629183701, 0.16754815549007968, 0.1200909900465293, -0.096733972674422797, -0.053706323445858031, 0.043260747726723302]} + - {mu0: 0, rank: 3, ndensity: 2, num_ms_combs: 20, mus: [0, 0, 0], ns: [1, 2, 3], ls: [2, 4, 4], ms_combs: [0, 0, 0, 0, 1, -1, 0, 2, -2, 0, 3, -3, 0, 4, -4, 1, -4, 3, 1, -3, 2, 1, -2, 1, 1, -1, 0, 1, 0, -1, 1, 1, -2, 1, 2, -3, 1, 3, -4, 2, -4, 2, 2, -3, 1, 2, -2, 0, 2, -1, -1, 2, 0, -2, 2, 1, -3, 2, 2, -4], ctildes: [-0.0071607173088096216, -0.061199835714097379, 0.012173219424976352, 0.10403972071396551, -0.0057285738470476922, -0.048959868571277855, -0.0050125021161667328, -0.042839884999868148, 0.020050008464666931, 0.17135953999947257, -0.017363816676494583, -0.14840171482035885, 0.01640726454890027, 0.14022643983935643, -0.011162453577746522, -0.095401102384516434, 0.0039220863979527176, 0.033520530536221435, 0.0039220863979527142, 0.0335205305362214, -0.011162453577746523, -0.095401102384516448, 0.016407264548900267, 0.1402264398393564, -0.01736381667649459, -0.14840171482035888, 0.0092813504185992185, 0.079324053209645098, -0.013922025627898824, -0.11898607981446764, 0.016640003330351309, 0.14221556670679741, -0.017540103598899119, -0.14990836984169706, 0.016640003330351302, 0.14221556670679733, -0.013922025627898824, -0.11898607981446764, 0.0092813504185992202, 0.079324053209645112]} + - {mu0: 0, rank: 3, ndensity: 2, num_ms_combs: 4, mus: [0, 0, 0], ns: [1, 2, 3], ls: [3, 0, 3], ms_combs: [0, 0, 0, 1, 0, -1, 2, 0, -2, 3, 0, -3], ctildes: [-0.083868752365226062, 0.54865456489344466, 0.16773750473045212, -1.0973091297868893, -0.1677375047304521, 1.0973091297868891, 0.1677375047304521, -1.0973091297868891]} + - {mu0: 0, rank: 3, ndensity: 2, num_ms_combs: 8, mus: [0, 0, 0], ns: [1, 2, 3], ls: [3, 1, 2], ms_combs: [0, 0, 0, 0, 1, -1, 1, -1, 0, 1, 0, -1, 1, 1, -2, 2, -1, -1, 2, 0, -2, 3, -1, -2], ctildes: [-0.11262312323915573, 1.032213202118228, -0.13004598103820597, 1.1918971402080896, 0.18391279011634523, -1.6855971006359864, 0.21236419776217413, -1.9463598795948791, 0.075082082159437194, -0.68814213474548569, -0.23743039109171502, 2.1760964997262278, -0.16788863960072581, 1.5387325914727261, 0.29079165380207728, -2.6651630276928864]} + - {mu0: 0, rank: 3, ndensity: 2, num_ms_combs: 11, mus: [0, 0, 0], ns: [1, 2, 3], ls: [3, 1, 4], ms_combs: [0, 0, 0, 0, 1, -1, 1, -1, 0, 1, 0, -1, 1, 1, -2, 2, -1, -1, 2, 0, -2, 2, 1, -3, 3, -1, -2, 3, 0, -3, 3, 1, -4], ctildes: [-0.019460626833387031, 0.080093107950154271, 0.030769952744046539, -0.12663832300211356, -0.023834302908256302, 0.09809362319576434, 0.037685341816232551, -0.15509963661847009, -0.037685341816232551, 0.15509963661847009, 0.016853397211282279, -0.06936266615288296, -0.033706794422564558, 0.13872533230576592, 0.04458989776764239, -0.18351636491292556, -0.0097303134166935087, 0.040046553975077108, 0.025743989479286231, -0.10595322268317917, -0.051487978958572468, 0.21190644536635839]} + - {mu0: 0, rank: 3, ndensity: 2, num_ms_combs: 8, mus: [0, 0, 0], ns: [1, 2, 3], ls: [3, 2, 1], ms_combs: [0, 0, 0, 0, 1, -1, 1, -2, 1, 1, -1, 0, 1, 0, -1, 2, -2, 0, 2, -1, -1, 3, -2, -1], ctildes: [-0.49396989884872039, -0.78574416693458315, -0.5703873081437616, -0.90729921258771928, 0.32931326589914678, 0.52382944462305525, 0.93143857380790163, 1.4816134099125824, 0.80664946698238937, 1.2831148515719826, -0.73636684844295575, -1.1713182467931136, -1.0413799839499613, -1.6564941504698971, 1.2754247945125705, 2.0287827152781737]} + - {mu0: 0, rank: 3, ndensity: 2, num_ms_combs: 14, mus: [0, 0, 0], ns: [1, 2, 3], ls: [3, 2, 3], ms_combs: [0, 0, 0, 0, 1, -1, 0, 2, -2, 1, -2, 1, 1, -1, 0, 1, 0, -1, 1, 1, -2, 1, 2, -3, 2, -2, 0, 2, -1, -1, 2, 1, -3, 3, -2, -1, 3, -1, -2, 3, 0, -3], ctildes: [0.077133035646777037, -0.11103477882488579, -0.05454129255933976, 0.078513445054625236, -0.1724747110171079, 0.24828131331909883, 0.18893657964650959, -0.27197855182375658, -0.054541292559339719, 0.078513445054625181, -0.11569955347016553, 0.16655216823732863, 0.14936748125119528, -0.21501792461930333, 0.12195803774338712, -0.17556140028983663, -0.17247471101710793, 0.24828131331909886, 0.14936748125119526, -0.2150179246193033, -0.19283258911694259, 0.27758694706221443, 0.12195803774338713, -0.17556140028983666, -0.19283258911694259, 0.27758694706221443, 0.19283258911694259, -0.27758694706221443]} + - {mu0: 0, rank: 3, ndensity: 2, num_ms_combs: 4, mus: [0, 0, 0], ns: [1, 2, 3], ls: [3, 3, 0], ms_combs: [0, 0, 0, 1, -1, 0, 2, -2, 0, 3, -3, 0], ctildes: [0.28704238110606073, 0.13301711775006719, -0.57408476221212146, -0.26603423550013439, 0.57408476221212146, 0.26603423550013433, -0.57408476221212146, -0.26603423550013433]} + - {mu0: 0, rank: 3, ndensity: 2, num_ms_combs: 14, mus: [0, 0, 0], ns: [1, 2, 3], ls: [3, 3, 2], ms_combs: [0, 0, 0, 0, 1, -1, 0, 2, -2, 1, -3, 2, 1, -2, 1, 1, -1, 0, 1, 0, -1, 1, 1, -2, 2, -3, 1, 2, -1, -1, 2, 0, -2, 3, -3, 0, 3, -2, -1, 3, -1, -2], ctildes: [-0.018484833582841344, 0.59932029439238488, 0.013070751175531945, -0.42378344426757353, 0.041333344444004258, -1.3401209185565595, -0.02922708814547471, 0.94760858912128776, -0.035795726311880056, 1.1605787596129167, 0.027727250374262009, -0.89898044158857704, 0.013070751175531945, -0.42378344426757353, -0.04527841025822392, 1.4680289137559421, 0.046212083957103363, -1.4983007359809624, -0.035795726311880056, 1.1605787596129167, 0.041333344444004258, -1.3401209185565595, -0.046212083957103356, 1.4983007359809621, 0.046212083957103363, -1.4983007359809624, -0.02922708814547471, 0.94760858912128776]} + - {mu0: 0, rank: 3, ndensity: 2, num_ms_combs: 22, mus: [0, 0, 0], ns: [1, 2, 3], ls: [3, 3, 4], ms_combs: [0, 0, 0, 0, 1, -1, 0, 2, -2, 0, 3, -3, 1, -3, 2, 1, -2, 1, 1, -1, 0, 1, 0, -1, 1, 1, -2, 1, 2, -3, 1, 3, -4, 2, -3, 1, 2, -2, 0, 2, -1, -1, 2, 0, -2, 2, 1, -3, 2, 2, -4, 3, -3, 0, 3, -2, -1, 3, -1, -2, 3, 0, -3, 3, 1, -4], ctildes: [0.030909787486158243, -0.37547637856580041, -0.039904364056233745, 0.48473792035987195, -0.01784577412572759, 0.2167813882393107, 0.081779610766230998, -0.99341712086425027, -0.075713207398952403, 0.91972553795430101, 0.058284054230660166, -0.70800504922465191, -0.010303262495386082, 0.12515879285526682, -0.039904364056233745, 0.48473792035987195, 0.065163553632020199, -0.7915737092397036, -0.038551278223732215, 0.46830132180661832, -0.066772772580227888, 0.81112168262072626, 0.056433292846199204, -0.68552294116946, -0.072122837467702552, 0.87611154998686747, 0.058284054230660166, -0.70800504922465191, -0.01784577412572759, 0.2167813882393107, -0.038551278223732215, 0.46830132180661832, 0.086203278727772614, -1.0471535895126036, -0.030909787486158243, 0.37547637856580041, 0.056433292846199204, -0.68552294116946, -0.075713207398952403, 0.91972553795430101, 0.081779610766230998, -0.99341712086425027, -0.066772772580227888, 0.81112168262072626]} + - {mu0: 0, rank: 3, ndensity: 2, num_ms_combs: 11, mus: [0, 0, 0], ns: [1, 2, 3], ls: [3, 4, 1], ms_combs: [0, 0, 0, 0, 1, -1, 1, -2, 1, 1, -1, 0, 1, 0, -1, 2, -3, 1, 2, -2, 0, 2, -1, -1, 3, -4, 1, 3, -3, 0, 3, -2, -1], ctildes: [0.084471601199444829, -0.39446692896560082, -0.1335613286958284, 0.62370697857157309, 0.16357855233646218, -0.76388192325667825, -0.16357855233646215, 0.76388192325667814, 0.10345616034725565, -0.48312134818420988, -0.19354875328528928, 0.90383728057081736, 0.14630910507413461, -0.68323676287408408, -0.073154552537067277, 0.34161838143704193, 0.22349084962115645, -1.0436613944823614, -0.1117454248105782, 0.52183069724118059, 0.042235800599722421, -0.19723346448280046]} + - {mu0: 0, rank: 3, ndensity: 2, num_ms_combs: 22, mus: [0, 0, 0], ns: [1, 2, 3], ls: [3, 4, 3], ms_combs: [0, 0, 0, 0, 1, -1, 0, 2, -2, 0, 3, -3, 1, -4, 3, 1, -3, 2, 1, -2, 1, 1, -1, 0, 1, 0, -1, 1, 1, -2, 1, 2, -3, 2, -4, 2, 2, -3, 1, 2, -2, 0, 2, -1, -1, 2, 0, -2, 2, 1, -3, 3, -4, 1, 3, -3, 0, 3, -2, -1, 3, -1, -2, 3, 0, -3], ctildes: [0.0043280834926612503, 0.04611598663533617, -0.0055875317626907497, -0.059535482744193589, -0.0024988201695631479, -0.026625077297856514, 0.011451032575105517, 0.12201143210147783, -0.0093497289456656672, -0.099621917144952027, -0.0053980685236301087, -0.057516740680824643, 0.0091244011601241369, 0.097221036209031414, -0.0055875317626907523, -0.05953548274419361, -0.0014426944975537517, -0.015371995545112073, 0.0081611124992062058, 0.086957138322545077, -0.010601596121182926, -0.11296063624158208, 0.01207044816603885, 0.12861134200655142, -0.0053980685236301095, -0.05751674068082465, -0.0024988201695631475, -0.026625077297856511, 0.0081611124992062058, 0.086957138322545091, -0.01009886148287625, -0.10760396881578439, 0.0079019631989877062, 0.084195887139267955, -0.0093497289456656672, -0.099621917144952027, 0.011451032575105515, 0.12201143210147782, -0.010601596121182926, -0.11296063624158209, 0.0079019631989877045, 0.084195887139267941, -0.0043280834926612503, -0.04611598663533617]} + - {mu0: 0, rank: 3, ndensity: 2, num_ms_combs: 5, mus: [0, 0, 0], ns: [1, 2, 3], ls: [4, 0, 4], ms_combs: [0, 0, 0, 1, 0, -1, 2, 0, -2, 3, 0, -3, 4, 0, -4], ctildes: [-0.26489671215446242, 1.0071204238792706, 0.52979342430892462, -2.0142408477585403, -0.52979342430892462, 2.0142408477585403, 0.52979342430892462, -2.0142408477585403, -0.52979342430892462, 2.0142408477585407]} + - {mu0: 0, rank: 3, ndensity: 2, num_ms_combs: 11, mus: [0, 0, 0], ns: [1, 2, 3], ls: [4, 1, 3], ms_combs: [0, 0, 0, 0, 1, -1, 1, -1, 0, 1, 0, -1, 1, 1, -2, 2, -1, -1, 2, 0, -2, 2, 1, -3, 3, -1, -2, 3, 0, -3, 4, -1, -3], ctildes: [-0.060738404440152584, 0.21824038992092704, -0.07438904933458497, 0.26728879828615593, 0.096035849737683213, -0.34506835479669196, 0.11761941443596079, -0.42262069781678169, 0.052601001230505665, -0.18900172180334412, -0.11761941443596079, 0.42262069781678169, -0.10520200246101133, 0.37800344360668825, -0.030369202220076285, 0.10912019496046349, 0.13916916796892054, -0.50005155325466277, 0.080349356589752494, -0.28870489888026996, -0.16069871317950499, 0.57740979776053991]} + - {mu0: 0, rank: 3, ndensity: 2, num_ms_combs: 13, mus: [0, 0, 0], ns: [1, 2, 3], ls: [4, 2, 2], ms_combs: [0, 0, 0, 0, 1, -1, 0, 2, -2, 1, -2, 1, 1, -1, 0, 1, 0, -1, 1, 1, -2, 2, -2, 0, 2, -1, -1, 2, 0, -2, 3, -2, -1, 3, -1, -2, 4, -2, -2], ctildes: [-0.10964472437082332, 0.43202887059203154, -0.14619296582776448, 0.57603849412270891, -0.036548241456941112, 0.1440096235306772, 0.081724352355796281, -0.32201530762874753, 0.20018296283112125, -0.78877319305578675, 0.20018296283112127, -0.78877319305578686, 0.081724352355796281, -0.32201530762874753, -0.14155073049590039, 0.55774687362791242, -0.23115137495544943, 0.91079683068063833, -0.14155073049590042, 0.55774687362791264, 0.21622231239125264, -0.85197242234162651, 0.21622231239125264, -0.8519724223416264, -0.30578452667138156, 1.2048709544433864]} + - {mu0: 0, rank: 3, ndensity: 2, num_ms_combs: 20, mus: [0, 0, 0], ns: [1, 2, 3], ls: [4, 2, 4], ms_combs: [0, 0, 0, 0, 1, -1, 0, 2, -2, 1, -2, 1, 1, -1, 0, 1, 0, -1, 1, 1, -2, 1, 2, -3, 2, -2, 0, 2, -1, -1, 2, 0, -2, 2, 1, -3, 2, 2, -4, 3, -2, -1, 3, -1, -2, 3, 0, -3, 3, 1, -4, 4, -2, -2, 4, -1, -3, 4, 0, -4], ctildes: [0.01396083278405482, 0.033744411112380565, -0.0076466630373844711, -0.018482575155978821, -0.032442043723098481, -0.078414925359495791, 0.034196916705253384, 0.082656588896034833, -0.0076466630373844772, -0.018482575155978835, -0.023733415732893188, -0.057365498891046939, 0.021762784528160578, 0.052602331066321437, 0.027142961162187215, 0.065606633531943306, -0.032442043723098501, -0.078414925359495832, 0.021762784528160575, 0.052602331066321423, 0.011168666227243846, 0.026995528889904426, -0.031988286498776122, -0.077318159102096398, -0.018095307441458149, -0.043737755687962218, 0.027142961162187215, 0.065606633531943306, -0.031988286498776122, -0.077318159102096412, 0.0097725829488383709, 0.023621087778666384, 0.033853220377138671, 0.081825848325388889, -0.018095307441458146, -0.043737755687962211, 0.033853220377138664, 0.081825848325388861, -0.039090331795353477, -0.094484351114665535]} + - {mu0: 0, rank: 3, ndensity: 2, num_ms_combs: 11, mus: [0, 0, 0], ns: [1, 2, 3], ls: [4, 3, 1], ms_combs: [0, 0, 0, 0, 1, -1, 1, -2, 1, 1, -1, 0, 1, 0, -1, 2, -3, 1, 2, -2, 0, 2, -1, -1, 3, -3, 0, 3, -2, -1, 4, -3, -1], ctildes: [-0.029386699420021999, -0.057391945171878525, -0.035991209401798126, -0.070290490508445491, 0.025449628231116479, 0.049702882491450461, 0.056907098726874189, 0.11113902392856737, 0.046464451541009304, 0.090744633045319947, -0.014693349710011003, -0.028695972585939269, -0.050899256462232978, -0.099405764982900963, -0.056907098726874196, -0.11113902392856738, 0.038874949259192118, 0.075922407091522356, 0.067333387258582858, 0.13150146651544439, -0.077749898518384264, -0.15184481418304474]} + - {mu0: 0, rank: 3, ndensity: 2, num_ms_combs: 22, mus: [0, 0, 0], ns: [1, 2, 3], ls: [4, 3, 3], ms_combs: [0, 0, 0, 0, 1, -1, 0, 2, -2, 0, 3, -3, 1, -3, 2, 1, -2, 1, 1, -1, 0, 1, 0, -1, 1, 1, -2, 1, 2, -3, 2, -3, 1, 2, -2, 0, 2, -1, -1, 2, 0, -2, 2, 1, -3, 3, -3, 0, 3, -2, -1, 3, -1, -2, 3, 0, -3, 4, -3, -1, 4, -2, -2, 4, -1, -3], ctildes: [0.016522832954256831, -0.083740614362812285, -0.0055076109847522836, 0.027913538120937463, -0.038553276893265938, 0.19539476684656201, -0.016522832954256831, 0.083740614362812285, 0.03016642774312063, -0.15288874488617796, 0.031155752603646836, -0.15790281673779247, -0.021330885621334606, 0.10810866827611657, -0.021330885621334596, 0.10810866827611652, 0.031155752603646829, -0.15790281673779244, 0.030166427743120637, -0.15288874488617799, -0.040472509843171994, 0.20512177593607042, -0.0095394620539153967, 0.048347666244474299, 0.034833190355960192, -0.17654071603219776, -0.0095394620539153985, 0.048347666244474306, -0.040472509843171987, 0.20512177593607039, 0.043715306951226232, -0.22155684023976491, -0.020607593324575662, 0.10444289610120155, -0.020607593324575658, 0.10444289610120154, 0.043715306951226239, -0.22155684023976494, -0.035693398659882275, 0.1809004025369185, 0.046079979526422052, -0.23354141544923435, -0.035693398659882275, 0.1809004025369185]} + - {mu0: 0, rank: 3, ndensity: 2, num_ms_combs: 5, mus: [0, 0, 0], ns: [1, 2, 3], ls: [4, 4, 0], ms_combs: [0, 0, 0, 1, -1, 0, 2, -2, 0, 3, -3, 0, 4, -4, 0], ctildes: [0.080972324773551138, 0.24551697993196622, -0.16194464954710219, -0.49103395986393222, 0.16194464954710219, 0.49103395986393222, -0.16194464954710219, -0.49103395986393222, 0.16194464954710222, 0.49103395986393228]} + - {mu0: 0, rank: 3, ndensity: 2, num_ms_combs: 20, mus: [0, 0, 0], ns: [1, 2, 3], ls: [4, 4, 2], ms_combs: [0, 0, 0, 0, 1, -1, 0, 2, -2, 1, -3, 2, 1, -2, 1, 1, -1, 0, 1, 0, -1, 1, 1, -2, 2, -4, 2, 2, -3, 1, 2, -2, 0, 2, -1, -1, 2, 0, -2, 3, -4, 1, 3, -3, 0, 3, -2, -1, 3, -1, -2, 4, -4, 0, 4, -3, -1, 4, -2, -2], ctildes: [-0.0065116482638481379, -0.13738314896208884, 0.0035665766406489763, 0.075247849707628489, 0.015131703169346566, 0.31924958878782167, -0.012660091175171256, -0.26710336942621166, -0.01015065507060238, -0.21415913469552719, 0.01106980204854183, 0.23355135323555096, 0.0035665766406489763, 0.075247849707628489, -0.015950215630907903, -0.33651861421389001, 0.008440060783447504, 0.17806891295080779, 0.014920060534005934, 0.31478433966508296, -0.0052093186110785108, -0.10990651916967108, -0.01015065507060238, -0.21415913469552719, 0.015131703169346566, 0.31924958878782167, -0.015789907887603708, -0.33313643174859797, -0.0045581537846936943, -0.096168204273462135, 0.014920060534005934, 0.31478433966508296, -0.012660091175171256, -0.26710336942621166, 0.01823261513877478, 0.38467281709384865, -0.015789907887603708, -0.33313643174859797, 0.008440060783447504, 0.17806891295080779]} + - {mu0: 0, rank: 3, ndensity: 2, num_ms_combs: 31, mus: [0, 0, 0], ns: [1, 2, 3], ls: [4, 4, 4], ms_combs: [0, 0, 0, 0, 1, -1, 0, 2, -2, 0, 3, -3, 0, 4, -4, 1, -4, 3, 1, -3, 2, 1, -2, 1, 1, -1, 0, 1, 0, -1, 1, 1, -2, 1, 2, -3, 1, 3, -4, 2, -4, 2, 2, -3, 1, 2, -2, 0, 2, -1, -1, 2, 0, -2, 2, 1, -3, 2, 2, -4, 3, -4, 1, 3, -3, 0, 3, -2, -1, 3, -1, -2, 3, 0, -3, 3, 1, -4, 4, -4, 0, 4, -3, -1, 4, -2, -2, 4, -1, -3, 4, 0, -4], ctildes: [-0.0021693527448238942, 0.050020904885152581, 0.0021693527448238934, -0.05002090488515256, 0.0026514311325625353, -0.061136661526297549, -0.0050618230712557504, 0.11671544473202262, -0.0033745487141705012, 0.077810296488015102, 0.0053356300059856521, -0.12302888115756412, 0.0020166785833845868, -0.046500546231633563, -0.0045733971479877018, 0.10545332670648354, 0.0021693527448238942, -0.050020904885152581, 0.0021693527448238934, -0.05002090488515256, -0.004573397147987701, 0.10545332670648351, 0.0020166785833845868, -0.046500546231633563, 0.005335630005985653, -0.12302888115756414, -0.006050035750153759, 0.13950163869490068, 0.0020166785833845868, -0.046500546231633563, 0.0026514311325625379, -0.061136661526297612, -0.0045733971479877018, 0.10545332670648354, 0.0026514311325625353, -0.061136661526297549, 0.0020166785833845868, -0.046500546231633563, -0.0060500357501537599, 0.1395016386949007, 0.0053356300059856487, -0.12302888115756405, -0.0050618230712557531, 0.11671544473202269, 0.0020166785833845868, -0.046500546231633563, 0.0020166785833845868, -0.046500546231633563, -0.0050618230712557504, 0.11671544473202262, 0.005335630005985653, -0.12302888115756414, -0.0033745487141705012, 0.077810296488015102, 0.0053356300059856487, -0.12302888115756405, -0.006050035750153759, 0.13950163869490068, 0.0053356300059856521, -0.12302888115756412, -0.0033745487141705012, 0.077810296488015102]} + - {mu0: 0, rank: 3, ndensity: 2, num_ms_combs: 1, mus: [0, 0, 0], ns: [1, 3, 3], ls: [0, 0, 0], ms_combs: [0, 0, 0], ctildes: [-0.36560261281322998, 1.4263884802517246]} + - {mu0: 0, rank: 3, ndensity: 2, num_ms_combs: 2, mus: [0, 0, 0], ns: [1, 3, 3], ls: [0, 1, 1], ms_combs: [0, 0, 0, 0, 1, -1], ctildes: [0.13509861289563108, 0.1498433884338477, -0.2701972257912621, -0.29968677686769535]} + - {mu0: 0, rank: 3, ndensity: 2, num_ms_combs: 3, mus: [0, 0, 0], ns: [1, 3, 3], ls: [0, 2, 2], ms_combs: [0, 0, 0, 0, 1, -1, 0, 2, -2], ctildes: [0.011970413075690989, -0.61878500314995188, -0.023940826151381982, 1.237570006299904, 0.023940826151381985, -1.2375700062999042]} + - {mu0: 0, rank: 3, ndensity: 2, num_ms_combs: 4, mus: [0, 0, 0], ns: [1, 3, 3], ls: [0, 3, 3], ms_combs: [0, 0, 0, 0, 1, -1, 0, 2, -2, 0, 3, -3], ctildes: [0.0078461298635156742, -0.015905883709480644, -0.015692259727031348, 0.031811767418961287, 0.015692259727031348, -0.03181176741896128, -0.015692259727031348, 0.03181176741896128]} + - {mu0: 0, rank: 3, ndensity: 2, num_ms_combs: 5, mus: [0, 0, 0], ns: [1, 3, 3], ls: [0, 4, 4], ms_combs: [0, 0, 0, 0, 1, -1, 0, 2, -2, 0, 3, -3, 0, 4, -4], ctildes: [0.00096508930404913865, 0.08481599705918641, -0.0019301786080982764, -0.16963199411837274, 0.0019301786080982764, 0.16963199411837274, -0.0019301786080982764, -0.16963199411837274, 0.0019301786080982769, 0.16963199411837276]} + - {mu0: 0, rank: 3, ndensity: 2, num_ms_combs: 2, mus: [0, 0, 0], ns: [1, 3, 3], ls: [1, 0, 1], ms_combs: [0, 0, 0, 1, 0, -1], ctildes: [-1.0017807823820983, 1.7036114456523681, 2.0035615647641962, -3.4072228913047353]} + - {mu0: 0, rank: 3, ndensity: 2, num_ms_combs: 5, mus: [0, 0, 0], ns: [1, 3, 3], ls: [1, 1, 2], ms_combs: [0, 0, 0, 0, 1, -1, 1, -1, 0, 1, 0, -1, 1, 1, -2], ctildes: [-0.35507907340080197, 0.48344701544150637, 0.6150149958346679, -0.83735479351222486, -0.35507907340080203, 0.48344701544150642, 0.6150149958346679, -0.83735479351222486, -0.86976254817221998, 1.184198505503111]} + - {mu0: 0, rank: 3, ndensity: 2, num_ms_combs: 8, mus: [0, 0, 0], ns: [1, 3, 3], ls: [1, 2, 3], ms_combs: [0, 0, 0, 0, 1, -1, 0, 2, -2, 1, -2, 1, 1, -1, 0, 1, 0, -1, 1, 1, -2, 1, 2, -3], ctildes: [-0.053941006038138346, 0.1668374281436438, 0.101712136409579, -0.31459167145625039, -0.080410504184002599, 0.24870655368028266, 0.035960670692092224, -0.11122495209576252, -0.062285708712956793, 0.19264726809927518, 0.088085294003883555, -0.27244437930012066, -0.11371762557427498, 0.35172418126572791, 0.13927507870892231, -0.43077238714960592]} + - {mu0: 0, rank: 3, ndensity: 2, num_ms_combs: 11, mus: [0, 0, 0], ns: [1, 3, 3], ls: [1, 3, 4], ms_combs: [0, 0, 0, 0, 1, -1, 0, 2, -2, 0, 3, -3, 1, -3, 2, 1, -2, 1, 1, -1, 0, 1, 0, -1, 1, 1, -2, 1, 2, -3, 1, 3, -4], ctildes: [0.086363941144239148, -0.64124597843691522, -0.16724305288223779, 1.241767497654326, 0.14958673400371036, -1.1106706148019536, -0.11424875525553788, 0.84828869408218188, 0.043181970572119546, -0.32062298921845739, -0.074793367001855182, 0.55533530740097681, 0.10577379398957193, -0.78536272339109336, -0.13655338086226201, 1.0138989161419352, 0.16724305288223779, -1.2417674976543258, -0.1978846488040934, 1.4692791176365914, 0.22849751051107581, -1.6965773881643642]} + - {mu0: 0, rank: 3, ndensity: 2, num_ms_combs: 3, mus: [0, 0, 0], ns: [1, 3, 3], ls: [2, 0, 2], ms_combs: [0, 0, 0, 1, 0, -1, 2, 0, -2], ctildes: [-0.49500117294713625, -0.28106984501484483, 0.99000234589427261, 0.56213969002968978, -0.99000234589427283, -0.56213969002968989]} + - {mu0: 0, rank: 3, ndensity: 2, num_ms_combs: 4, mus: [0, 0, 0], ns: [1, 3, 3], ls: [2, 1, 1], ms_combs: [0, -1, 1, 0, 0, 0, 1, -1, 0, 2, -1, -1], ctildes: [-0.35543777621359052, 0.62706702277900384, -0.35543777621359057, 0.62706702277900395, 1.2312725746624709, -2.1722238864083709, -0.87064118703285276, 1.5359942403347557]} + - {mu0: 0, rank: 3, ndensity: 2, num_ms_combs: 8, mus: [0, 0, 0], ns: [1, 3, 3], ls: [2, 1, 3], ms_combs: [0, 0, 0, 0, 1, -1, 1, -1, 0, 1, 0, -1, 1, 1, -2, 2, -1, -1, 2, 0, -2, 2, 1, -3], ctildes: [0.15387408760308741, 0.11110842203763104, -0.25127533284258852, -0.1814392934120011, 0.1776784917979011, 0.12829695474532163, -0.29014776211476256, -0.20950804978598864, 0.3243950598040235, 0.23423712057744034, -0.10258272506872491, -0.074072281358420686, 0.22938194657084066, 0.16563065636591906, -0.39730118579974566, -0.28688071211675337]} + - {mu0: 0, rank: 3, ndensity: 2, num_ms_combs: 7, mus: [0, 0, 0], ns: [1, 3, 3], ls: [2, 2, 2], ms_combs: [0, -2, 2, 0, -1, 1, 0, 0, 0, 1, -2, 1, 1, -1, 0, 2, -2, 0, 2, -1, -1], ctildes: [0.15160731508868922, -0.15587725100342534, 0.075803657544344566, -0.077938625501712613, -0.075803657544344566, 0.077938625501712613, -0.37136056324064148, 0.38181972746612908, 0.15160731508868913, -0.15587725100342523, 0.30321463017737832, -0.31175450200685056, -0.18568028162032074, 0.19090986373306454]} + - {mu0: 0, rank: 3, ndensity: 2, num_ms_combs: 13, mus: [0, 0, 0], ns: [1, 3, 3], ls: [2, 2, 4], ms_combs: [0, 0, 0, 0, 1, -1, 0, 2, -2, 1, -2, 1, 1, -1, 0, 1, 0, -1, 1, 1, -2, 1, 2, -3, 2, -2, 0, 2, -1, -1, 2, 0, -2, 2, 1, -3, 2, 2, -4], ctildes: [-0.025209141870039458, -0.26895853458733943, 0.046025385525228578, 0.4910488547567306, -0.032544862211614289, -0.34722397509237213, 0.018789784958614792, 0.20046985548867302, -0.033612189160052611, -0.35861137944978594, 0.046025385525228578, 0.4910488547567306, -0.053145537445094031, -0.56701437695811152, 0.049713098188876807, 0.53039338298808569, -0.0084030472900131509, -0.089652844862446443, 0.018789784958614792, 0.20046985548867302, -0.032544862211614289, -0.34722397509237213, 0.049713098188876807, 0.53039338298808569, -0.070304937686294935, -0.75008951561469805]} + - {mu0: 0, rank: 3, ndensity: 2, num_ms_combs: 9, mus: [0, 0, 0], ns: [1, 3, 3], ls: [2, 3, 3], ms_combs: [0, -3, 3, 0, -1, 1, 0, 0, 0, 1, -3, 2, 1, -2, 1, 1, -1, 0, 2, -3, 1, 2, -2, 0, 2, -1, -1], ctildes: [-0.054287467998024905, 0.031283408201001742, 0.032572480798814939, -0.018770044920601043, -0.021714987199209963, 0.012513363280400699, 0.10857493599604981, -0.062566816402003483, -0.084101783785647435, 0.048464047590035322, 0.030709629403880879, -0.01769656806204415, -0.06866881891090397, 0.039570729155182428, 0.097112375015942498, -0.055961461844251451, -0.053190638409132816, 0.030651355003061174]} + - {mu0: 0, rank: 3, ndensity: 2, num_ms_combs: 13, mus: [0, 0, 0], ns: [1, 3, 3], ls: [2, 4, 4], ms_combs: [0, -4, 4, 0, -3, 3, 0, -2, 2, 0, -1, 1, 0, 0, 0, 1, -4, 3, 1, -3, 2, 1, -2, 1, 1, -1, 0, 2, -4, 2, 2, -3, 1, 2, -2, 0, 2, -1, -1], ctildes: [-0.075779409406080911, 0.48001088588730983, 0.018944852351520231, -0.12000272147182747, 0.021651259830308824, -0.13714596739637419, -0.046008927139406276, 0.29143518071729529, 0.027064074787886052, -0.17143245924546788, 0.131253787258895, -0.83140324254296694, -0.12402317132943366, 0.78560222106478805, 0.084377434666432521, -0.53447351306333613, -0.029647208518664068, 0.18779485003465565, -0.070158100377042856, 0.44440372626409996, 0.10523715056556426, -0.66660558939614978, -0.12578245312079359, 0.7967460715884942, 0.066293173590843663, -0.41992205050186854]} + - {mu0: 0, rank: 3, ndensity: 2, num_ms_combs: 4, mus: [0, 0, 0], ns: [1, 3, 3], ls: [3, 0, 3], ms_combs: [0, 0, 0, 1, 0, -1, 2, 0, -2, 3, 0, -3], ctildes: [-0.024605695432553292, -0.077658743574203992, 0.049211390865106584, 0.15531748714840798, -0.049211390865106577, -0.15531748714840796, 0.049211390865106577, 0.15531748714840796]} + - {mu0: 0, rank: 3, ndensity: 2, num_ms_combs: 8, mus: [0, 0, 0], ns: [1, 3, 3], ls: [3, 1, 2], ms_combs: [0, 0, 0, 0, 1, -1, 1, -1, 0, 1, 0, -1, 1, 1, -2, 2, -1, -1, 2, 0, -2, 3, -1, -2], ctildes: [0.56207317558847436, 0.061251947109898383, 0.64902619846054721, 0.070727656304577127, -0.9178616521983578, -0.10002401078079591, -1.0598553439511129, -0.11549777909943709, -0.37471545039231641, -0.040834631406598945, 1.1849542976955545, 0.1291304426582979, 0.83788921929666993, 0.091309011661303105, -1.451266698936053, -0.15815184738627605]} + - {mu0: 0, rank: 3, ndensity: 2, num_ms_combs: 11, mus: [0, 0, 0], ns: [1, 3, 3], ls: [3, 1, 4], ms_combs: [0, 0, 0, 0, 1, -1, 1, -1, 0, 1, 0, -1, 1, 1, -2, 2, -1, -1, 2, 0, -2, 2, 1, -3, 3, -1, -2, 3, 0, -3, 3, 1, -4], ctildes: [0.028738505209645691, -0.11478559904013354, -0.045439566505547549, 0.18149196777682952, 0.035197336866974023, -0.14058307373401485, -0.055651876035926397, 0.22228135673343952, 0.05565187603592639, -0.22228135673343949, -0.024888275578344592, 0.099407244757370283, 0.049776551156689185, -0.19881448951474057, -0.065848187741542025, 0.26300684814613101, 0.014369252604822837, -0.057392799520066727, -0.038017468918228305, 0.15184707457588378, 0.076034937836456623, -0.30369414915176762]} + - {mu0: 0, rank: 3, ndensity: 2, num_ms_combs: 14, mus: [0, 0, 0], ns: [1, 3, 3], ls: [3, 2, 3], ms_combs: [0, 0, 0, 0, 1, -1, 0, 2, -2, 1, -2, 1, 1, -1, 0, 1, 0, -1, 1, 1, -2, 1, 2, -3, 2, -2, 0, 2, -1, -1, 2, 1, -3, 3, -2, -1, 3, -1, -2, 3, 0, -3], ctildes: [-0.036497477126504647, -0.17595388617995519, 0.02580761357235235, 0.1244181860939723, 0.08161083986210807, 0.39344485040364058, -0.089400195858836781, -0.43099723940063905, 0.025807613572352329, 0.12441818609397221, 0.054746215689756954, 0.26393082926993272, -0.070677060544769313, -0.34073323543772099, -0.057707578284826012, -0.27820752174334096, 0.081610839862108084, 0.39344485040364063, -0.070677060544769313, -0.34073323543772094, 0.091243692816261601, 0.43988471544988794, -0.057707578284826019, -0.27820752174334101, 0.091243692816261601, 0.43988471544988794, -0.091243692816261601, -0.43988471544988794]} + - {mu0: 0, rank: 3, ndensity: 2, num_ms_combs: 22, mus: [0, 0, 0], ns: [1, 3, 3], ls: [3, 3, 4], ms_combs: [0, 0, 0, 0, 1, -1, 0, 2, -2, 0, 3, -3, 1, -3, 2, 1, -2, 1, 1, -1, 0, 1, 0, -1, 1, 1, -2, 1, 2, -3, 1, 3, -4, 2, -3, 1, 2, -2, 0, 2, -1, -1, 2, 0, -2, 2, 1, -3, 2, 2, -4, 3, -3, 0, 3, -2, -1, 3, -1, -2, 3, 0, -3, 3, 1, -4], ctildes: [-0.011483891780162382, 0.098212407385649167, 0.014825640538072378, -0.12679167273185243, 0.0066302280106212818, -0.056702959841865734, -0.030383521733488478, 0.25984560560339076, 0.028129675122739787, -0.2405702845051903, -0.021654234005774042, 0.18519109135746192, 0.0038279639267207941, -0.032737469128549725, 0.014825640538072378, -0.12679167273185243, -0.024210169618799179, 0.20704993455132945, 0.014322929502719031, -0.12249239318912185, 0.024808041611936608, -0.21216304854426304, -0.020966621919810302, 0.17931050317335662, 0.026795747487045555, -0.229162283899848, -0.021654234005774042, 0.18519109135746192, 0.0066302280106212818, -0.056702959841865734, 0.014322929502719031, -0.12249239318912185, -0.032027044005017026, 0.27390131789750882, 0.011483891780162382, -0.098212407385649167, -0.020966621919810302, 0.17931050317335662, 0.028129675122739787, -0.2405702845051903, -0.030383521733488478, 0.25984560560339076, 0.024808041611936608, -0.21216304854426304]} + - {mu0: 0, rank: 3, ndensity: 2, num_ms_combs: 5, mus: [0, 0, 0], ns: [1, 3, 3], ls: [4, 0, 4], ms_combs: [0, 0, 0, 1, 0, -1, 2, 0, -2, 3, 0, -3, 4, 0, -4], ctildes: [0.060078569739227657, -0.0028874257955820168, -0.12015713947845526, 0.005774851591164031, 0.12015713947845526, -0.005774851591164031, -0.12015713947845526, 0.005774851591164031, 0.12015713947845527, -0.0057748515911640319]} + - {mu0: 0, rank: 3, ndensity: 2, num_ms_combs: 11, mus: [0, 0, 0], ns: [1, 3, 3], ls: [4, 1, 3], ms_combs: [0, 0, 0, 0, 1, -1, 1, -1, 0, 1, 0, -1, 1, 1, -2, 2, -1, -1, 2, 0, -2, 2, 1, -3, 3, -1, -2, 3, 0, -3, 4, -1, -3], ctildes: [0.069463641176715621, -0.46953886343647694, 0.08507523827936804, -0.57506531491286061, -0.1098316603435401, 0.7424061292130113, -0.13451576272217369, 0.9092580992433168, -0.060157277898402493, 0.40663258380006123, 0.13451576272217369, -0.9092580992433168, 0.12031455579680499, -0.81326516760012246, 0.034731820588357804, -0.23476943171823841, -0.15916119686977534, 1.0758486917105941, -0.091891759857307798, 0.62114153176641806, 0.1837835197146156, -1.2422830635328361]} + - {mu0: 0, rank: 3, ndensity: 2, num_ms_combs: 9, mus: [0, 0, 0], ns: [1, 3, 3], ls: [4, 2, 2], ms_combs: [0, -2, 2, 0, -1, 1, 0, 0, 0, 1, -2, 1, 1, -1, 0, 2, -2, 0, 2, -1, -1, 3, -2, -1, 4, -2, -2], ctildes: [0.0093017469169076137, 0.04963783378965915, 0.037206987667630462, 0.19855133515863663, 0.027905240750722839, 0.14891350136897744, -0.041598676831409026, -0.22198714121902771, -0.10189553221188864, -0.54375522544576926, 0.072051021799638737, 0.3844930072183233, 0.058829412951754104, 0.31393722578438055, -0.11005975376525268, -0.58732276991972321, 0.07782399822313181, 0.41529991335550259]} + - {mu0: 0, rank: 3, ndensity: 2, num_ms_combs: 20, mus: [0, 0, 0], ns: [1, 3, 3], ls: [4, 2, 4], ms_combs: [0, 0, 0, 0, 1, -1, 0, 2, -2, 1, -2, 1, 1, -1, 0, 1, 0, -1, 1, 1, -2, 1, 2, -3, 2, -2, 0, 2, -1, -1, 2, 0, -2, 2, 1, -3, 2, 2, -4, 3, -2, -1, 3, -1, -2, 3, 0, -3, 3, 1, -4, 4, -2, -2, 4, -1, -3, 4, 0, -4], ctildes: [-0.0060586576817232675, -0.095701119429079093, 0.003318463480481787, 0.052417661889802514, 0.014079048181011501, 0.22238930505733806, -0.014840619846415644, -0.23441891041439702, 0.00331846348048179, 0.052417661889802562, 0.010299718058929552, 0.1626919030294344, -0.0094445126373709463, -0.14918328107314374, -0.01177937682469961, -0.1860642418701671, 0.014079048181011508, 0.2223893050573382, -0.0094445126373709446, -0.14918328107314371, -0.0048469261453786099, -0.076560895543263202, 0.01388212871816126, 0.21927881193788182, 0.0078529178831330763, 0.12404282791344476, -0.01177937682469961, -0.1860642418701671, 0.013882128718161262, 0.21927881193788185, -0.0042410603772062858, -0.066990783600355333, -0.014691464102577026, -0.23206288166933467, 0.0078529178831330745, 0.12404282791344474, -0.014691464102577021, -0.23206288166933461, 0.01696424150882514, 0.26796313440142133]} + - {mu0: 0, rank: 3, ndensity: 2, num_ms_combs: 14, mus: [0, 0, 0], ns: [1, 3, 3], ls: [4, 3, 3], ms_combs: [0, -3, 3, 0, -2, 2, 0, -1, 1, 0, 0, 0, 1, -3, 2, 1, -2, 1, 1, -1, 0, 2, -3, 1, 2, -2, 0, 2, -1, -1, 3, -3, 0, 3, -2, -1, 4, -3, -1, 4, -2, -2], ctildes: [0.0028893978151260471, -0.0038574694152543436, 0.00674192823529411, -0.0090007619689268029, 0.00096313260504201682, -0.0012858231384181161, -0.0028893978151260471, 0.0038574694152543436, -0.010550589073004516, 0.014085486757473776, -0.010896601539313185, 0.014547428169312281, 0.0074603930790341846, -0.0099599432025230233, 0.01415510062194276, -0.018897663531530683, 0.0033363892127178788, -0.004454222010575686, -0.0060913854414082827, 0.0081322595710943951, -0.015289256115113807, 0.020411809525601479, 0.0072074244521965741, -0.0096222192878939808, 0.012483625342918751, -0.0166661726882016, -0.00805814550890286, 0.010757968211070279]} + - {mu0: 0, rank: 3, ndensity: 2, num_ms_combs: 19, mus: [0, 0, 0], ns: [1, 3, 3], ls: [4, 4, 4], ms_combs: [0, -4, 4, 0, -3, 3, 0, -2, 2, 0, -1, 1, 0, 0, 0, 1, -4, 3, 1, -3, 2, 1, -2, 1, 1, -1, 0, 2, -4, 2, 2, -3, 1, 2, -2, 0, 2, -1, -1, 3, -4, 1, 3, -3, 0, 3, -2, -1, 4, -4, 0, 4, -3, -1, 4, -2, -2], ctildes: [-0.0022629503659909876, 0.028189560756314954, -0.0033944255489864808, 0.042284341134472422, 0.0017780324304214896, -0.022148940594247452, 0.0014547538067084919, -0.018121860486202467, -0.0014547538067084926, 0.018121860486202478, 0.0071560773884431573, -0.089143218229654014, 0.002704743018936165, -0.033692969500517714, -0.0061337806186655642, 0.076408472768274865, 0.0029095076134169847, -0.036243720972404941, -0.0081142290568084943, 0.10107890850155314, 0.002704743018936165, -0.033692969500517714, 0.0035560648608429809, -0.044297881188494925, -0.0030668903093327821, 0.038204236384137433, 0.0071560773884431564, -0.089143218229653987, -0.0067888510979729633, 0.084568682268944859, 0.002704743018936165, -0.033692969500517714, -0.0045259007319819753, 0.056379121512629908, 0.0071560773884431546, -0.089143218229653973, -0.0040571145284042471, 0.050539454250776564]} + - {mu0: 0, rank: 3, ndensity: 2, num_ms_combs: 1, mus: [0, 0, 0], ns: [2, 2, 2], ls: [0, 0, 0], ms_combs: [0, 0, 0], ctildes: [-0.28814963096908991, -0.13484312984206206]} + - {mu0: 0, rank: 3, ndensity: 2, num_ms_combs: 2, mus: [0, 0, 0], ns: [2, 2, 2], ls: [0, 1, 1], ms_combs: [0, 0, 0, 0, 1, -1], ctildes: [0.68714059417305551, -0.18580401792644732, -1.3742811883461108, 0.37160803585289459]} + - {mu0: 0, rank: 3, ndensity: 2, num_ms_combs: 3, mus: [0, 0, 0], ns: [2, 2, 2], ls: [0, 2, 2], ms_combs: [0, 0, 0, 0, 1, -1, 0, 2, -2], ctildes: [0.84032082659875196, 0.31051911527864562, -1.6806416531975041, -0.62103823055729124, 1.6806416531975046, 0.62103823055729146]} + - {mu0: 0, rank: 3, ndensity: 2, num_ms_combs: 4, mus: [0, 0, 0], ns: [2, 2, 2], ls: [0, 3, 3], ms_combs: [0, 0, 0, 0, 1, -1, 0, 2, -2, 0, 3, -3], ctildes: [-0.19243854683376019, -0.1009873341572381, 0.38487709366752038, 0.20197466831447619, -0.38487709366752032, -0.20197466831447614, 0.38487709366752032, 0.20197466831447614]} + - {mu0: 0, rank: 3, ndensity: 2, num_ms_combs: 5, mus: [0, 0, 0], ns: [2, 2, 2], ls: [0, 4, 4], ms_combs: [0, 0, 0, 0, 1, -1, 0, 2, -2, 0, 3, -3, 0, 4, -4], ctildes: [0.14137187780368099, 0.53979133851723871, -0.28274375560736187, -1.079582677034477, 0.28274375560736187, 1.079582677034477, -0.28274375560736187, -1.079582677034477, 0.28274375560736187, 1.0795826770344772]} + - {mu0: 0, rank: 3, ndensity: 2, num_ms_combs: 4, mus: [0, 0, 0], ns: [2, 2, 2], ls: [1, 1, 2], ms_combs: [-1, 1, 0, 0, 0, 0, 0, 1, -1, 1, 1, -2], ctildes: [0.020407679111265096, 0.96726077722742021, 0.020407679111265092, 0.9672607772274201, -0.070694274170546439, -3.3506896206529064, 0.049988400657054385, 2.3692953524150511]} + - {mu0: 0, rank: 3, ndensity: 2, num_ms_combs: 8, mus: [0, 0, 0], ns: [2, 2, 2], ls: [1, 2, 3], ms_combs: [0, 0, 0, 0, 1, -1, 0, 2, -2, 1, -2, 1, 1, -1, 0, 1, 0, -1, 1, 1, -2, 1, 2, -3], ctildes: [-0.038758813266443044, -0.52492505373875087, 0.073084319177186649, 0.98980817363568951, -0.057778227460658187, -0.78251206883505109, 0.025839208844295358, 0.34995003582583384, -0.044754822545702649, -0.60613124216089309, 0.063292877025733857, 0.85719902324198582, -0.08171075288474039, -1.106639180467158, 0.1000748255331313, 1.3553506607581431]} + - {mu0: 0, rank: 3, ndensity: 2, num_ms_combs: 11, mus: [0, 0, 0], ns: [2, 2, 2], ls: [1, 3, 4], ms_combs: [0, 0, 0, 0, 1, -1, 0, 2, -2, 0, 3, -3, 1, -3, 2, 1, -2, 1, 1, -1, 0, 1, 0, -1, 1, 1, -2, 1, 2, -3, 1, 3, -4], ctildes: [-0.098461740804291806, 0.4280579067377891, 0.19067034118680665, -0.82893057200393205, -0.17054073767471103, 0.74141804305143044, 0.13025263991132841, -0.56626738398153464, -0.049230870402145875, 0.21402895336889441, 0.085270368837355517, -0.37070902152571522, -0.12059051207834434, 0.52426172593572629, 0.15568168166335059, -0.67681897786767464, -0.19067034118680662, 0.82893057200393194, 0.22560419014639455, -0.9808038797251325, -0.26050527982265687, 1.1325347679630693]} + - {mu0: 0, rank: 3, ndensity: 2, num_ms_combs: 5, mus: [0, 0, 0], ns: [2, 2, 2], ls: [2, 2, 2], ms_combs: [-2, 0, 2, -2, 1, 1, -1, -1, 2, -1, 0, 1, 0, 0, 0], ctildes: [0.49180129948880957, 2.1292304909192272, -0.40155407952842559, -1.7385094158426124, -0.20077703976421279, -0.86925470792130621, 0.2459006497444047, 1.0646152454596132, -0.081966883248134892, -0.35487174848653774]} + - {mu0: 0, rank: 3, ndensity: 2, num_ms_combs: 10, mus: [0, 0, 0], ns: [2, 2, 2], ls: [2, 2, 4], ms_combs: [-2, 1, 1, -2, 2, 0, -1, 1, 0, -1, 2, -1, 0, 0, 0, 0, 1, -1, 0, 2, -2, 1, 1, -2, 1, 2, -3, 2, 2, -4], ctildes: [-0.10721799171368267, -0.17204036201133649, 0.047949343576560728, 0.076938788866204172, 0.19179737430624297, 0.3077551554648168, -0.10721799171368267, -0.17204036201133649, 0.14384803072968222, 0.23081636659861263, -0.52525874188895494, -0.84282220418294695, 0.37141401826719445, 0.59596529591235459, 0.30325827602379224, 0.486603626464018, -0.5673442842923766, -0.91035202669504067, 0.401172990690568, 0.64371609134298025]} + - {mu0: 0, rank: 3, ndensity: 2, num_ms_combs: 9, mus: [0, 0, 0], ns: [2, 2, 2], ls: [2, 3, 3], ms_combs: [0, -3, 3, 0, -1, 1, 0, 0, 0, 1, -3, 2, 1, -2, 1, 1, -1, 0, 2, -3, 1, 2, -2, 0, 2, -1, -1], ctildes: [0.27500351452974081, -0.54785035263096848, -0.16500210871784449, 0.32871021157858105, 0.11000140581189634, -0.21914014105238741, -0.55000702905948162, 1.095700705261937, 0.42603361276887836, -0.84872611678144083, -0.15556547997929041, 0.30991095953663933, 0.34785498818607585, -0.69298197249611204, -0.49194124203188139, 0.98002450398406116, 0.26944731522796994, -0.53678152773988186]} + - {mu0: 0, rank: 3, ndensity: 2, num_ms_combs: 13, mus: [0, 0, 0], ns: [2, 2, 2], ls: [2, 4, 4], ms_combs: [0, -4, 4, 0, -3, 3, 0, -2, 2, 0, -1, 1, 0, 0, 0, 1, -4, 3, 1, -3, 2, 1, -2, 1, 1, -1, 0, 2, -4, 2, 2, -3, 1, 2, -2, 0, 2, -1, -1], ctildes: [0.0098330120648687444, 0.45467958578707807, -0.0024582530162171865, -0.11366989644676953, -0.0028094320185339259, -0.12990845308202226, 0.0059700430393845958, 0.27605546279929744, -0.0035117900231674106, -0.16238556635252796, -0.017031276487790417, -0.78752814374759095, 0.016093043605955367, 0.74414414957873287, -0.010948677742150987, -0.50626809240916582, 0.003846973225820761, 0.17788447540906291, 0.0091036002109614825, 0.42095149947793492, -0.013655400316442221, -0.6314272492169023, 0.016321325130125679, 0.75469991297735994, -0.0086020936405568661, -0.3977617791565542]} + - {mu0: 0, rank: 3, ndensity: 2, num_ms_combs: 17, mus: [0, 0, 0], ns: [2, 2, 2], ls: [3, 3, 4], ms_combs: [-3, 1, 2, -3, 2, 1, -3, 3, 0, -2, 1, 1, -2, 2, 0, -2, 3, -1, -1, 1, 0, -1, 2, -1, -1, 3, -2, 0, 0, 0, 0, 1, -1, 0, 2, -2, 0, 3, -3, 1, 1, -2, 1, 2, -3, 1, 3, -4, 2, 2, -4], ctildes: [-0.050618773712407014, -0.69367336133952662, 0.037729006319540487, 0.51703359671198534, -0.020665027833467308, -0.28319096390718346, 0.038966350171875913, 0.53399000253206419, -0.048218398278090373, -0.66077891578342796, 0.037729006319540487, 0.51703359671198534, -0.0068883426111557688, -0.094396987969061166, 0.038966350171875913, 0.53399000253206419, -0.050618773712407014, -0.69367336133952662, 0.020665027833467308, 0.28319096390718346, -0.053356872431954382, -0.73119592467263117, -0.023861918764926916, -0.32700075848776394, 0.10934904896716469, 1.4985057280781506, 0.0435657039096876, 0.59701897248349045, -0.051547636027321327, -0.7064023746472966, -0.089283124609388503, -1.2235248034764232, 0.05763200911825387, 0.78978186458931499]} + - {mu0: 0, rank: 3, ndensity: 2, num_ms_combs: 13, mus: [0, 0, 0], ns: [2, 2, 2], ls: [4, 4, 4], ms_combs: [-4, 0, 4, -4, 1, 3, -4, 2, 2, -3, -1, 4, -3, 0, 3, -3, 1, 2, -2, -2, 4, -2, -1, 3, -2, 0, 2, -2, 1, 1, -1, -1, 2, -1, 0, 1, 0, 0, 0], ctildes: [-0.0016123852247406341, -0.0033207385948887309, 0.0033992065171885855, 0.0070007316492503779, -0.0019271689498780715, -0.0039690417727319058, 0.0016996032585942921, 0.0035003658246251877, -0.0024185778371109512, -0.0049811078923330965, 0.0012847792999187142, 0.002646027848487937, -0.00096358447493903566, -0.0019845208863659525, 0.00064238964995935711, 0.0013230139242439685, 0.0012668741051533551, 0.0026091517531268596, -0.0014568027930808225, -0.0030003135639644477, -0.00072840139654041123, -0.0015001567819822239, 0.0010365333587618363, 0.0021347605252856126, -0.00034551111958727883, -0.0007115868417618712]} + - {mu0: 0, rank: 3, ndensity: 2, num_ms_combs: 1, mus: [0, 0, 0], ns: [2, 2, 3], ls: [0, 0, 0], ms_combs: [0, 0, 0], ctildes: [-0.70348099778171547, -0.29760707540232756]} + - {mu0: 0, rank: 3, ndensity: 2, num_ms_combs: 2, mus: [0, 0, 0], ns: [2, 2, 3], ls: [0, 1, 1], ms_combs: [0, 0, 0, 0, 1, -1], ctildes: [0.40300088886402913, -0.64121403391745957, -0.80600177772805803, 1.2824280678349189]} + - {mu0: 0, rank: 3, ndensity: 2, num_ms_combs: 3, mus: [0, 0, 0], ns: [2, 2, 3], ls: [0, 2, 2], ms_combs: [0, 0, 0, 0, 1, -1, 0, 2, -2], ctildes: [0.07468782253025319, -0.23559145779342167, -0.14937564506050641, 0.47118291558684339, 0.14937564506050643, -0.4711829155868435]} + - {mu0: 0, rank: 3, ndensity: 2, num_ms_combs: 4, mus: [0, 0, 0], ns: [2, 2, 3], ls: [0, 3, 3], ms_combs: [0, 0, 0, 0, 1, -1, 0, 2, -2, 0, 3, -3], ctildes: [0.36367381827599171, 1.240821686649201, -0.72734763655198342, -2.481643373298402, 0.72734763655198331, 2.4816433732984016, -0.72734763655198331, -2.4816433732984016]} + - {mu0: 0, rank: 3, ndensity: 2, num_ms_combs: 5, mus: [0, 0, 0], ns: [2, 2, 3], ls: [0, 4, 4], ms_combs: [0, 0, 0, 0, 1, -1, 0, 2, -2, 0, 3, -3, 0, 4, -4], ctildes: [0.27975745215291731, -0.73075600028684107, -0.55951490430583439, 1.4615120005736815, 0.55951490430583439, -1.4615120005736815, -0.55951490430583439, 1.4615120005736815, 0.5595149043058345, -1.4615120005736817]} + - {mu0: 0, rank: 3, ndensity: 2, num_ms_combs: 2, mus: [0, 0, 0], ns: [2, 2, 3], ls: [1, 1, 0], ms_combs: [0, 0, 0, 1, -1, 0], ctildes: [-0.34293679239463071, -1.5175970391862943, 0.6858735847892613, 3.0351940783725877]} + - {mu0: 0, rank: 3, ndensity: 2, num_ms_combs: 4, mus: [0, 0, 0], ns: [2, 2, 3], ls: [1, 1, 2], ms_combs: [-1, 1, 0, 0, 0, 0, 0, 1, -1, 1, 1, -2], ctildes: [-0.148505934837765, 0.27087390166908115, -0.14850593483776497, 0.2708739016690811, 0.51443964872904391, -0.93833472027052944, -0.36376376412753247, 0.66350284372607371]} + - {mu0: 0, rank: 3, ndensity: 2, num_ms_combs: 5, mus: [0, 0, 0], ns: [2, 2, 3], ls: [1, 2, 1], ms_combs: [0, 0, 0, 0, 1, -1, 1, -2, 1, 1, -1, 0, 1, 0, -1], ctildes: [-0.086790296093995453, 0.49212172520994607, 0.15032520243874681, -0.85237983157207631, -0.21259194005535678, 1.2054471181025246, 0.15032520243874681, -0.85237983157207631, -0.086790296093995439, 0.49212172520994601]} + - {mu0: 0, rank: 3, ndensity: 2, num_ms_combs: 8, mus: [0, 0, 0], ns: [2, 2, 3], ls: [1, 2, 3], ms_combs: [0, 0, 0, 0, 1, -1, 0, 2, -2, 1, -2, 1, 1, -1, 0, 1, 0, -1, 1, 1, -2, 1, 2, -3], ctildes: [0.092743722584420263, 0.012891367242515878, -0.17487924040514005, -0.024308195189197603, 0.13825417879009741, 0.019217315651453008, -0.061829148389613495, -0.0085942448283439169, 0.10709122639952601, 0.014885668695377729, -0.15144986478537734, -0.021051514553995804, 0.19552093469971044, 0.02717738842668959, -0.23946326202316015, -0.033285367093405203]} + - {mu0: 0, rank: 3, ndensity: 2, num_ms_combs: 8, mus: [0, 0, 0], ns: [2, 2, 3], ls: [1, 3, 2], ms_combs: [0, 0, 0, 0, 1, -1, 0, 2, -2, 1, -3, 2, 1, -2, 1, 1, -1, 0, 1, 0, -1, 1, 1, -2], ctildes: [-0.26470120253303764, -0.33508946339514456, 0.49912537413158586, 0.63185075165564852, -0.39459325505987469, -0.49952187913031415, 0.68345556608768776, 0.86519727414598357, -0.55803913292662044, -0.70643061616818392, 0.43225525367136564, 0.54719880233408402, -0.30565062107453395, -0.38692798378758808, 0.17646746835535851, 0.22339297559676316]} + - {mu0: 0, rank: 3, ndensity: 2, num_ms_combs: 11, mus: [0, 0, 0], ns: [2, 2, 3], ls: [1, 3, 4], ms_combs: [0, 0, 0, 0, 1, -1, 0, 2, -2, 0, 3, -3, 1, -3, 2, 1, -2, 1, 1, -1, 0, 1, 0, -1, 1, 1, -2, 1, 2, -3, 1, 3, -4], ctildes: [-0.12830067980793208, -0.54768836740089233, 0.24845319810160543, 1.060593962927592, -0.22222329607296459, -0.94862407905279511, 0.16972584590615716, 0.72452360805286764, -0.064150339903965986, -0.273844183700446, 0.1111116480364823, 0.47431203952639756, -0.15713559959081919, -0.67077851909507513, 0.20286118677051981, 0.8659713444829662, -0.2484531981016054, -1.0605939629275918, 0.29397378846707034, 1.254911700430686, -0.33945169181231438, -1.4490472161057355]} + - {mu0: 0, rank: 3, ndensity: 2, num_ms_combs: 11, mus: [0, 0, 0], ns: [2, 2, 3], ls: [1, 4, 3], ms_combs: [0, 0, 0, 0, 1, -1, 0, 2, -2, 0, 3, -3, 1, -4, 3, 1, -3, 2, 1, -2, 1, 1, -1, 0, 1, 0, -1, 1, 1, -2, 1, 2, -3], ctildes: [0.035098861611087398, -0.21607975494225454, -0.067968653245290153, 0.4184366461719658, 0.060793011598232173, -0.37426111404701701, -0.046431429562204536, 0.28584664746649258, 0.092862859124409072, -0.57169329493298515, -0.080421595069793822, 0.49510091658519478, 0.067968653245290153, -0.4184366461719658, -0.055496172985041607, 0.34165209093427479, 0.042987150749862403, -0.26464257167707755, -0.030396505799116087, 0.1871305570235085, 0.017549430805543692, -0.10803987747112724]} + - {mu0: 0, rank: 3, ndensity: 2, num_ms_combs: 3, mus: [0, 0, 0], ns: [2, 2, 3], ls: [2, 2, 0], ms_combs: [0, 0, 0, 1, -1, 0, 2, -2, 0], ctildes: [0.35827452290334949, -0.70586778558629637, -0.71654904580669909, 1.411735571172593, 0.7165490458066992, -1.4117355711725932]} + - {mu0: 0, rank: 3, ndensity: 2, num_ms_combs: 8, mus: [0, 0, 0], ns: [2, 2, 3], ls: [2, 2, 2], ms_combs: [-2, 1, 1, -2, 2, 0, -1, 1, 0, -1, 2, -1, 0, 0, 0, 0, 1, -1, 0, 2, -2, 1, 1, -2], ctildes: [0.01327397277927423, 0.21458116772549887, -0.010838153389605112, -0.1752047897793487, -0.0054190766948025559, -0.087602394889674348, 0.01327397277927423, 0.21458116772549887, 0.0054190766948025559, 0.087602394889674348, -0.010838153389605112, -0.1752047897793487, -0.021676306779210234, -0.35040957955869756, 0.01327397277927423, 0.21458116772549887]} + - {mu0: 0, rank: 3, ndensity: 2, num_ms_combs: 10, mus: [0, 0, 0], ns: [2, 2, 3], ls: [2, 2, 4], ms_combs: [-2, 1, 1, -2, 2, 0, -1, 1, 0, -1, 2, -1, 0, 0, 0, 0, 1, -1, 0, 2, -2, 1, 1, -2, 1, 2, -3, 2, 2, -4], ctildes: [-0.014605050904932877, -0.21852324087865074, 0.0065315773276549477, 0.097726564253644801, 0.026126309310619798, 0.39090625701457932, -0.014605050904932877, -0.21852324087865074, 0.019594731982964849, 0.29317969276093447, -0.071549844768918514, -1.0705408741839855, 0.050593380428947093, 0.75698671167287046, 0.041309322137811029, 0.6180770618886211, -0.077282665159782493, -1.1563163021055471, 0.054647096602651546, 0.81763909841538496]} + - {mu0: 0, rank: 3, ndensity: 2, num_ms_combs: 8, mus: [0, 0, 0], ns: [2, 2, 3], ls: [2, 3, 1], ms_combs: [0, 0, 0, 0, 1, -1, 1, -2, 1, 1, -1, 0, 1, 0, -1, 2, -3, 1, 2, -2, 0, 2, -1, -1], ctildes: [-0.23215921027003841, -0.34331262939946222, 0.37911440283273462, 0.56062717618793667, -0.48943458949285046, -0.72376657226905672, 0.43776360505828726, 0.64735650217425011, -0.26807436508851507, -0.3964232779999553, 0.59943250336301623, 0.88642939747119709, -0.34608251717764876, -0.51178025124759352, 0.1547728068466922, 0.22887508626630806]} + - {mu0: 0, rank: 3, ndensity: 2, num_ms_combs: 14, mus: [0, 0, 0], ns: [2, 2, 3], ls: [2, 3, 3], ms_combs: [0, 0, 0, 0, 1, -1, 0, 3, -3, 1, -3, 2, 1, -2, 1, 1, -1, 0, 1, 0, -1, 1, 1, -2, 1, 2, -3, 2, -3, 1, 2, -2, 0, 2, -1, -1, 2, 0, -2, 2, 1, -3], ctildes: [0.013713710181816267, 0.67533905239673631, -0.020570565272724398, -1.0130085785951042, 0.034284275454540665, 1.6883476309918406, -0.034284275454540665, -1.6883476309918406, 0.026556485574444743, 1.3077884514880289, -0.0096970574647892802, -0.47753682354982913, -0.0096970574647892872, -0.47753682354982951, 0.02655648557444475, 1.3077884514880291, -0.034284275454540665, -1.6883476309918406, 0.021683279672990611, 1.0678047992167408, -0.030664788190272175, -1.5101040290193948, 0.033591592425860177, 1.654236081746717, -0.030664788190272171, -1.5101040290193946, 0.021683279672990607, 1.0678047992167408]} + - {mu0: 0, rank: 3, ndensity: 2, num_ms_combs: 13, mus: [0, 0, 0], ns: [2, 2, 3], ls: [2, 4, 2], ms_combs: [0, 0, 0, 0, 1, -1, 0, 2, -2, 1, -3, 2, 1, -2, 1, 1, -1, 0, 1, 0, -1, 1, 1, -2, 2, -4, 2, 2, -3, 1, 2, -2, 0, 2, -1, -1, 2, 0, -2], ctildes: [-0.33616277186711713, -0.096095881339709213, 0.61374644381694377, 0.17544627297699505, -0.43398427235208931, -0.12405924935593932, 0.66292192615792689, 0.18950363361099787, -0.70869334910377602, -0.20258790586316933, 0.61374644381694377, 0.17544627297699503, -0.4482170291561563, -0.12812784178627901, 0.25056093646654265, 0.071625641011114474, -0.93751317876703555, -0.26799860877165504, 0.662921926157927, 0.1895036336109979, -0.4339842723520892, -0.12405924935593927, 0.25056093646654265, 0.071625641011114474, -0.11205425728903906, -0.032031960446569745]} + - {mu0: 0, rank: 3, ndensity: 2, num_ms_combs: 20, mus: [0, 0, 0], ns: [2, 2, 3], ls: [2, 4, 4], ms_combs: [0, 0, 0, 0, 1, -1, 0, 2, -2, 0, 3, -3, 0, 4, -4, 1, -4, 3, 1, -3, 2, 1, -2, 1, 1, -1, 0, 1, 0, -1, 1, 1, -2, 1, 2, -3, 1, 3, -4, 2, -4, 2, 2, -3, 1, 2, -2, 0, 2, -1, -1, 2, 0, -2, 2, 1, -3, 2, 2, -4], ctildes: [-0.054134793464101799, 0.22694417492001054, 0.092029148888973025, -0.3858050973640178, -0.043307834771281396, 0.18155533993600828, -0.037894355424871247, 0.15886092244400732, 0.15157742169948496, -0.63544368977602927, -0.13126989783190054, 0.55031037802055904, 0.1240383943900235, -0.51999443005012325, -0.084387791463364656, 0.35377095729893093, 0.029650847526171798, -0.12430244389808801, 0.029650847526171774, -0.12430244389808789, -0.08438779146336467, 0.35377095729893099, 0.12403839439002348, -0.51999443005012314, -0.13126989783190057, 0.55031037802055927, 0.070166711840541629, -0.29415327013414078, -0.10525006776081243, 0.44122990520121114, 0.1257978921221066, -0.52737060599039021, -0.1326026213180031, 0.55589742865095693, 0.12579789212210654, -0.52737060599038987, -0.10525006776081243, 0.44122990520121114, 0.070166711840541643, -0.29415327013414083]} + - {mu0: 0, rank: 3, ndensity: 2, num_ms_combs: 4, mus: [0, 0, 0], ns: [2, 2, 3], ls: [3, 3, 0], ms_combs: [0, 0, 0, 1, -1, 0, 2, -2, 0, 3, -3, 0], ctildes: [0.15171011661381173, -0.38208606995367372, -0.30342023322762346, 0.76417213990734745, 0.3034202332276234, -0.76417213990734734, -0.3034202332276234, 0.76417213990734734]} + - {mu0: 0, rank: 3, ndensity: 2, num_ms_combs: 12, mus: [0, 0, 0], ns: [2, 2, 3], ls: [3, 3, 2], ms_combs: [-3, 1, 2, -3, 2, 1, -3, 3, 0, -2, 1, 1, -2, 3, -1, -1, 1, 0, -1, 2, -1, -1, 3, -2, 0, 0, 0, 0, 1, -1, 0, 2, -2, 1, 1, -2], ctildes: [-0.010092002603849961, -0.65998790297143584, 0.015956857190257918, 1.0435325007739737, -0.015956857190257918, -1.0435325007739735, -0.0123601284311358, -0.80831679934475564, 0.015956857190257918, 1.0435325007739737, 0.009574114314154749, 0.62611950046438403, -0.0123601284311358, -0.80831679934475564, -0.010092002603849961, -0.65998790297143584, -0.0063827428761031677, -0.41741300030958944, 0.0090265615405253595, 0.59031112614866654, 0.028544493907738416, 1.8667276867687665, -0.015634463205837116, -1.0224488627626913]} + - {mu0: 0, rank: 3, ndensity: 2, num_ms_combs: 17, mus: [0, 0, 0], ns: [2, 2, 3], ls: [3, 3, 4], ms_combs: [-3, 1, 2, -3, 2, 1, -3, 3, 0, -2, 1, 1, -2, 2, 0, -2, 3, -1, -1, 1, 0, -1, 2, -1, -1, 3, -2, 0, 0, 0, 0, 1, -1, 0, 2, -2, 0, 3, -3, 1, 1, -2, 1, 2, -3, 1, 3, -4, 2, 2, -4], ctildes: [-0.015417485296495574, 0.54765416290203484, 0.011491515055022534, -0.40819731213656446, -0.0062941620155460669, 0.22357887577017382, 0.011868385714878421, -0.42158437116574521, -0.014686378036274152, 0.52168404346373876, 0.011491515055022534, -0.40819731213656446, -0.0020980540051820222, 0.074526291923391275, 0.011868385714878421, -0.42158437116574521, -0.015417485296495574, 0.54765416290203484, 0.0062941620155460669, -0.22357887577017382, -0.016251456443027477, 0.57727817494777312, -0.0072678722679972779, 0.25816664822204749, 0.033305574809367883, -1.1830682073905683, 0.013269258620827798, -0.47134565608905449, -0.015700398532679922, 0.5577037013680558, -0.027193887957681486, 0.96597114633869341, 0.017553579196455137, -0.62353169378110784]} + - {mu0: 0, rank: 3, ndensity: 2, num_ms_combs: 11, mus: [0, 0, 0], ns: [2, 2, 3], ls: [3, 4, 1], ms_combs: [0, 0, 0, 0, 1, -1, 1, -2, 1, 1, -1, 0, 1, 0, -1, 2, -3, 1, 2, -2, 0, 2, -1, -1, 3, -4, 1, 3, -3, 0, 3, -2, -1], ctildes: [0.06743779034895267, -0.13218852991270044, -0.10662850893580586, 0.20900841753671603, 0.13059271946326051, -0.25598198745576495, -0.13059271946326048, 0.25598198745576495, 0.082594087867860988, -0.16189722406737353, -0.15451938948731905, 0.30288197216494211, 0.11680567923456416, -0.22895724998663558, -0.058402839617282057, 0.11447862499331775, 0.17842362223104058, -0.34973797632422815, -0.089211811115520262, 0.17486898816211402, 0.033718895174476342, -0.066094264956350235]} + - {mu0: 0, rank: 3, ndensity: 2, num_ms_combs: 22, mus: [0, 0, 0], ns: [2, 2, 3], ls: [3, 4, 3], ms_combs: [0, 0, 0, 0, 1, -1, 0, 2, -2, 0, 3, -3, 1, -4, 3, 1, -3, 2, 1, -2, 1, 1, -1, 0, 1, 0, -1, 1, 1, -2, 1, 2, -3, 2, -4, 2, 2, -3, 1, 2, -2, 0, 2, -1, -1, 2, 0, -2, 2, 1, -3, 3, -4, 1, 3, -3, 0, 3, -2, -1, 3, -1, -2, 3, 0, -3], ctildes: [-0.015568798910964124, -0.10185573441625365, 0.020099232967538738, 0.13149518770329199, 0.0089886502422042869, 0.058806435683731062, -0.041191170130384307, -0.26948494287124991, 0.033632449575871061, 0.22003353446587909, 0.019417703816135671, 0.12703642035462009, -0.032821909994530434, -0.21473074233637499, 0.020099232967538745, 0.13149518770329205, 0.005189599636988047, 0.033951911472084588, -0.029356808759659916, -0.19206101468925058, 0.03813561323986054, 0.24949457669626088, -0.04341930569983643, -0.28406207153116841, 0.019417703816135674, 0.12703642035462009, 0.0089886502422042852, 0.058806435683731055, -0.02935680875965992, -0.19206101468925063, 0.036327197458916292, 0.23766338030459186, -0.028424607855989718, -0.1859622778367914, 0.033632449575871061, 0.22003353446587909, -0.041191170130384307, -0.26948494287124986, 0.038135613239860547, 0.24949457669626091, -0.028424607855989711, -0.18596227783679137, 0.015568798910964124, 0.10185573441625365]} + - {mu0: 0, rank: 3, ndensity: 2, num_ms_combs: 5, mus: [0, 0, 0], ns: [2, 2, 3], ls: [4, 4, 0], ms_combs: [0, 0, 0, 1, -1, 0, 2, -2, 0, 3, -3, 0, 4, -4, 0], ctildes: [-0.040360597856338075, -0.29659243088369563, 0.080721195712676108, 0.59318486176739105, -0.080721195712676108, -0.59318486176739105, 0.080721195712676108, 0.59318486176739105, -0.080721195712676122, -0.59318486176739116]} + - {mu0: 0, rank: 3, ndensity: 2, num_ms_combs: 18, mus: [0, 0, 0], ns: [2, 2, 3], ls: [4, 4, 2], ms_combs: [-4, 2, 2, -4, 3, 1, -4, 4, 0, -3, 1, 2, -3, 2, 1, -3, 3, 0, -3, 4, -1, -2, 1, 1, -2, 2, 0, -2, 3, -1, -2, 4, -2, -1, 1, 0, -1, 2, -1, -1, 3, -2, 0, 0, 0, 0, 1, -1, 0, 2, -2, 1, 1, -2], ctildes: [0.0032697737093376699, -0.31674959675221048, -0.0061171864763112614, 0.59258423422278783, 0.007063518517562887, -0.68425733429210933, -0.0049046605640065046, 0.47512439512831578, 0.0057801979070453919, -0.55993946950398099, -0.0017658796293907211, 0.17106433357302728, -0.0061171864763112614, 0.59258423422278783, -0.0039324770204858103, 0.38094700771464923, -0.002018148147875111, 0.19550209551203129, 0.0057801979070453919, -0.55993946950398099, 0.0032697737093376699, -0.31674959675221048, 0.0042885648142346093, -0.4154419529630663, -0.0039324770204858103, 0.38094700771464923, -0.0049046605640065046, 0.47512439512831578, -0.0025226851848438887, 0.2443776193900391, 0.0027634631624461744, -0.26770226937867253, 0.011724381250349473, -1.1357645401001233, -0.0061792914845461913, 0.59860047206167222]} + - {mu0: 0, rank: 3, ndensity: 2, num_ms_combs: 25, mus: [0, 0, 0], ns: [2, 2, 3], ls: [4, 4, 4], ms_combs: [-4, 1, 3, -4, 2, 2, -4, 3, 1, -4, 4, 0, -3, 1, 2, -3, 2, 1, -3, 3, 0, -3, 4, -1, -2, 1, 1, -2, 2, 0, -2, 3, -1, -2, 4, -2, -1, 1, 0, -1, 2, -1, -1, 3, -2, -1, 4, -3, 0, 0, 0, 0, 1, -1, 0, 2, -2, 0, 3, -3, 0, 4, -4, 1, 1, -2, 1, 2, -3, 1, 3, -4, 2, 2, -4], ctildes: [0.02738860723598387, -0.14875145087473937, -0.031055761501216046, 0.16866829121768642, 0.027388607235983856, -0.14875145087473929, -0.017322076161095566, 0.094078678003764502, 0.01035192050040535, -0.056222763739228812, 0.01035192050040535, -0.056222763739228812, -0.025983114241643356, 0.14111801700564677, 0.027388607235983856, -0.14875145087473929, -0.023475949059414747, 0.12750124360691947, 0.013610202698003665, -0.073918961288672147, 0.01035192050040535, -0.056222763739228812, -0.031055761501216046, 0.16866829121768642, 0.011135620389275723, -0.060479150145277195, -0.023475949059414747, 0.12750124360691947, 0.01035192050040535, -0.056222763739228812, 0.02738860723598387, -0.14875145087473937, -0.011135620389275723, 0.060479150145277195, 0.02227124077855144, -0.12095830029055434, 0.027220405396007303, -0.14783792257734413, -0.051966228483286685, 0.28223603401129344, -0.034644152322191132, 0.188157356007529, -0.023475949059414743, 0.12750124360691945, 0.020703841000810699, -0.11244552747845762, 0.054777214471967747, -0.2975029017494788, -0.031055761501216052, 0.16866829121768645]} + - {mu0: 0, rank: 3, ndensity: 2, num_ms_combs: 1, mus: [0, 0, 0], ns: [2, 3, 3], ls: [0, 0, 0], ms_combs: [0, 0, 0], ctildes: [-1.1204875063297361, -0.589140228246818]} + - {mu0: 0, rank: 3, ndensity: 2, num_ms_combs: 2, mus: [0, 0, 0], ns: [2, 3, 3], ls: [0, 1, 1], ms_combs: [0, 0, 0, 0, 1, -1], ctildes: [-1.0141636436158097, 0.83392584616534815, 2.0283272872316189, -1.6678516923306961]} + - {mu0: 0, rank: 3, ndensity: 2, num_ms_combs: 3, mus: [0, 0, 0], ns: [2, 3, 3], ls: [0, 2, 2], ms_combs: [0, 0, 0, 0, 1, -1, 0, 2, -2], ctildes: [-0.55605795027140004, -1.0114675279739436, 1.1121159005428003, 2.0229350559478871, -1.1121159005428005, -2.0229350559478876]} + - {mu0: 0, rank: 3, ndensity: 2, num_ms_combs: 4, mus: [0, 0, 0], ns: [2, 3, 3], ls: [0, 3, 3], ms_combs: [0, 0, 0, 0, 1, -1, 0, 2, -2, 0, 3, -3], ctildes: [-0.2340850935398833, -0.37009359117774338, 0.4681701870797666, 0.74018718235548675, -0.46817018707976654, -0.74018718235548664, 0.46817018707976654, 0.74018718235548664]} + - {mu0: 0, rank: 3, ndensity: 2, num_ms_combs: 5, mus: [0, 0, 0], ns: [2, 3, 3], ls: [0, 4, 4], ms_combs: [0, 0, 0, 0, 1, -1, 0, 2, -2, 0, 3, -3, 0, 4, -4], ctildes: [0.13640125591742525, -0.033642408146839102, -0.27280251183485038, 0.067284816293678162, 0.27280251183485038, -0.067284816293678162, -0.27280251183485038, 0.067284816293678162, 0.27280251183485038, -0.067284816293678176]} + - {mu0: 0, rank: 3, ndensity: 2, num_ms_combs: 2, mus: [0, 0, 0], ns: [2, 3, 3], ls: [1, 0, 1], ms_combs: [0, 0, 0, 1, 0, -1], ctildes: [-0.41757018846019495, -0.86695679008275828, 0.83514037692038978, 1.7339135801655163]} + - {mu0: 0, rank: 3, ndensity: 2, num_ms_combs: 5, mus: [0, 0, 0], ns: [2, 3, 3], ls: [1, 1, 2], ms_combs: [0, 0, 0, 0, 1, -1, 1, -1, 0, 1, 0, -1, 1, 1, -2], ctildes: [0.15846896233345861, 0.014083319529278826, -0.27447629418426905, -0.024393024963937939, 0.15846896233345864, 0.014083319529278828, -0.27447629418426905, -0.024393024963937939, 0.38816809778530087, 0.034496946731306516]} + - {mu0: 0, rank: 3, ndensity: 2, num_ms_combs: 8, mus: [0, 0, 0], ns: [2, 3, 3], ls: [1, 2, 3], ms_combs: [0, 0, 0, 0, 1, -1, 0, 2, -2, 1, -2, 1, 1, -1, 0, 1, 0, -1, 1, 1, -2, 1, 2, -3], ctildes: [0.075514062205669752, -0.13991083449612007, -0.14239068122819162, 0.2638183995564673, 0.11256971756601682, -0.20856675781469794, -0.050342708137113161, 0.093273889664080026, 0.087196128284091137, -0.16155511591776017, -0.12331394720578594, 0.22847343600165396, 0.15919762129436979, -0.2949579375617305, -0.19497647021801986, 0.36124822130496997]} + - {mu0: 0, rank: 3, ndensity: 2, num_ms_combs: 11, mus: [0, 0, 0], ns: [2, 3, 3], ls: [1, 3, 4], ms_combs: [0, 0, 0, 0, 1, -1, 0, 2, -2, 0, 3, -3, 1, -3, 2, 1, -2, 1, 1, -1, 0, 1, 0, -1, 1, 1, -2, 1, 2, -3, 1, 3, -4], ctildes: [0.15855035422017544, -0.77075130151125282, -0.30703144071501304, 1.4925534774103861, 0.27461726906738632, -1.3349804142173283, -0.20974240377389211, 1.0196081332390678, 0.079275177110087677, -0.38537565075562613, -0.13730863453369316, 0.66749020710866414, 0.19418373318847965, -0.94397370364429911, -0.25069012158112192, 1.2186648111573681, 0.30703144071501298, -1.4925534774103859, -0.36328449983800737, 1.7660130905805229, 0.41948480754778428, -2.0392162664781361]} + - {mu0: 0, rank: 3, ndensity: 2, num_ms_combs: 3, mus: [0, 0, 0], ns: [2, 3, 3], ls: [2, 0, 2], ms_combs: [0, 0, 0, 1, 0, -1, 2, 0, -2], ctildes: [-1.2749450803928966, -1.2321525410053744, 2.5498901607857936, 2.4643050820107488, -2.549890160785794, -2.4643050820107497]} + - {mu0: 0, rank: 3, ndensity: 2, num_ms_combs: 4, mus: [0, 0, 0], ns: [2, 3, 3], ls: [2, 1, 1], ms_combs: [0, -1, 1, 0, 0, 0, 1, -1, 0, 2, -1, -1], ctildes: [-0.59727149511888922, -1.0534445369583896, -0.59727149511888933, -1.0534445369583898, 2.0690091509170863, 3.6492389219356025, -1.4630104009504923, -2.5804015878705502]} + - {mu0: 0, rank: 3, ndensity: 2, num_ms_combs: 8, mus: [0, 0, 0], ns: [2, 3, 3], ls: [2, 1, 3], ms_combs: [0, 0, 0, 0, 1, -1, 1, -1, 0, 1, 0, -1, 1, 1, -2, 2, -1, -1, 2, 0, -2, 2, 1, -3], ctildes: [0.098314597981536189, 0.30722071262442163, -0.16054706621441642, -0.5016893228960394, 0.11352391921981948, 0.35474792226867691, -0.18538378379299597, -0.57930093124717885, 0.20726537124363131, 0.64767813084881176, -0.065543065321024108, -0.20481380841628105, 0.14655874951151895, 0.45797759834942287, -0.25384720044771125, -0.7932404690695728]} + - {mu0: 0, rank: 3, ndensity: 2, num_ms_combs: 7, mus: [0, 0, 0], ns: [2, 3, 3], ls: [2, 2, 2], ms_combs: [0, -2, 2, 0, -1, 1, 0, 0, 0, 1, -2, 1, 1, -1, 0, 2, -2, 0, 2, -1, -1], ctildes: [0.13851787510498439, -0.28876019947058235, 0.069258937552492156, -0.14438009973529109, -0.069258937552492156, 0.14438009973529109, -0.33929811426178047, 0.70731514672721574, 0.13851787510498431, -0.28876019947058218, 0.27703575020996868, -0.57752039894116447, -0.16964905713089024, 0.35365757336360787]} + - {mu0: 0, rank: 3, ndensity: 2, num_ms_combs: 13, mus: [0, 0, 0], ns: [2, 3, 3], ls: [2, 2, 4], ms_combs: [0, 0, 0, 0, 1, -1, 0, 2, -2, 1, -2, 1, 1, -1, 0, 1, 0, -1, 1, 1, -2, 1, 2, -3, 2, -2, 0, 2, -1, -1, 2, 0, -2, 2, 1, -3, 2, 2, -4], ctildes: [-0.1294726477613666, -0.083696431446235869, 0.23638363252940403, 0.15280807828596032, -0.16714846952305051, -0.10805162837608735, 0.096503213873767188, 0.062383636729311451, -0.17263019701515547, -0.11159524192831449, 0.23638363252940403, 0.15280807828596032, -0.27295230774574591, -0.17644757026549718, 0.25532350462846615, 0.16505158866555292, -0.043157549253788854, -0.027898810482078613, 0.096503213873767188, 0.062383636729311451, -0.16714846952305051, -0.10805162837608735, 0.25532350462846615, 0.16505158866555292, -0.36108196303820655, -0.23341819518205037]} + - {mu0: 0, rank: 3, ndensity: 2, num_ms_combs: 9, mus: [0, 0, 0], ns: [2, 3, 3], ls: [2, 3, 3], ms_combs: [0, -3, 3, 0, -1, 1, 0, 0, 0, 1, -3, 2, 1, -2, 1, 1, -1, 0, 2, -3, 1, 2, -2, 0, 2, -1, -1], ctildes: [-0.063605569316151994, -0.54355002774521688, 0.038163341589691195, 0.32613001664713004, -0.025442227726460803, -0.21742001109808676, 0.12721113863230399, 1.0871000554904338, -0.098537324274999283, -0.84206408211512185, 0.035980743507745661, 0.30747832842622308, -0.080455388364303507, -0.68754244396904063, 0.11378110139079256, 0.97233184896816149, -0.06232047584951951, -0.53256808706056824]} + - {mu0: 0, rank: 3, ndensity: 2, num_ms_combs: 13, mus: [0, 0, 0], ns: [2, 3, 3], ls: [2, 4, 4], ms_combs: [0, -4, 4, 0, -3, 3, 0, -2, 2, 0, -1, 1, 0, 0, 0, 1, -4, 3, 1, -3, 2, 1, -2, 1, 1, -1, 0, 2, -4, 2, 2, -3, 1, 2, -2, 0, 2, -1, -1], ctildes: [-0.01087149823570609, -0.075124885528610158, 0.002717874558926523, 0.018781221382152543, 0.0031061423530588817, 0.021464253008174325, -0.0066005525002501278, -0.045611537642370463, 0.0038826779413236056, 0.02683031626021793, 0.018829987298638353, 0.13012011864834866, -0.017792665565250717, -0.12295195518205305, 0.012104991834838945, 0.08364864770250989, -0.0042532605839813165, -0.029391138881438014, -0.010065051581258529, -0.06955212901549937, 0.015097577371887791, 0.10432819352324903, -0.01804505640651986, -0.12469604165916247, 0.0095105797918026737, 0.065720584475032498]} + - {mu0: 0, rank: 3, ndensity: 2, num_ms_combs: 4, mus: [0, 0, 0], ns: [2, 3, 3], ls: [3, 0, 3], ms_combs: [0, 0, 0, 1, 0, -1, 2, 0, -2, 3, 0, -3], ctildes: [-0.29125482494002963, 0.81122515527316241, 0.58250964988005927, -1.6224503105463248, -0.58250964988005915, 1.6224503105463246, 0.58250964988005915, -1.6224503105463246]} + - {mu0: 0, rank: 3, ndensity: 2, num_ms_combs: 8, mus: [0, 0, 0], ns: [2, 3, 3], ls: [3, 1, 2], ms_combs: [0, 0, 0, 0, 1, -1, 1, -1, 0, 1, 0, -1, 1, 1, -2, 2, -1, -1, 2, 0, -2, 3, -1, -2], ctildes: [0.24404222803039846, 0.55519987807894478, 0.28179569209397332, 0.64108959812585264, -0.39851928957760979, -0.90663760436589791, -0.46017043822978282, -1.0468949298761772, -0.16269481868693239, -0.37013325205263004, 0.51448619055883116, 1.1704641142515038, 0.36379667417098399, 0.82764311232274457, -0.63011432328872441, -1.4335199210774283]} + - {mu0: 0, rank: 3, ndensity: 2, num_ms_combs: 11, mus: [0, 0, 0], ns: [2, 3, 3], ls: [3, 1, 4], ms_combs: [0, 0, 0, 0, 1, -1, 1, -1, 0, 1, 0, -1, 1, 1, -2, 2, -1, -1, 2, 0, -2, 2, 1, -3, 3, -1, -2, 3, 0, -3, 3, 1, -4], ctildes: [0.011004935322927895, 0.72965809372425994, -0.017400330561646374, -1.1536907446726357, 0.013478238096752079, 0.89364500815815062, -0.021310965615894729, -1.4129768227097539, 0.021310965615894725, 1.4129768227097537, -0.0095305535566602585, -0.63190244524213579, 0.019061107113320517, 1.2638048904842716, -0.025215474567705169, -1.6718567229643011, 0.005502467661463944, 0.36482904686212975, -0.014558161029408743, -0.96524692904992482, 0.029116322058817493, 1.9304938580998501]} + - {mu0: 0, rank: 3, ndensity: 2, num_ms_combs: 14, mus: [0, 0, 0], ns: [2, 3, 3], ls: [3, 2, 3], ms_combs: [0, 0, 0, 0, 1, -1, 0, 2, -2, 1, -2, 1, 1, -1, 0, 1, 0, -1, 1, 1, -2, 1, 2, -3, 2, -2, 0, 2, -1, -1, 2, 1, -3, 3, -2, -1, 3, -1, -2, 3, 0, -3], ctildes: [-0.051792275375516707, -0.26760425442083502, 0.036622669131108919, 0.18922478297534265, 0.11581104844904379, 0.59838130395313571, -0.12686464728772995, -0.65549387632897538, 0.036622669131108884, 0.18922478297534248, 0.077688413063275036, 0.40140638163125247, -0.10029530999578236, -0.51821341037307334, -0.081890777694642647, -0.42311947776051084, 0.1158110484490438, 0.59838130395313582, -0.10029530999578234, -0.51821341037307322, 0.12948068843879176, 0.66901063605208755, -0.081890777694642661, -0.4231194777605109, 0.12948068843879176, 0.66901063605208755, -0.12948068843879176, -0.66901063605208755]} + - {mu0: 0, rank: 3, ndensity: 2, num_ms_combs: 22, mus: [0, 0, 0], ns: [2, 3, 3], ls: [3, 3, 4], ms_combs: [0, 0, 0, 0, 1, -1, 0, 2, -2, 0, 3, -3, 1, -3, 2, 1, -2, 1, 1, -1, 0, 1, 0, -1, 1, 1, -2, 1, 2, -3, 1, 3, -4, 2, -3, 1, 2, -2, 0, 2, -1, -1, 2, 0, -2, 2, 1, -3, 2, 2, -4, 3, -3, 0, 3, -2, -1, 3, -1, -2, 3, 0, -3, 3, 1, -4], ctildes: [0.035220903317348604, -0.13479512312567737, -0.045469990662157546, 0.1740197556719088, -0.020334798011373001, 0.077824000622058576, 0.093185751128754185, -0.35663437373487356, -0.086273241407403362, 0.33017927147354204, 0.066413172200567847, -0.25417212168811198, -0.011740301105782868, 0.044931707708559122, -0.045469990662157546, 0.1740197556719088, 0.074252183820934528, -0.28417307103998363, -0.043928184355402709, 0.16811905604809776, -0.076085847187809821, 0.29119074679582524, 0.06430427747540117, -0.24610109859206572, -0.082182107740480057, 0.31452195395991378, 0.066413172200567847, -0.25417212168811198, -0.020334798011373001, 0.077824000622058576, -0.043928184355402709, 0.16811905604809776, 0.098226406346823281, -0.37592563763664394, -0.035220903317348604, 0.13479512312567737, 0.06430427747540117, -0.24610109859206572, -0.086273241407403362, 0.33017927147354204, 0.093185751128754185, -0.35663437373487356, -0.076085847187809821, 0.29119074679582524]} + - {mu0: 0, rank: 3, ndensity: 2, num_ms_combs: 5, mus: [0, 0, 0], ns: [2, 3, 3], ls: [4, 0, 4], ms_combs: [0, 0, 0, 1, 0, -1, 2, 0, -2, 3, 0, -3, 4, 0, -4], ctildes: [-0.00536140873123907, -0.63926736620453029, 0.010722817462478135, 1.2785347324090599, -0.010722817462478135, -1.2785347324090599, 0.010722817462478135, 1.2785347324090599, -0.010722817462478136, -1.2785347324090601]} + - {mu0: 0, rank: 3, ndensity: 2, num_ms_combs: 11, mus: [0, 0, 0], ns: [2, 3, 3], ls: [4, 1, 3], ms_combs: [0, 0, 0, 0, 1, -1, 1, -1, 0, 1, 0, -1, 1, 1, -2, 2, -1, -1, 2, 0, -2, 2, 1, -3, 3, -1, -2, 3, 0, -3, 4, -1, -3], ctildes: [-0.078883628181238244, 0.57522011994291689, -0.096612319051732537, 0.70449789182134193, 0.12472596757527926, -0.90950286748743092, 0.15275748911717688, -1.1139089724711748, 0.06831522594763835, -0.4981552366384977, -0.15275748911717688, 1.1139089724711748, -0.1366304518952767, 0.9963104732769954, -0.039441814090619108, 0.28761005997145839, 0.18074509861663793, -1.3179948704499966, 0.10435323134102134, -0.76094469324485137, -0.20870646268204268, 1.5218893864897027]} + - {mu0: 0, rank: 3, ndensity: 2, num_ms_combs: 9, mus: [0, 0, 0], ns: [2, 3, 3], ls: [4, 2, 2], ms_combs: [0, -2, 2, 0, -1, 1, 0, 0, 0, 1, -2, 1, 1, -1, 0, 2, -2, 0, 2, -1, -1, 3, -2, -1, 4, -2, -2], ctildes: [0.044426446061644975, 0.040354002222968033, 0.17770578424657993, 0.16141600889187216, 0.1332793381849349, 0.12106200666890407, -0.19868110678513193, -0.1804685842694683, -0.48666733315499, -0.44205594606266418, 0.34412577145586609, 0.31258075712474448, 0.28097751580283081, 0.25522111945615389, -0.52566079876052696, -0.47747499343691641, 0.37169831540750564, 0.33762580570624579]} + - {mu0: 0, rank: 3, ndensity: 2, num_ms_combs: 20, mus: [0, 0, 0], ns: [2, 3, 3], ls: [4, 2, 4], ms_combs: [0, 0, 0, 0, 1, -1, 0, 2, -2, 1, -2, 1, 1, -1, 0, 1, 0, -1, 1, 1, -2, 1, 2, -3, 2, -2, 0, 2, -1, -1, 2, 0, -2, 2, 1, -3, 2, 2, -4, 3, -2, -1, 3, -1, -2, 3, 0, -3, 3, 1, -4, 4, -2, -2, 4, -1, -3, 4, 0, -4], ctildes: [0.057627968274698634, -0.21492455291036522, -0.031564138167244495, 0.11771902579071958, -0.13391529684220621, 0.49943952846775125, 0.14115911718630864, -0.52645548782619978, -0.031564138167244522, 0.11771902579071969, -0.097967546066987654, 0.36537173994762079, 0.08983311208987084, -0.33503422089130008, 0.11204157580931885, -0.41786108913999492, -0.13391529684220629, 0.49943952846775153, 0.089833112089870826, -0.33503422089129997, 0.046102374619758871, -0.17193964232829204, -0.13204226338266001, 0.49245401620814466, -0.074694383872879258, 0.27857405942666336, 0.11204157580931885, -0.41786108913999492, -0.13204226338266001, 0.49245401620814477, 0.040339577792289028, -0.1504471870372556, 0.1397403965842435, -0.52116434360868891, -0.074694383872879244, 0.27857405942666336, 0.13974039658424345, -0.5211643436086888, -0.16135831116915611, 0.60178874814902239]} + - {mu0: 0, rank: 3, ndensity: 2, num_ms_combs: 14, mus: [0, 0, 0], ns: [2, 3, 3], ls: [4, 3, 3], ms_combs: [0, -3, 3, 0, -2, 2, 0, -1, 1, 0, 0, 0, 1, -3, 2, 1, -2, 1, 1, -1, 0, 2, -3, 1, 2, -2, 0, 2, -1, -1, 3, -3, 0, 3, -2, -1, 4, -3, -1, 4, -2, -2], ctildes: [-0.0065546718120995349, 0.025875350678399613, -0.015294234228232248, 0.060375818249599103, -0.0021848906040331808, 0.0086251168927998813, 0.0065546718120995349, -0.025875350678399613, 0.023934277390201188, -0.094483421666107126, 0.024719215396202115, -0.097582058294806928, -0.016924089845411126, 0.066809868169812325, -0.032111202742095682, 0.12676281215531524, -0.0075686830703306351, 0.029878281359099958, 0.013818461494091834, -0.054550028932883882, 0.034684063080920924, -0.13691948596326364, -0.016350224135747441, 0.064544464667466689, -0.028319418918253507, 0.11179429215138656, 0.018280106307444512, -0.072162905283894421]} + - {mu0: 0, rank: 3, ndensity: 2, num_ms_combs: 19, mus: [0, 0, 0], ns: [2, 3, 3], ls: [4, 4, 4], ms_combs: [0, -4, 4, 0, -3, 3, 0, -2, 2, 0, -1, 1, 0, 0, 0, 1, -4, 3, 1, -3, 2, 1, -2, 1, 1, -1, 0, 2, -4, 2, 2, -3, 1, 2, -2, 0, 2, -1, -1, 3, -4, 1, 3, -3, 0, 3, -2, -1, 4, -4, 0, 4, -3, -1, 4, -2, -2], ctildes: [-0.019196142849600958, 0.13274718035152952, -0.028794214274401431, 0.19912077052729424, 0.015082683667543601, -0.10430135599048743, 0.012340377546172043, -0.085337473083126114, -0.012340377546172048, 0.085337473083126142, 0.06070353369469407, -0.41978344287598457, 0.02294377912271291, -0.15866322776462055, -0.052031600309737781, 0.35981437960798679, 0.024680755092344093, -0.17067494616625226, -0.068831337368138734, 0.47598968329386165, 0.02294377912271291, -0.15866322776462055, 0.03016536733508722, -0.20860271198097496, -0.026015800154868891, 0.17990718980399339, 0.060703533694694063, -0.41978344287598451, -0.057588428548802868, 0.39824154105458853, 0.02294377912271291, -0.15866322776462055, -0.038392285699201917, 0.26549436070305904, 0.060703533694694049, -0.4197834428759844, -0.034415668684069367, 0.23799484164693077]} + - {mu0: 0, rank: 3, ndensity: 2, num_ms_combs: 1, mus: [0, 0, 0], ns: [3, 3, 3], ls: [0, 0, 0], ms_combs: [0, 0, 0], ctildes: [0.9207831136796869, 0.083332028842070041]} + - {mu0: 0, rank: 3, ndensity: 2, num_ms_combs: 2, mus: [0, 0, 0], ns: [3, 3, 3], ls: [0, 1, 1], ms_combs: [0, 0, 0, 0, 1, -1], ctildes: [0.58856082433908974, -0.6420449214763545, -1.1771216486781793, 1.2840898429527088]} + - {mu0: 0, rank: 3, ndensity: 2, num_ms_combs: 3, mus: [0, 0, 0], ns: [3, 3, 3], ls: [0, 2, 2], ms_combs: [0, 0, 0, 0, 1, -1, 0, 2, -2], ctildes: [1.1342554769206683, 0.33954252507530691, -2.2685109538413371, -0.67908505015061393, 2.2685109538413375, 0.67908505015061404]} + - {mu0: 0, rank: 3, ndensity: 2, num_ms_combs: 4, mus: [0, 0, 0], ns: [3, 3, 3], ls: [0, 3, 3], ms_combs: [0, 0, 0, 0, 1, -1, 0, 2, -2, 0, 3, -3], ctildes: [0.13480883100140309, -0.35467680407961466, -0.26961766200280618, 0.70935360815922932, 0.26961766200280612, -0.70935360815922921, -0.26961766200280612, 0.70935360815922921]} + - {mu0: 0, rank: 3, ndensity: 2, num_ms_combs: 5, mus: [0, 0, 0], ns: [3, 3, 3], ls: [0, 4, 4], ms_combs: [0, 0, 0, 0, 1, -1, 0, 2, -2, 0, 3, -3, 0, 4, -4], ctildes: [0.032735440283539828, -0.76521355654070466, -0.065470880567079615, 1.5304271130814087, 0.065470880567079615, -1.5304271130814087, -0.065470880567079615, 1.5304271130814087, 0.065470880567079628, -1.5304271130814089]} + - {mu0: 0, rank: 3, ndensity: 2, num_ms_combs: 4, mus: [0, 0, 0], ns: [3, 3, 3], ls: [1, 1, 2], ms_combs: [-1, 1, 0, 0, 0, 0, 0, 1, -1, 1, 1, -2], ctildes: [0.63676380198979987, 0.23645157991798876, 0.63676380198979976, 0.23645157991798874, -2.2058145149341235, -0.81909229989577881, 1.5597464015496338, 0.57918571967399057]} + - {mu0: 0, rank: 3, ndensity: 2, num_ms_combs: 8, mus: [0, 0, 0], ns: [3, 3, 3], ls: [1, 2, 3], ms_combs: [0, 0, 0, 0, 1, -1, 0, 2, -2, 1, -2, 1, 1, -1, 0, 1, 0, -1, 1, 1, -2, 1, 2, -3], ctildes: [-0.15413544068972362, -0.29295249647416843, 0.29064057422101458, 0.55239652485976698, -0.22977154874940597, -0.43670779752967187, 0.10275696045981574, 0.19530166431611223, -0.17798027634774702, -0.33827240539826797, 0.25170212064589526, 0.47838942349080044, -0.32494604048888065, -0.61759809006054556, 0.39797599656776028, 0.75640009338289393]} + - {mu0: 0, rank: 3, ndensity: 2, num_ms_combs: 11, mus: [0, 0, 0], ns: [3, 3, 3], ls: [1, 3, 4], ms_combs: [0, 0, 0, 0, 1, -1, 0, 2, -2, 0, 3, -3, 1, -3, 2, 1, -2, 1, 1, -1, 0, 1, 0, -1, 1, 1, -2, 1, 2, -3, 1, 3, -4], ctildes: [-0.15720239110867062, 0.96602404268343556, 0.30442112137393296, -1.8706975146744533, -0.27228252847153134, 1.6732027232607956, 0.20795921618912683, -1.2779296887248071, -0.078601195554335257, 0.48301202134171745, 0.13614126423576567, -0.83660136163039778, -0.19253282228083907, 1.1831329919175071, 0.24855880476400055, -1.5274181246816856, -0.30442112137393296, 1.8706975146744531, 0.36019592834176783, -2.213439149372046, -0.41591843237825371, 2.5558593774496146]} + - {mu0: 0, rank: 3, ndensity: 2, num_ms_combs: 5, mus: [0, 0, 0], ns: [3, 3, 3], ls: [2, 2, 2], ms_combs: [-2, 0, 2, -2, 1, 1, -1, -1, 2, -1, 0, 1, 0, 0, 0], ctildes: [-0.18427716240537592, 0.90116124914817597, 0.15046167304705271, -0.73579507879404427, 0.075230836523526357, -0.36789753939702213, -0.092138581202687933, 0.45058062457408782, 0.030712860400895974, -0.15019354152469594]} + - {mu0: 0, rank: 3, ndensity: 2, num_ms_combs: 10, mus: [0, 0, 0], ns: [3, 3, 3], ls: [2, 2, 4], ms_combs: [-2, 1, 1, -2, 2, 0, -1, 1, 0, -1, 2, -1, 0, 0, 0, 0, 1, -1, 0, 2, -2, 1, 1, -2, 1, 2, -3, 2, 2, -4], ctildes: [-0.047784374630166312, -0.065565993849824727, 0.021369821987073653, 0.029322003852108251, 0.085479287948294638, 0.11728801540843305, -0.047784374630166312, -0.065565993849824727, 0.064109465961220982, 0.087966011556324789, -0.23409467104380222, -0.32120645882106125, 0.16552992933470664, 0.22712726519328988, 0.13515462134299608, 0.1854486354657861, -0.25285114365232819, -0.34694262837885331, 0.17879275830733513, 0.24532548520937153]} + - {mu0: 0, rank: 3, ndensity: 2, num_ms_combs: 9, mus: [0, 0, 0], ns: [3, 3, 3], ls: [2, 3, 3], ms_combs: [0, -3, 3, 0, -1, 1, 0, 0, 0, 1, -3, 2, 1, -2, 1, 1, -1, 0, 2, -3, 1, 2, -2, 0, 2, -1, -1], ctildes: [0.076051243748383554, 0.32953183911409495, -0.045630746249030123, -0.19771910346845692, 0.030420497499353424, 0.13181273564563797, -0.15210248749676711, -0.6590636782281899, 0.1178180801983402, 0.51050852997359675, -0.043021080137722434, -0.18641135844356072, 0.096198059653413354, 0.41682846925788108, -0.13604460063583326, -0.58948447440771234, 0.074514696595027533, 0.32287394393218083]} + - {mu0: 0, rank: 3, ndensity: 2, num_ms_combs: 13, mus: [0, 0, 0], ns: [3, 3, 3], ls: [2, 4, 4], ms_combs: [0, -4, 4, 0, -3, 3, 0, -2, 2, 0, -1, 1, 0, 0, 0, 1, -4, 3, 1, -3, 2, 1, -2, 1, 1, -1, 0, 2, -4, 2, 2, -3, 1, 2, -2, 0, 2, -1, -1], ctildes: [0.043770795483647824, -0.24438801331345975, -0.010942698870911958, 0.061097003328364943, -0.012505941566756516, 0.069825146660988477, 0.026575125829357612, -0.1483784366546006, -0.015632426958445661, 0.087281433326235669, -0.075813241665384362, 0.42329245581973141, 0.071636779832945474, -0.39997377498171593, -0.048737083927747109, 0.272116578741256, 0.017124465747385121, -0.095612019768324868, 0.040523882241794769, -0.22625933486908292, -0.06078582336269215, 0.33938900230362434, 0.072652955125036653, -0.40564744524674834, -0.038291469489519857, 0.21379497566802799]} + - {mu0: 0, rank: 3, ndensity: 2, num_ms_combs: 17, mus: [0, 0, 0], ns: [3, 3, 3], ls: [3, 3, 4], ms_combs: [-3, 1, 2, -3, 2, 1, -3, 3, 0, -2, 1, 1, -2, 2, 0, -2, 3, -1, -1, 1, 0, -1, 2, -1, -1, 3, -2, 0, 0, 0, 0, 1, -1, 0, 2, -2, 0, 3, -3, 1, 1, -2, 1, 2, -3, 1, 3, -4, 2, 2, -4], ctildes: [0.028776354171500276, -0.17600360819444069, -0.021448628024028086, 0.13118534406933613, 0.011747897396297649, -0.071853172160852019, -0.022152047769615588, 0.13548764075920766, 0.027411760591361176, -0.16765740170865467, -0.021448628024028086, 0.13118534406933613, 0.0039159657987658828, -0.023951057386950674, -0.022152047769615588, 0.13548764075920766, 0.028776354171500276, -0.17600360819444069, -0.011747897396297649, 0.071853172160852019, 0.030332940645876164, -0.18552409276743609, 0.013565303448329102, -0.082968896578392876, -0.062164029877013548, 0.38021124889744767, -0.024766742326841541, 0.15147978742432983, 0.029304404714612948, -0.17923330158586098, 0.050756717851270595, -0.3104411847550268, -0.032763320491019952, 0.20038892308885309]} + - {mu0: 0, rank: 3, ndensity: 2, num_ms_combs: 13, mus: [0, 0, 0], ns: [3, 3, 3], ls: [4, 4, 4], ms_combs: [-4, 0, 4, -4, 1, 3, -4, 2, 2, -3, -1, 4, -3, 0, 3, -3, 1, 2, -2, -2, 4, -2, -1, 3, -2, 0, 2, -2, 1, 1, -1, -1, 2, -1, 0, 1, 0, 0, 0], ctildes: [-0.021997123907167796, 0.16762460232255286, 0.046374009013061657, -0.35338369014614496, -0.026291591816920503, 0.20034972032421558, 0.023187004506530821, -0.17669184507307242, -0.032995685860751689, 0.2514369034838293, 0.017527727877947003, -0.13356648021614373, -0.01314579590846025, 0.10017486016210778, 0.0087638639389735016, -0.066783240108071865, 0.017283454498488979, -0.13170504468200578, -0.01987457529131214, 0.15145015291977643, -0.0099372876456560698, 0.075725076459888213, 0.01414100822603644, -0.10775867292164112, -0.0047136694086788149, 0.035919557640547051]} + - {mu0: 0, rank: 4, ndensity: 2, num_ms_combs: 1, mus: [0, 0, 0, 0], ns: [1, 1, 1, 1], ls: [0, 0, 0, 0], ms_combs: [0, 0, 0, 0], ctildes: [-0.0029681466865226556, 0.061458617765261486]} + - {mu0: 0, rank: 4, ndensity: 2, num_ms_combs: 2, mus: [0, 0, 0, 0], ns: [1, 1, 1, 1], ls: [0, 0, 1, 1], ms_combs: [0, 0, 0, 0, 0, 0, 1, -1], ctildes: [-0.01014853990859162, 0.084623312241505225, 0.020297079817183237, -0.16924662448301042]} + - {mu0: 0, rank: 4, ndensity: 2, num_ms_combs: 3, mus: [0, 0, 0, 0], ns: [1, 1, 1, 1], ls: [0, 0, 2, 2], ms_combs: [0, 0, 0, 0, 0, 0, 1, -1, 0, 0, 2, -2], ctildes: [-0.0017975004487086016, 0.05239955202099781, 0.0035950008974172036, -0.10479910404199563, -0.0035950008974172045, 0.10479910404199566]} + - {mu0: 0, rank: 4, ndensity: 2, num_ms_combs: 4, mus: [0, 0, 0, 0], ns: [1, 1, 1, 1], ls: [0, 1, 1, 2], ms_combs: [0, -1, 1, 0, 0, 0, 0, 0, 0, 0, 1, -1, 0, 1, 1, -2], ctildes: [-0.030245324457733386, 0.31433661422151576, -0.030245324457733393, 0.31433661422151582, 0.10477287730439969, -1.0888939730216862, -0.074085612026367126, 0.76996431231679574]} + - {mu0: 0, rank: 4, ndensity: 2, num_ms_combs: 5, mus: [0, 0, 0, 0], ns: [1, 1, 1, 1], ls: [0, 2, 2, 2], ms_combs: [0, -2, 0, 2, 0, -2, 1, 1, 0, -1, -1, 2, 0, -1, 0, 1, 0, 0, 0, 0], ctildes: [0.0095778478596494042, -0.12416397388594128, -0.0078202800300496769, 0.10137946015227051, -0.0039101400150248385, 0.050689730076135253, 0.0047889239298247004, -0.062081986942970617, -0.0015963079766082335, 0.020693995647656872]} + - {mu0: 0, rank: 4, ndensity: 2, num_ms_combs: 3, mus: [0, 0, 0, 0], ns: [1, 1, 1, 1], ls: [1, 1, 1, 1], ms_combs: [-1, -1, 1, 1, -1, 0, 0, 1, 0, 0, 0, 0], ctildes: [-0.091702785110780505, 0.6603607567812898, 0.091702785110780519, -0.66036075678128991, -0.02292569627769514, 0.16509018919532253]} + - {mu0: 0, rank: 4, ndensity: 2, num_ms_combs: 10, mus: [0, 0, 0, 0], ns: [1, 1, 1, 1], ls: [1, 1, 2, 2], ms_combs: [-1, 1, -2, 2, -1, 1, -1, 1, -1, 1, 0, 0, 0, 0, -2, 2, 0, 0, -1, 1, 0, 0, 0, 0, 0, 1, -2, 1, 0, 1, -1, 0, 1, 1, -2, 0, 1, 1, -1, -1], ctildes: [-0.0091117029730398008, -0.020997020512106869, 0.0074395952016821382, -0.0045022625699270733, -0.0034411129722814577, 0.0065010117986358607, 0.0028837437151622396, -0.015000772825980504, -0.0045558514865199004, -0.010498510256053433, 0.0025566103718195603, 0.0094991356416990382, 0.0047294349760069118, 0.072122863930806427, -0.0019307837438148226, -0.029444035903109532, -0.002730540556512423, -0.041640154905177862, 0.0016721077713576621, 0.025499283082033943]} + - {mu0: 0, rank: 4, ndensity: 2, num_ms_combs: 6, mus: [0, 0, 0, 0], ns: [1, 1, 1, 1], ls: [2, 2, 2, 2], ms_combs: [-2, -2, 2, 2, -2, -1, 1, 2, -2, 0, 0, 2, -1, -1, 1, 1, -1, 0, 0, 1, 0, 0, 0, 0], ctildes: [-0.010407200271111193, 0.2691020744256698, 0.020814400542222378, -0.53820414885133938, -0.010407200271111187, 0.26910207442566969, -0.010407200271111187, 0.26910207442566964, 0.010407200271111186, -0.26910207442566958, -0.002601800067777796, 0.067275518606417395]} + - {mu0: 0, rank: 4, ndensity: 2, num_ms_combs: 1, mus: [0, 0, 0, 0], ns: [1, 1, 1, 2], ls: [0, 0, 0, 0], ms_combs: [0, 0, 0, 0], ctildes: [0.079514437856555112, -0.43875614175322825]} + - {mu0: 0, rank: 4, ndensity: 2, num_ms_combs: 2, mus: [0, 0, 0, 0], ns: [1, 1, 1, 2], ls: [0, 0, 1, 1], ms_combs: [0, 0, 0, 0, 0, 0, 1, -1], ctildes: [0.055462229710258243, 0.032906444987456526, -0.11092445942051646, -0.065812889974913039]} + - {mu0: 0, rank: 4, ndensity: 2, num_ms_combs: 3, mus: [0, 0, 0, 0], ns: [1, 1, 1, 2], ls: [0, 0, 2, 2], ms_combs: [0, 0, 0, 0, 0, 0, 1, -1, 0, 0, 2, -2], ctildes: [0.0046053516906974807, -0.03641439409823255, -0.0092107033813949631, 0.0728287881964651, 0.0092107033813949649, -0.072828788196465127]} + - {mu0: 0, rank: 4, ndensity: 2, num_ms_combs: 2, mus: [0, 0, 0, 0], ns: [1, 1, 1, 2], ls: [0, 1, 1, 0], ms_combs: [0, 0, 0, 0, 0, 1, -1, 0], ctildes: [0.1433466937802797, -0.27889390048231127, -0.28669338756055934, 0.55778780096462244]} + - {mu0: 0, rank: 4, ndensity: 2, num_ms_combs: 4, mus: [0, 0, 0, 0], ns: [1, 1, 1, 2], ls: [0, 1, 1, 2], ms_combs: [0, -1, 1, 0, 0, 0, 0, 0, 0, 0, 1, -1, 0, 1, 1, -2], ctildes: [0.014845052558862931, 0.018760474314254795, 0.014845052558862933, 0.018760474314254798, -0.051424770545961944, -0.064988189372760408, 0.036362803974011926, 0.0459535894025144]} + - {mu0: 0, rank: 4, ndensity: 2, num_ms_combs: 5, mus: [0, 0, 0, 0], ns: [1, 1, 1, 2], ls: [0, 1, 2, 1], ms_combs: [0, 0, 0, 0, 0, 0, 1, -1, 0, 1, -2, 1, 0, 1, -1, 0, 0, 1, 0, -1], ctildes: [0.025757358871255666, -0.018692409269171015, -0.04461305423379977, 0.032376202570075625, 0.063092386356326052, -0.045786864772739605, -0.04461305423379977, 0.032376202570075625, 0.025757358871255662, -0.018692409269171012]} + - {mu0: 0, rank: 4, ndensity: 2, num_ms_combs: 3, mus: [0, 0, 0, 0], ns: [1, 1, 1, 2], ls: [0, 2, 2, 0], ms_combs: [0, 0, 0, 0, 0, 1, -1, 0, 0, 2, -2, 0], ctildes: [0.0093119247062790449, 0.099483353692720849, -0.018623849412558093, -0.19896670738544173, 0.018623849412558097, 0.19896670738544178]} + - {mu0: 0, rank: 4, ndensity: 2, num_ms_combs: 8, mus: [0, 0, 0, 0], ns: [1, 1, 1, 2], ls: [0, 2, 2, 2], ms_combs: [0, -2, 1, 1, 0, -2, 2, 0, 0, -1, 1, 0, 0, -1, 2, -1, 0, 0, 0, 0, 0, 0, 1, -1, 0, 0, 2, -2, 0, 1, 1, -2], ctildes: [-0.011408760300648047, 0.065796577006364321, 0.009315213778103109, -0.053722680162444317, 0.0046576068890515519, -0.026861340081222145, -0.011408760300648047, 0.065796577006364321, -0.0046576068890515519, 0.026861340081222145, 0.0093152137781031038, -0.053722680162444289, 0.018630427556206211, -0.10744536032488859, -0.011408760300648047, 0.065796577006364321]} + - {mu0: 0, rank: 4, ndensity: 2, num_ms_combs: 5, mus: [0, 0, 0, 0], ns: [1, 1, 1, 2], ls: [1, 1, 1, 1], ms_combs: [0, 0, 0, 0, 0, 0, 1, -1, 1, -1, -1, 1, 1, -1, 0, 0, 1, -1, 1, -1], ctildes: [0.097985268317358126, -0.41152745101933563, -0.1959705366347162, 0.82305490203867093, 0.19597053663471617, -0.82305490203867082, -0.1959705366347162, 0.82305490203867093, 0.19597053663471617, -0.82305490203867082]} + - {mu0: 0, rank: 4, ndensity: 2, num_ms_combs: 4, mus: [0, 0, 0, 0], ns: [1, 1, 1, 2], ls: [1, 1, 2, 0], ms_combs: [-1, 1, 0, 0, 0, 0, 0, 0, 0, 1, -1, 0, 1, 1, -2, 0], ctildes: [0.20901530813666733, -0.68966352673021092, 0.20901530813666727, -0.68966352673021081, -0.7240502665047448, 2.3890645368477239, 0.51198085336543209, -1.6893237346973242]} + - {mu0: 0, rank: 4, ndensity: 2, num_ms_combs: 20, mus: [0, 0, 0, 0], ns: [1, 1, 1, 2], ls: [1, 1, 2, 2], ms_combs: [-1, 1, -2, 2, -1, 1, -1, 1, -1, 1, 0, 0, -1, 1, 1, -1, -1, 1, 2, -2, 0, 0, 0, 0, 0, 0, 1, -1, 0, 0, 2, -2, 0, 1, -2, 1, 0, 1, -1, 0, 0, 1, 0, -1, 0, 1, 1, -2, 1, -1, -2, 2, 1, -1, -1, 1, 1, -1, 0, 0, 1, -1, 1, -1, 1, -1, 2, -2, 1, 1, -2, 0, 1, 1, -1, -1, 1, 1, 0, -2], ctildes: [0.033302308937348479, -0.34988086048804085, 0.01665115446867424, -0.17494043024402042, -0.033302308937348479, 0.34988086048804085, 0.01665115446867424, -0.17494043024402042, 0.033302308937348479, -0.34988086048804085, -0.074697514597487635, 0.60085794657605651, 0.11609272025762682, -0.85183503266407234, -0.016185793445581395, -0.19780754880004991, -0.14128973087261087, 1.4844187743508683, 0.057681291088842679, -0.6060114269612048, 0.057681291088842679, -0.6060114269612048, -0.14128973087261087, 1.4844187743508683, 0.08279041132027834, -0.50195417217603155, -0.082790411320278312, 0.50195417217603144, 0.082790411320278298, -0.50195417217603133, -0.082790411320278312, 0.50195417217603144, 0.08279041132027834, -0.50195417217603155, 0.081573664153031702, -0.85702957896160847, -0.099906926812045452, 1.0496425814641226, 0.081573664153031702, -0.85702957896160847]} + - {mu0: 0, rank: 4, ndensity: 2, num_ms_combs: 13, mus: [0, 0, 0, 0], ns: [1, 1, 1, 2], ls: [1, 2, 2, 1], ms_combs: [0, -2, 1, 1, 0, -2, 2, 0, 0, -1, 1, 0, 0, -1, 2, -1, 0, 0, 0, 0, 0, 0, 1, -1, 1, -2, 0, 1, 1, -2, 1, 0, 1, -2, 2, -1, 1, -1, -1, 1, 1, -1, 0, 0, 1, -1, 1, -1, 1, 0, 0, -1], ctildes: [0.0084202744020071482, 0.55499893214303686, 0.16056643444038468, -0.63136275799653774, -0.086237250349303102, -0.076762129471365181, -0.11353761462374126, 0.44644088756799305, 0.030730427826137947, 0.15640187931366631, 0.042913974443631209, -0.40883609439946711, 0.12137904934703134, -1.1563630989747469, -0.10511734022173409, 1.0014398197110301, -0.1724745006986062, -0.15352425894273014, -0.074329184091081579, 0.70812488746790281, 0.042913974443631195, -0.40883609439946711, 0.24680368478968784, -0.55460062852517278, -0.13579003974335749, 0.39532112884057025]} + - {mu0: 0, rank: 4, ndensity: 2, num_ms_combs: 5, mus: [0, 0, 0, 0], ns: [1, 1, 1, 2], ls: [2, 2, 2, 0], ms_combs: [-2, 0, 2, 0, -2, 1, 1, 0, -1, -1, 2, 0, -1, 0, 1, 0, 0, 0, 0, 0], ctildes: [-0.11110019956572194, 0.42404246670788792, 0.090712933085799988, -0.34622922423514957, 0.045356466542899994, -0.17311461211757478, -0.055550099782860955, 0.21202123335394391, 0.018516699927620315, -0.070673744451314621]} + - {mu0: 0, rank: 4, ndensity: 2, num_ms_combs: 13, mus: [0, 0, 0, 0], ns: [1, 1, 1, 2], ls: [2, 2, 2, 2], ms_combs: [0, 0, 0, 0, 0, 0, 1, -1, 0, 0, 2, -2, 1, -1, -2, 2, 1, -1, -1, 1, 1, -1, 0, 0, 1, -1, 1, -1, 1, -1, 2, -2, 2, -2, -2, 2, 2, -2, -1, 1, 2, -2, 0, 0, 2, -2, 1, -1, 2, -2, 2, -2], ctildes: [0.013088735181951904, -0.039205266530467096, -0.026177470363903811, 0.078410533060934207, 0.026177470363903818, -0.07841053306093422, -0.026177470363903822, 0.078410533060934234, 0.026177470363903815, -0.078410533060934207, -0.026177470363903811, 0.078410533060934207, 0.026177470363903815, -0.078410533060934207, -0.026177470363903822, 0.078410533060934234, 0.026177470363903829, -0.078410533060934248, -0.026177470363903822, 0.078410533060934234, 0.026177470363903818, -0.07841053306093422, -0.026177470363903822, 0.078410533060934234, 0.026177470363903829, -0.078410533060934248]} + - {mu0: 0, rank: 4, ndensity: 2, num_ms_combs: 1, mus: [0, 0, 0, 0], ns: [1, 1, 2, 2], ls: [0, 0, 0, 0], ms_combs: [0, 0, 0, 0], ctildes: [-0.39847483488446617, -1.5625565627251885]} + - {mu0: 0, rank: 4, ndensity: 2, num_ms_combs: 2, mus: [0, 0, 0, 0], ns: [1, 1, 2, 2], ls: [0, 0, 1, 1], ms_combs: [0, 0, 0, 0, 0, 0, 1, -1], ctildes: [0.01131756200690677, -0.15280433648699682, -0.022635124013813537, 0.30560867297399358]} + - {mu0: 0, rank: 4, ndensity: 2, num_ms_combs: 3, mus: [0, 0, 0, 0], ns: [1, 1, 2, 2], ls: [0, 0, 2, 2], ms_combs: [0, 0, 0, 0, 0, 0, 1, -1, 0, 0, 2, -2], ctildes: [0.0069857986157835644, 0.039594844753127856, -0.013971597231567131, -0.079189689506255712, 0.013971597231567134, 0.079189689506255739]} + - {mu0: 0, rank: 4, ndensity: 2, num_ms_combs: 2, mus: [0, 0, 0, 0], ns: [1, 1, 2, 2], ls: [0, 1, 0, 1], ms_combs: [0, 0, 0, 0, 0, 1, 0, -1], ctildes: [0.046355118583235641, -0.94884657744452416, -0.092710237166471268, 1.8976931548890479]} + - {mu0: 0, rank: 4, ndensity: 2, num_ms_combs: 5, mus: [0, 0, 0, 0], ns: [1, 1, 2, 2], ls: [0, 1, 1, 2], ms_combs: [0, 0, 0, 0, 0, 0, 1, -1, 0, 1, -1, 0, 0, 1, 0, -1, 0, 1, 1, -2], ctildes: [0.057656722648539299, -0.32334489467464683, -0.099864373025177283, 0.56004978594449562, 0.057656722648539292, -0.32334489467464678, -0.099864373025177283, 0.56004978594449562, 0.14122955073009155, -0.79203000288685443]} + - {mu0: 0, rank: 4, ndensity: 2, num_ms_combs: 3, mus: [0, 0, 0, 0], ns: [1, 1, 2, 2], ls: [0, 2, 0, 2], ms_combs: [0, 0, 0, 0, 0, 1, 0, -1, 0, 2, 0, -2], ctildes: [0.023444889410087221, 0.67852143261626074, -0.046889778820174442, -1.3570428652325217, 0.046889778820174456, 1.3570428652325219]} + - {mu0: 0, rank: 4, ndensity: 2, num_ms_combs: 4, mus: [0, 0, 0, 0], ns: [1, 1, 2, 2], ls: [0, 2, 1, 1], ms_combs: [0, 0, -1, 1, 0, 0, 0, 0, 0, 1, -1, 0, 0, 2, -1, -1], ctildes: [0.028579726677287573, -0.2849936299462229, 0.028579726677287566, -0.28499362994622285, -0.099003077342987467, 0.98724689380068242, 0.07000574734756268, -0.69808897331181796]} + - {mu0: 0, rank: 4, ndensity: 2, num_ms_combs: 7, mus: [0, 0, 0, 0], ns: [1, 1, 2, 2], ls: [0, 2, 2, 2], ms_combs: [0, 0, -2, 2, 0, 0, -1, 1, 0, 0, 0, 0, 0, 1, -2, 1, 0, 1, -1, 0, 0, 2, -2, 0, 0, 2, -1, -1], ctildes: [-0.084901340502308242, 0.75222413169680891, -0.042450670251154121, 0.37611206584840445, 0.042450670251154121, -0.37611206584840445, 0.20796496270894607, -1.8425652948653164, -0.084901340502308242, 0.75222413169680891, -0.16980268100461657, 1.5044482633936185, 0.10398248135447304, -0.9212826474326582]} + - {mu0: 0, rank: 4, ndensity: 2, num_ms_combs: 2, mus: [0, 0, 0, 0], ns: [1, 1, 2, 2], ls: [1, 1, 0, 0], ms_combs: [0, 0, 0, 0, 1, -1, 0, 0], ctildes: [-0.53634534266878331, -1.3976329303936745, 1.0726906853375664, 2.7952658607873486]} + - {mu0: 0, rank: 4, ndensity: 2, num_ms_combs: 4, mus: [0, 0, 0, 0], ns: [1, 1, 2, 2], ls: [1, 1, 0, 2], ms_combs: [-1, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, -1, 1, 1, 0, -2], ctildes: [-0.022489093854819439, 0.93627759477312422, -0.022489093854819436, 0.93627759477312411, 0.077904506345464586, -3.2433607282708721, -0.055086804721868436, 2.2934023647944732]} + - {mu0: 0, rank: 4, ndensity: 2, num_ms_combs: 6, mus: [0, 0, 0, 0], ns: [1, 1, 2, 2], ls: [1, 1, 1, 1], ms_combs: [-1, 1, -1, 1, -1, 1, 0, 0, 0, 0, -1, 1, 0, 0, 0, 0, 0, 1, -1, 0, 1, 1, -1, -1], ctildes: [0.67656643730074939, -5.036518134239933, -0.32525378576472153, 3.6551885463482297, -0.32525378576472153, 3.6551885463482297, 0.17565632576801404, -0.69066479394585278, -0.052117731542612906, -4.5477179169130526, 0.026058865771306453, 2.2738589584565263]} + - {mu0: 0, rank: 4, ndensity: 2, num_ms_combs: 10, mus: [0, 0, 0, 0], ns: [1, 1, 2, 2], ls: [1, 1, 2, 2], ms_combs: [-1, 1, -2, 2, -1, 1, -1, 1, -1, 1, 0, 0, 0, 0, -2, 2, 0, 0, -1, 1, 0, 0, 0, 0, 0, 1, -2, 1, 0, 1, -1, 0, 1, 1, -2, 0, 1, 1, -1, -1], ctildes: [-0.15498299787343672, 0.84010665695807352, 0.1030008552943899, -0.30186096348385338, -0.042836737217353801, 0.061222866162889938, 0.025509356357671577, 0.11819236499518321, -0.077491498936718362, 0.42005332847903676, 0.047409439898200303, -0.29973427981855505, 0.14702770207299987, -1.5223887192003076, -0.060023808022132356, 0.62151259203499554, -0.084886483370178595, 0.87895153684154748, 0.051982142579046813, -0.53824569347422013]} + - {mu0: 0, rank: 4, ndensity: 2, num_ms_combs: 5, mus: [0, 0, 0, 0], ns: [1, 1, 2, 2], ls: [1, 2, 0, 1], ms_combs: [0, 0, 0, 0, 0, 1, 0, -1, 1, -2, 0, 1, 1, -1, 0, 0, 1, 0, 0, -1], ctildes: [-0.1573443324174654, -0.16078685264750545, 0.27252837803005681, 0.27849099797456994, -0.38541332834164826, -0.39384574633445496, 0.27252837803005681, 0.27849099797456994, -0.15734433241746537, -0.16078685264750545]} + - {mu0: 0, rank: 4, ndensity: 2, num_ms_combs: 19, mus: [0, 0, 0, 0], ns: [1, 1, 2, 2], ls: [1, 2, 1, 2], ms_combs: [0, 0, 0, 0, 0, 0, 1, -1, 0, 1, -1, 0, 0, 1, 0, -1, 0, 1, 1, -2, 0, 2, -1, -1, 0, 2, 0, -2, 1, -2, -1, 2, 1, -2, 0, 1, 1, -2, 1, 0, 1, -1, -1, 1, 1, -1, 0, 0, 1, -1, 1, -1, 1, 0, -1, 0, 1, 0, 0, -1, 1, 0, 1, -2, 1, 1, -1, -1, 1, 1, 0, -2, 1, 2, -1, -2], ctildes: [0.16800263939774412, -0.24328680243412754, 0.10176114605776652, -1.3453894433973559, -0.17825306947967365, 2.4269791052895129, -0.29184264621701789, -0.13788247757789818, -0.02063223388488071, 0.21539161964847636, 0.20799841571259844, -2.8647344023536401, 0.15935474848160691, 2.0112507249163576, -0.13017636349955494, -2.3158604746447402, -0.02063223388488071, 0.21539161964847636, -0.10817591551526523, 1.5295987687704184, 0.14476555599058097, 2.1635555997805489, 0.10176114605776652, -1.3453894433973559, 0.13248789773541095, -1.873368247338459, -0.2035173810600773, -1.3867946424702042, -0.17825306947967365, 2.4269791052895124, -0.10817591551526526, 1.5295987687704184, 0.30643183870804386, -0.014422397286292465, 0.20799841571259844, -2.8647344023536401, -0.45350892893448075, 2.0400955194889425]} + - {mu0: 0, rank: 4, ndensity: 2, num_ms_combs: 3, mus: [0, 0, 0, 0], ns: [1, 1, 2, 2], ls: [2, 2, 0, 0], ms_combs: [0, 0, 0, 0, 1, -1, 0, 0, 2, -2, 0, 0], ctildes: [0.16458404977480029, 0.33833632623155702, -0.32916809954960063, -0.67667265246311414, 0.32916809954960075, 0.67667265246311425]} + - {mu0: 0, rank: 4, ndensity: 2, num_ms_combs: 8, mus: [0, 0, 0, 0], ns: [1, 1, 2, 2], ls: [2, 2, 0, 2], ms_combs: [-2, 1, 0, 1, -2, 2, 0, 0, -1, 1, 0, 0, -1, 2, 0, -1, 0, 0, 0, 0, 0, 1, 0, -1, 0, 2, 0, -2, 1, 1, 0, -2], ctildes: [-0.11087919693118821, 1.8139880783181468, 0.090532485190327167, -1.4811150637904225, 0.045266242595163583, -0.74055753189521123, -0.11087919693118821, 1.8139880783181468, -0.045266242595163583, 0.74055753189521123, 0.090532485190327167, -1.4811150637904225, 0.18106497038065442, -2.9622301275808467, -0.11087919693118821, 1.8139880783181468]} + - {mu0: 0, rank: 4, ndensity: 2, num_ms_combs: 11, mus: [0, 0, 0, 0], ns: [1, 1, 2, 2], ls: [2, 2, 1, 1], ms_combs: [-2, 1, 0, 1, -2, 2, -1, 1, -2, 2, 0, 0, -1, 1, -1, 1, -1, 1, 0, 0, -1, 2, -1, 0, 0, 0, -1, 1, 0, 0, 0, 0, 0, 1, -1, 0, 0, 2, -1, -1, 1, 1, -1, -1], ctildes: [0.44329324770126305, -1.5416888130033077, -0.54444073920786495, 3.3828457205451259, -0.041235291899838489, -0.60128424611848541, 0.23098507770409385, -2.2927071063910471, -0.27222036960393242, 1.6914228602725629, 0.44329324770126305, -1.5416888130033077, -0.063249928601418398, 0.96466378416984377, 0.1883527950525947, -1.0274011991619609, -0.3619474210964288, 1.2587836446717251, -0.51187095178053554, 1.7801889023881894, 0.31345566150377102, -1.0901386141540783]} + - {mu0: 0, rank: 4, ndensity: 2, num_ms_combs: 21, mus: [0, 0, 0, 0], ns: [1, 1, 2, 2], ls: [2, 2, 2, 2], ms_combs: [-2, 1, -1, 2, -2, 1, 0, 1, -2, 2, -2, 2, -2, 2, -1, 1, -2, 2, 0, 0, -1, 1, -2, 2, -1, 1, -1, 1, -1, 1, 0, 0, -1, 2, -2, 1, -1, 2, -1, 0, 0, 0, -2, 2, 0, 0, -1, 1, 0, 0, 0, 0, 0, 1, -2, 1, 0, 1, -1, 0, 0, 2, -2, 0, 0, 2, -1, -1, 1, 1, -2, 0, 1, 1, -1, -1, 1, 2, -2, -1, 2, 2, -2, -2], ctildes: [0.037181031944081518, -0.568714578372298, 0.011637740723169761, -0.057705751045038528, -0.013869613322542404, 0.027400472897005101, -0.023311418621539105, 0.54131410547529279, 0.0069046215556738148, -0.24709877852357606, -0.023311418621539105, 0.54131410547529279, -0.028122876587829623, 0.098075295539216206, -0.021157884820961037, 0.31777360116578701, 0.037181031944081518, -0.568714578372298, 0.011637740723169761, -0.057705751045038528, 0.0069046215556738148, -0.24709877852357603, -0.021157884820961037, 0.31777360116578701, -0.019889292949299593, 0.18436617413843129, 0.023275481446339525, -0.11541150209007717, 0.12187294143912041, -1.3730118988852986, -0.093366414908545997, 1.2316622536008774, -0.023275481446339501, 0.11541150209007683, -0.023275481446339518, 0.115411502090077, -0.051434295209368708, 0.6393894010145087, 0.13137511694931187, -1.420128447313439, -0.065687558474655933, 0.71006422365671962]} + - {mu0: 0, rank: 4, ndensity: 2, num_ms_combs: 1, mus: [0, 0, 0, 0], ns: [1, 2, 2, 2], ls: [0, 0, 0, 0], ms_combs: [0, 0, 0, 0], ctildes: [-0.76134335381004425, -0.70068375725060694]} + - {mu0: 0, rank: 4, ndensity: 2, num_ms_combs: 2, mus: [0, 0, 0, 0], ns: [1, 2, 2, 2], ls: [0, 0, 1, 1], ms_combs: [0, 0, 0, 0, 0, 0, 1, -1], ctildes: [-0.047165685922840969, 0.35333298124279849, 0.094331371845681924, -0.70666596248559688]} + - {mu0: 0, rank: 4, ndensity: 2, num_ms_combs: 3, mus: [0, 0, 0, 0], ns: [1, 2, 2, 2], ls: [0, 0, 2, 2], ms_combs: [0, 0, 0, 0, 0, 0, 1, -1, 0, 0, 2, -2], ctildes: [-0.069852145823792594, 1.0114207516701346, 0.13970429164758519, -2.0228415033402696, -0.13970429164758524, 2.02284150334027]} + - {mu0: 0, rank: 4, ndensity: 2, num_ms_combs: 4, mus: [0, 0, 0, 0], ns: [1, 2, 2, 2], ls: [0, 1, 1, 2], ms_combs: [0, -1, 1, 0, 0, 0, 0, 0, 0, 0, 1, -1, 0, 1, 1, -2], ctildes: [-0.064279477640850319, -0.22484132791096745, -0.064279477640850333, -0.22484132791096748, 0.22267064231588082, 0.77887320716609998, -0.15745192115272352, -0.55074652647166389]} + - {mu0: 0, rank: 4, ndensity: 2, num_ms_combs: 5, mus: [0, 0, 0, 0], ns: [1, 2, 2, 2], ls: [0, 2, 2, 2], ms_combs: [0, -2, 0, 2, 0, -2, 1, 1, 0, -1, -1, 2, 0, -1, 0, 1, 0, 0, 0, 0], ctildes: [0.1115702623880457, -1.8538225523959755, -0.091096737773048567, 1.513639775678024, -0.045548368886524283, 0.756819887839012, 0.055785131194022827, -0.92691127619798741, -0.018595043731340941, 0.30897042539932912]} + - {mu0: 0, rank: 4, ndensity: 2, num_ms_combs: 2, mus: [0, 0, 0, 0], ns: [1, 2, 2, 2], ls: [1, 0, 0, 1], ms_combs: [0, 0, 0, 0, 1, 0, 0, -1], ctildes: [-0.096084930556729453, -0.65258154020822678, 0.19216986111345888, 1.3051630804164533]} + - {mu0: 0, rank: 4, ndensity: 2, num_ms_combs: 5, mus: [0, 0, 0, 0], ns: [1, 2, 2, 2], ls: [1, 0, 1, 2], ms_combs: [0, 0, 0, 0, 0, 0, 1, -1, 1, 0, -1, 0, 1, 0, 0, -1, 1, 0, 1, -2], ctildes: [-0.64687823707105574, -0.22310690192369287, 1.120425972917654, 0.38643248965112253, -0.64687823707105563, -0.22310690192369281, 1.120425972917654, 0.38643248965112253, -1.5845216065352161, -0.54649806780621812]} + - {mu0: 0, rank: 4, ndensity: 2, num_ms_combs: 4, mus: [0, 0, 0, 0], ns: [1, 2, 2, 2], ls: [1, 1, 1, 1], ms_combs: [0, -1, 0, 1, 0, 0, 0, 0, 1, -1, -1, 1, 1, -1, 0, 0], ctildes: [0.80236173137046052, -1.1214378260675379, -0.40118086568523037, 0.56071891303376908, -1.6047234627409208, 2.2428756521350754, 0.80236173137046052, -1.1214378260675379]} + - {mu0: 0, rank: 4, ndensity: 2, num_ms_combs: 12, mus: [0, 0, 0, 0], ns: [1, 2, 2, 2], ls: [1, 1, 2, 2], ms_combs: [0, 0, -2, 2, 0, 0, -1, 1, 0, 0, 0, 0, 0, 1, -2, 1, 0, 1, -1, 0, 1, -1, -2, 2, 1, -1, -1, 1, 1, -1, 0, 0, 1, 0, -2, 1, 1, 0, -1, 0, 1, 1, -2, 0, 1, 1, -1, -1], ctildes: [-0.19426100918108014, -0.70312939444746836, 0.018061415887690606, 0.92835489128019744, 0.020335890938386288, -0.50171502844555349, 0.24918385452013495, -0.31851695221306431, -0.10172888261904102, 0.13003400122474343, 0.036122831775381198, 1.8567097825603949, -0.21232242506877069, -1.6314842857276655, 0.13552781141661691, 0.77820456005837757, 0.24918385452013495, -0.31851695221306431, -0.10172888261904102, 0.13003400122474343, -0.287732730969817, 0.36779169620334379, 0.17619959329338955, -0.22522549683272927]} + - {mu0: 0, rank: 4, ndensity: 2, num_ms_combs: 3, mus: [0, 0, 0, 0], ns: [1, 2, 2, 2], ls: [2, 0, 0, 2], ms_combs: [0, 0, 0, 0, 1, 0, 0, -1, 2, 0, 0, -2], ctildes: [0.27063939036564638, 0.69763091165329749, -0.54127878073129276, -1.3952618233065952, 0.54127878073129287, 1.3952618233065956]} + - {mu0: 0, rank: 4, ndensity: 2, num_ms_combs: 4, mus: [0, 0, 0, 0], ns: [1, 2, 2, 2], ls: [2, 0, 1, 1], ms_combs: [0, 0, -1, 1, 0, 0, 0, 0, 1, 0, -1, 0, 2, 0, -1, -1], ctildes: [0.63456947186912349, 0.071544968845218007, 0.63456947186912338, 0.071544968845217993, -2.1982131324189429, -0.24783904213170005, 1.5543714124267569, 0.17524866733410357]} + - {mu0: 0, rank: 4, ndensity: 2, num_ms_combs: 7, mus: [0, 0, 0, 0], ns: [1, 2, 2, 2], ls: [2, 0, 2, 2], ms_combs: [0, 0, -2, 2, 0, 0, -1, 1, 0, 0, 0, 0, 1, 0, -2, 1, 1, 0, -1, 0, 2, 0, -2, 0, 2, 0, -1, -1], ctildes: [0.51702434959212995, -2.3668206859066432, 0.25851217479606498, -1.1834103429533216, -0.25851217479606498, 1.1834103429533216, -1.2664458410950665, 5.7975029931353701, 0.51702434959212995, -2.3668206859066432, 1.0340486991842603, -4.733641371813289, -0.63322292054753326, 2.898751496567685]} + - {mu0: 0, rank: 4, ndensity: 2, num_ms_combs: 13, mus: [0, 0, 0, 0], ns: [1, 2, 2, 2], ls: [2, 1, 1, 2], ms_combs: [0, -1, 1, 0, 0, 0, 0, 0, 0, 0, 1, -1, 0, 1, 1, -2, 1, -1, -1, 1, 1, -1, 0, 0, 1, -1, 1, -1, 1, 0, 0, -1, 1, 0, 1, -2, 2, -1, -1, 0, 2, -1, 0, -1, 2, -1, 1, -2, 2, 0, 0, -2], ctildes: [0.36656511585219032, 0.62343799571388869, -0.14617668374188925, 0.0027891586709707606, -0.042846172897535284, -0.36316273766744939, -0.060593638807477007, -0.51358966895784952, 0.074211748368411795, 0.62901631305583006, -0.042846172897535312, -0.36316273766744944, -0.70839298224824354, -1.037203887075834, 0.31709061693991586, 0.2040937870100018, 0.10495126103002739, 0.88956340087747554, -0.060593638807477007, -0.51358966895784952, 0.10495126103002739, 0.88956340087747554, 0.63418123387983172, 0.40818757402000377, -0.39130236530832757, -0.83311010006583197]} + - {mu0: 0, rank: 4, ndensity: 2, num_ms_combs: 9, mus: [0, 0, 0, 0], ns: [1, 2, 2, 2], ls: [2, 2, 2, 2], ms_combs: [0, -2, 0, 2, 0, -1, 0, 1, 0, 0, 0, 0, 1, -2, -1, 2, 1, -1, -1, 1, 1, -1, 0, 0, 2, -2, -2, 2, 2, -2, -1, 1, 2, -2, 0, 0], ctildes: [0.042659807567752367, -0.39124798251406701, -0.042659807567752353, 0.39124798251406689, 0.021329903783876173, -0.19562399125703342, -0.085319615135504748, 0.78249596502813412, 0.08531961513550472, -0.7824959650281339, -0.042659807567752353, 0.39124798251406689, 0.085319615135504762, -0.78249596502813423, -0.085319615135504748, 0.78249596502813412, 0.042659807567752367, -0.39124798251406701]} + - {mu0: 0, rank: 4, ndensity: 2, num_ms_combs: 1, mus: [0, 0, 0, 0], ns: [2, 2, 2, 2], ls: [0, 0, 0, 0], ms_combs: [0, 0, 0, 0], ctildes: [-0.41945604382116003, -0.23977817996350323]} + - {mu0: 0, rank: 4, ndensity: 2, num_ms_combs: 2, mus: [0, 0, 0, 0], ns: [2, 2, 2, 2], ls: [0, 0, 1, 1], ms_combs: [0, 0, 0, 0, 0, 0, 1, -1], ctildes: [0.35825695229427768, 0.045372486525354, -0.71651390458855524, -0.090744973050707986]} + - {mu0: 0, rank: 4, ndensity: 2, num_ms_combs: 3, mus: [0, 0, 0, 0], ns: [2, 2, 2, 2], ls: [0, 0, 2, 2], ms_combs: [0, 0, 0, 0, 0, 0, 1, -1, 0, 0, 2, -2], ctildes: [-0.34241965045542627, 0.21338277629482588, 0.68483930091085266, -0.42676555258965182, -0.68483930091085277, 0.42676555258965193]} + - {mu0: 0, rank: 4, ndensity: 2, num_ms_combs: 4, mus: [0, 0, 0, 0], ns: [2, 2, 2, 2], ls: [0, 1, 1, 2], ms_combs: [0, -1, 1, 0, 0, 0, 0, 0, 0, 0, 1, -1, 0, 1, 1, -2], ctildes: [1.0028637347660057, 0.093781926411135294, 1.0028637347660057, 0.093781926411135308, -3.4740218833660013, -0.32487012275154392, 2.4565044317185607, 0.22971786680252276]} + - {mu0: 0, rank: 4, ndensity: 2, num_ms_combs: 5, mus: [0, 0, 0, 0], ns: [2, 2, 2, 2], ls: [0, 2, 2, 2], ms_combs: [0, -2, 0, 2, 0, -2, 1, 1, 0, -1, -1, 2, 0, -1, 0, 1, 0, 0, 0, 0], ctildes: [-0.67724613131646771, 0.057017899602750201, 0.5529691506664256, -0.046554920077325881, 0.2764845753332128, -0.023277460038662941, -0.33862306565823375, 0.02850894980137509, 0.11287435521941125, -0.00950298326712503]} + - {mu0: 0, rank: 4, ndensity: 2, num_ms_combs: 3, mus: [0, 0, 0, 0], ns: [2, 2, 2, 2], ls: [1, 1, 1, 1], ms_combs: [-1, -1, 1, 1, -1, 0, 0, 1, 0, 0, 0, 0], ctildes: [1.4024523916798897, 4.6763860807116098, -1.4024523916798901, -4.6763860807116107, 0.35061309791997264, 1.1690965201779031]} + - {mu0: 0, rank: 4, ndensity: 2, num_ms_combs: 10, mus: [0, 0, 0, 0], ns: [2, 2, 2, 2], ls: [1, 1, 2, 2], ms_combs: [-1, 1, -2, 2, -1, 1, -1, 1, -1, 1, 0, 0, 0, 0, -2, 2, 0, 0, -1, 1, 0, 0, 0, 0, 0, 1, -2, 1, 0, 1, -1, 0, 1, 1, -2, 0, 1, 1, -1, -1], ctildes: [-0.063890653658620289, -1.5988188832577952, 0.23145208264144856, 1.5844904109173643, -0.14365294615119567, -0.78985712673527686, 0.19950675581213848, 0.78508096928846716, -0.031945326829310151, -0.7994094416288976, -0.011954241415816323, 0.40209279953785382, -0.47393529079626412, 0.040527039823849274, 0.19348327225806861, -0.016545094725647361, 0.27362666771968674, -0.023398297351758073, -0.16756142898282836, 0.014328472340430545]} + - {mu0: 0, rank: 4, ndensity: 2, num_ms_combs: 6, mus: [0, 0, 0, 0], ns: [2, 2, 2, 2], ls: [2, 2, 2, 2], ms_combs: [-2, -2, 2, 2, -2, -1, 1, 2, -2, 0, 0, 2, -1, -1, 1, 1, -1, 0, 0, 1, 0, 0, 0, 0], ctildes: [-0.038537565230940561, -0.084465736161037239, 0.077075130461881108, 0.16893147232207442, -0.038537565230940547, -0.084465736161037197, -0.03853756523094054, -0.084465736161037197, 0.03853756523094054, 0.084465736161037183, -0.0096343913077351333, -0.021116434040259292]} diff --git a/examples/PACKAGES/dispersion/potential_files/d2.table b/examples/PACKAGES/dispersion/potential_files/d2.table new file mode 100644 index 0000000000..e9df2343d3 --- /dev/null +++ b/examples/PACKAGES/dispersion/potential_files/d2.table @@ -0,0 +1,20005 @@ +# DATE: 2021-12-16 UNITS: metal CONTRIBUTOR: ... (header line) + +D2 +N 20000 R 0.20000 19.50000 + +0 0.20000 -0.001737059 -0.040148545 +1 0.20097 -0.001698861 -0.039020933 +2 0.20193 -0.001661732 -0.037931035 +3 0.20290 -0.001625638 -0.036877398 +4 0.20386 -0.001590544 -0.035858632 +5 0.20483 -0.001556417 -0.034873408 +6 0.20579 -0.001523225 -0.033920452 +7 0.20676 -0.001490937 -0.032998547 +8 0.20772 -0.001459525 -0.032106526 +9 0.20869 -0.001428959 -0.031243272 +10 0.20965 -0.001399213 -0.030407716 +11 0.21062 -0.001370261 -0.029598833 +12 0.21158 -0.001342076 -0.028815640 +13 0.21255 -0.001314636 -0.028057196 +14 0.21351 -0.001287916 -0.027322599 +15 0.21448 -0.001261893 -0.026610984 +16 0.21544 -0.001236547 -0.025921519 +17 0.21641 -0.001211855 -0.025253410 +18 0.21737 -0.001187799 -0.024605893 +19 0.21834 -0.001164357 -0.023978235 +20 0.21930 -0.001141512 -0.023369732 +21 0.22027 -0.001119245 -0.022779709 +22 0.22123 -0.001097539 -0.022207517 +23 0.22220 -0.001076377 -0.021652534 +24 0.22316 -0.001055743 -0.021114162 +25 0.22413 -0.001035620 -0.020591826 +26 0.22509 -0.001015993 -0.020084974 +27 0.22606 -0.000996849 -0.019593074 +28 0.22702 -0.000978172 -0.019115618 +29 0.22799 -0.000959949 -0.018652113 +30 0.22895 -0.000942168 -0.018202090 +31 0.22992 -0.000924814 -0.017765093 +32 0.23088 -0.000907875 -0.017340687 +33 0.23185 -0.000891340 -0.016928452 +34 0.23281 -0.000875198 -0.016527983 +35 0.23378 -0.000859436 -0.016138892 +36 0.23474 -0.000844045 -0.015760805 +37 0.23571 -0.000829013 -0.015393361 +38 0.23667 -0.000814331 -0.015036214 +39 0.23764 -0.000799988 -0.014689028 +40 0.23860 -0.000785976 -0.014351482 +41 0.23957 -0.000772286 -0.014023265 +42 0.24053 -0.000758907 -0.013704080 +43 0.24150 -0.000745833 -0.013393637 +44 0.24246 -0.000733053 -0.013091660 +45 0.24343 -0.000720562 -0.012797880 +46 0.24439 -0.000708350 -0.012512039 +47 0.24536 -0.000696410 -0.012233889 +48 0.24632 -0.000684735 -0.011963190 +49 0.24729 -0.000673317 -0.011699709 +50 0.24825 -0.000662151 -0.011443224 +51 0.24922 -0.000651229 -0.011193518 +52 0.25018 -0.000640544 -0.010950385 +53 0.25115 -0.000630091 -0.010713623 +54 0.25211 -0.000619864 -0.010483037 +55 0.25308 -0.000609856 -0.010258442 +56 0.25404 -0.000600062 -0.010039656 +57 0.25501 -0.000590477 -0.009826505 +58 0.25597 -0.000581094 -0.009618820 +59 0.25694 -0.000571910 -0.009416438 +60 0.25790 -0.000562918 -0.009219201 +61 0.25887 -0.000554114 -0.009026959 +62 0.25983 -0.000545494 -0.008839563 +63 0.26080 -0.000537052 -0.008656871 +64 0.26176 -0.000528784 -0.008478747 +65 0.26273 -0.000520685 -0.008305058 +66 0.26369 -0.000512753 -0.008135674 +67 0.26466 -0.000504981 -0.007970472 +68 0.26562 -0.000497368 -0.007809331 +69 0.26659 -0.000489907 -0.007652137 +70 0.26755 -0.000482597 -0.007498774 +71 0.26852 -0.000475433 -0.007349137 +72 0.26948 -0.000468411 -0.007203117 +73 0.27045 -0.000461529 -0.007060615 +74 0.27141 -0.000454782 -0.006921531 +75 0.27238 -0.000448169 -0.006785770 +76 0.27334 -0.000441684 -0.006653239 +77 0.27431 -0.000435326 -0.006523848 +78 0.27527 -0.000429092 -0.006397512 +79 0.27624 -0.000422977 -0.006274145 +80 0.27720 -0.000416981 -0.006153667 +81 0.27817 -0.000411099 -0.006035998 +82 0.27913 -0.000405330 -0.005921063 +83 0.28010 -0.000399670 -0.005808786 +84 0.28106 -0.000394118 -0.005699097 +85 0.28203 -0.000388670 -0.005591926 +86 0.28299 -0.000383324 -0.005487205 +87 0.28396 -0.000378078 -0.005384869 +88 0.28492 -0.000372930 -0.005284855 +89 0.28589 -0.000367877 -0.005187101 +90 0.28685 -0.000362918 -0.005091547 +91 0.28782 -0.000358049 -0.004998136 +92 0.28878 -0.000353270 -0.004906811 +93 0.28975 -0.000348578 -0.004817518 +94 0.29071 -0.000343971 -0.004730205 +95 0.29168 -0.000339448 -0.004644819 +96 0.29264 -0.000335006 -0.004561312 +97 0.29361 -0.000330643 -0.004479634 +98 0.29457 -0.000326359 -0.004399739 +99 0.29554 -0.000322151 -0.004321582 +100 0.29650 -0.000318017 -0.004245118 +101 0.29747 -0.000313957 -0.004170305 +102 0.29843 -0.000309968 -0.004097100 +103 0.29940 -0.000306048 -0.004025464 +104 0.30037 -0.000302198 -0.003955356 +105 0.30133 -0.000298414 -0.003886739 +106 0.30230 -0.000294695 -0.003819576 +107 0.30326 -0.000291041 -0.003753831 +108 0.30423 -0.000287450 -0.003689468 +109 0.30519 -0.000283920 -0.003626453 +110 0.30616 -0.000280450 -0.003564754 +111 0.30712 -0.000277039 -0.003504338 +112 0.30809 -0.000273686 -0.003445174 +113 0.30905 -0.000270389 -0.003387231 +114 0.31002 -0.000267148 -0.003330481 +115 0.31098 -0.000263960 -0.003274894 +116 0.31195 -0.000260826 -0.003220442 +117 0.31291 -0.000257744 -0.003167098 +118 0.31388 -0.000254713 -0.003114836 +119 0.31484 -0.000251732 -0.003063630 +120 0.31581 -0.000248800 -0.003013455 +121 0.31677 -0.000245915 -0.002964286 +122 0.31774 -0.000243078 -0.002916101 +123 0.31870 -0.000240287 -0.002868875 +124 0.31967 -0.000237541 -0.002822586 +125 0.32063 -0.000234839 -0.002777213 +126 0.32160 -0.000232180 -0.002732734 +127 0.32256 -0.000229564 -0.002689129 +128 0.32353 -0.000226990 -0.002646377 +129 0.32449 -0.000224456 -0.002604459 +130 0.32546 -0.000221962 -0.002563356 +131 0.32642 -0.000219508 -0.002523048 +132 0.32739 -0.000217092 -0.002483519 +133 0.32835 -0.000214714 -0.002444750 +134 0.32932 -0.000212374 -0.002406724 +135 0.33028 -0.000210069 -0.002369425 +136 0.33125 -0.000207800 -0.002332835 +137 0.33221 -0.000205566 -0.002296940 +138 0.33318 -0.000203367 -0.002261724 +139 0.33414 -0.000201201 -0.002227171 +140 0.33511 -0.000199068 -0.002193266 +141 0.33607 -0.000196967 -0.002159997 +142 0.33704 -0.000194899 -0.002127348 +143 0.33800 -0.000192861 -0.002095305 +144 0.33897 -0.000190854 -0.002063857 +145 0.33993 -0.000188877 -0.002032989 +146 0.34090 -0.000186930 -0.002002689 +147 0.34186 -0.000185012 -0.001972945 +148 0.34283 -0.000183122 -0.001943745 +149 0.34379 -0.000181260 -0.001915076 +150 0.34476 -0.000179426 -0.001886928 +151 0.34572 -0.000177618 -0.001859290 +152 0.34669 -0.000175837 -0.001832150 +153 0.34765 -0.000174082 -0.001805498 +154 0.34862 -0.000172352 -0.001779324 +155 0.34958 -0.000170647 -0.001753618 +156 0.35055 -0.000168967 -0.001728369 +157 0.35151 -0.000167311 -0.001703568 +158 0.35248 -0.000165679 -0.001679206 +159 0.35344 -0.000164070 -0.001655273 +160 0.35441 -0.000162484 -0.001631761 +161 0.35537 -0.000160920 -0.001608661 +162 0.35634 -0.000159379 -0.001585964 +163 0.35730 -0.000157859 -0.001563662 +164 0.35827 -0.000156361 -0.001541747 +165 0.35923 -0.000154883 -0.001520211 +166 0.36020 -0.000153426 -0.001499046 +167 0.36116 -0.000151990 -0.001478245 +168 0.36213 -0.000150573 -0.001457800 +169 0.36309 -0.000149176 -0.001437704 +170 0.36406 -0.000147798 -0.001417950 +171 0.36502 -0.000146439 -0.001398532 +172 0.36599 -0.000145099 -0.001379442 +173 0.36695 -0.000143777 -0.001360673 +174 0.36792 -0.000142472 -0.001342220 +175 0.36888 -0.000141186 -0.001324076 +176 0.36985 -0.000139917 -0.001306236 +177 0.37081 -0.000138665 -0.001288692 +178 0.37178 -0.000137429 -0.001271439 +179 0.37274 -0.000136211 -0.001254472 +180 0.37371 -0.000135008 -0.001237785 +181 0.37467 -0.000133821 -0.001221372 +182 0.37564 -0.000132651 -0.001205228 +183 0.37660 -0.000131495 -0.001189348 +184 0.37757 -0.000130355 -0.001173727 +185 0.37853 -0.000129230 -0.001158360 +186 0.37950 -0.000128119 -0.001143241 +187 0.38046 -0.000127023 -0.001128366 +188 0.38143 -0.000125941 -0.001113731 +189 0.38239 -0.000124873 -0.001099330 +190 0.38336 -0.000123819 -0.001085160 +191 0.38432 -0.000122779 -0.001071216 +192 0.38529 -0.000121752 -0.001057494 +193 0.38625 -0.000120738 -0.001043989 +194 0.38722 -0.000119737 -0.001030697 +195 0.38818 -0.000118748 -0.001017615 +196 0.38915 -0.000117772 -0.001004739 +197 0.39011 -0.000116809 -0.000992064 +198 0.39108 -0.000115858 -0.000979587 +199 0.39204 -0.000114918 -0.000967304 +200 0.39301 -0.000113991 -0.000955212 +201 0.39397 -0.000113074 -0.000943307 +202 0.39494 -0.000112170 -0.000931586 +203 0.39590 -0.000111276 -0.000920045 +204 0.39687 -0.000110394 -0.000908681 +205 0.39783 -0.000109522 -0.000897491 +206 0.39880 -0.000108662 -0.000886471 +207 0.39976 -0.000107811 -0.000875619 +208 0.40073 -0.000106972 -0.000864932 +209 0.40170 -0.000106142 -0.000854406 +210 0.40266 -0.000105322 -0.000844038 +211 0.40363 -0.000104513 -0.000833827 +212 0.40459 -0.000103713 -0.000823768 +213 0.40556 -0.000102923 -0.000813860 +214 0.40652 -0.000102142 -0.000804099 +215 0.40749 -0.000101371 -0.000794484 +216 0.40845 -0.000100609 -0.000785011 +217 0.40942 -0.000099856 -0.000775677 +218 0.41038 -0.000099111 -0.000766481 +219 0.41135 -0.000098376 -0.000757420 +220 0.41231 -0.000097650 -0.000748492 +221 0.41328 -0.000096931 -0.000739694 +222 0.41424 -0.000096222 -0.000731024 +223 0.41521 -0.000095520 -0.000722480 +224 0.41617 -0.000094827 -0.000714060 +225 0.41714 -0.000094142 -0.000705761 +226 0.41810 -0.000093465 -0.000697582 +227 0.41907 -0.000092796 -0.000689520 +228 0.42003 -0.000092134 -0.000681573 +229 0.42100 -0.000091480 -0.000673740 +230 0.42196 -0.000090834 -0.000666018 +231 0.42293 -0.000090195 -0.000658406 +232 0.42389 -0.000089563 -0.000650901 +233 0.42486 -0.000088938 -0.000643503 +234 0.42582 -0.000088321 -0.000636208 +235 0.42679 -0.000087710 -0.000629016 +236 0.42775 -0.000087107 -0.000621924 +237 0.42872 -0.000086510 -0.000614932 +238 0.42968 -0.000085920 -0.000608036 +239 0.43065 -0.000085336 -0.000601237 +240 0.43161 -0.000084759 -0.000594531 +241 0.43258 -0.000084189 -0.000587918 +242 0.43354 -0.000083625 -0.000581396 +243 0.43451 -0.000083067 -0.000574964 +244 0.43547 -0.000082515 -0.000568619 +245 0.43644 -0.000081969 -0.000562361 +246 0.43740 -0.000081429 -0.000556189 +247 0.43837 -0.000080896 -0.000550100 +248 0.43933 -0.000080368 -0.000544093 +249 0.44030 -0.000079846 -0.000538168 +250 0.44126 -0.000079329 -0.000532322 +251 0.44223 -0.000078818 -0.000526555 +252 0.44319 -0.000078313 -0.000520865 +253 0.44416 -0.000077813 -0.000515251 +254 0.44512 -0.000077318 -0.000509711 +255 0.44609 -0.000076829 -0.000504246 +256 0.44705 -0.000076345 -0.000498852 +257 0.44802 -0.000075866 -0.000493530 +258 0.44898 -0.000075392 -0.000488278 +259 0.44995 -0.000074924 -0.000483095 +260 0.45091 -0.000074460 -0.000477980 +261 0.45188 -0.000074001 -0.000472932 +262 0.45284 -0.000073547 -0.000467949 +263 0.45381 -0.000073098 -0.000463031 +264 0.45477 -0.000072653 -0.000458177 +265 0.45574 -0.000072213 -0.000453386 +266 0.45670 -0.000071778 -0.000448656 +267 0.45767 -0.000071347 -0.000443988 +268 0.45863 -0.000070921 -0.000439379 +269 0.45960 -0.000070499 -0.000434829 +270 0.46056 -0.000070082 -0.000430337 +271 0.46153 -0.000069669 -0.000425902 +272 0.46249 -0.000069260 -0.000421524 +273 0.46346 -0.000068855 -0.000417201 +274 0.46442 -0.000068455 -0.000412932 +275 0.46539 -0.000068058 -0.000408717 +276 0.46635 -0.000067666 -0.000404555 +277 0.46732 -0.000067277 -0.000400446 +278 0.46828 -0.000066893 -0.000396387 +279 0.46925 -0.000066512 -0.000392379 +280 0.47021 -0.000066136 -0.000388421 +281 0.47118 -0.000065763 -0.000384512 +282 0.47214 -0.000065393 -0.000380651 +283 0.47311 -0.000065028 -0.000376838 +284 0.47407 -0.000064666 -0.000373072 +285 0.47504 -0.000064308 -0.000369352 +286 0.47600 -0.000063953 -0.000365677 +287 0.47697 -0.000063602 -0.000362047 +288 0.47793 -0.000063254 -0.000358461 +289 0.47890 -0.000062910 -0.000354919 +290 0.47986 -0.000062569 -0.000351420 +291 0.48083 -0.000062232 -0.000347963 +292 0.48179 -0.000061898 -0.000344547 +293 0.48276 -0.000061567 -0.000341173 +294 0.48372 -0.000061239 -0.000337839 +295 0.48469 -0.000060915 -0.000334544 +296 0.48565 -0.000060593 -0.000331289 +297 0.48662 -0.000060275 -0.000328073 +298 0.48758 -0.000059960 -0.000324895 +299 0.48855 -0.000059648 -0.000321754 +300 0.48951 -0.000059339 -0.000318651 +301 0.49048 -0.000059033 -0.000315584 +302 0.49144 -0.000058730 -0.000312553 +303 0.49241 -0.000058430 -0.000309557 +304 0.49337 -0.000058133 -0.000306597 +305 0.49434 -0.000057838 -0.000303670 +306 0.49530 -0.000057546 -0.000300778 +307 0.49627 -0.000057258 -0.000297919 +308 0.49723 -0.000056971 -0.000295094 +309 0.49820 -0.000056688 -0.000292301 +310 0.49916 -0.000056407 -0.000289540 +311 0.50013 -0.000056129 -0.000286810 +312 0.50110 -0.000055854 -0.000284112 +313 0.50206 -0.000055581 -0.000281445 +314 0.50303 -0.000055310 -0.000278808 +315 0.50399 -0.000055043 -0.000276201 +316 0.50496 -0.000054777 -0.000273623 +317 0.50592 -0.000054515 -0.000271075 +318 0.50689 -0.000054254 -0.000268555 +319 0.50785 -0.000053996 -0.000266063 +320 0.50882 -0.000053741 -0.000263600 +321 0.50978 -0.000053487 -0.000261164 +322 0.51075 -0.000053237 -0.000258755 +323 0.51171 -0.000052988 -0.000256373 +324 0.51268 -0.000052742 -0.000254017 +325 0.51364 -0.000052498 -0.000251688 +326 0.51461 -0.000052256 -0.000249384 +327 0.51557 -0.000052016 -0.000247105 +328 0.51654 -0.000051779 -0.000244852 +329 0.51750 -0.000051544 -0.000242623 +330 0.51847 -0.000051311 -0.000240419 +331 0.51943 -0.000051080 -0.000238238 +332 0.52040 -0.000050851 -0.000236081 +333 0.52136 -0.000050624 -0.000233948 +334 0.52233 -0.000050399 -0.000231838 +335 0.52329 -0.000050177 -0.000229750 +336 0.52426 -0.000049956 -0.000227685 +337 0.52522 -0.000049737 -0.000225642 +338 0.52619 -0.000049520 -0.000223621 +339 0.52715 -0.000049305 -0.000221622 +340 0.52812 -0.000049093 -0.000219644 +341 0.52908 -0.000048882 -0.000217687 +342 0.53005 -0.000048672 -0.000215751 +343 0.53101 -0.000048465 -0.000213835 +344 0.53198 -0.000048260 -0.000211939 +345 0.53294 -0.000048056 -0.000210063 +346 0.53391 -0.000047854 -0.000208207 +347 0.53487 -0.000047654 -0.000206371 +348 0.53584 -0.000047456 -0.000204553 +349 0.53680 -0.000047259 -0.000202755 +350 0.53777 -0.000047065 -0.000200975 +351 0.53873 -0.000046871 -0.000199214 +352 0.53970 -0.000046680 -0.000197470 +353 0.54066 -0.000046490 -0.000195745 +354 0.54163 -0.000046302 -0.000194038 +355 0.54259 -0.000046116 -0.000192348 +356 0.54356 -0.000045931 -0.000190675 +357 0.54452 -0.000045748 -0.000189020 +358 0.54549 -0.000045566 -0.000187381 +359 0.54645 -0.000045386 -0.000185759 +360 0.54742 -0.000045208 -0.000184153 +361 0.54838 -0.000045031 -0.000182564 +362 0.54935 -0.000044855 -0.000180991 +363 0.55031 -0.000044681 -0.000179433 +364 0.55128 -0.000044509 -0.000177891 +365 0.55224 -0.000044338 -0.000176365 +366 0.55321 -0.000044168 -0.000174854 +367 0.55417 -0.000044000 -0.000173357 +368 0.55514 -0.000043834 -0.000171876 +369 0.55610 -0.000043669 -0.000170409 +370 0.55707 -0.000043505 -0.000168957 +371 0.55803 -0.000043343 -0.000167519 +372 0.55900 -0.000043182 -0.000166096 +373 0.55996 -0.000043022 -0.000164686 +374 0.56093 -0.000042864 -0.000163290 +375 0.56189 -0.000042707 -0.000161908 +376 0.56286 -0.000042551 -0.000160539 +377 0.56382 -0.000042397 -0.000159183 +378 0.56479 -0.000042244 -0.000157841 +379 0.56575 -0.000042092 -0.000156511 +380 0.56672 -0.000041942 -0.000155194 +381 0.56768 -0.000041793 -0.000153890 +382 0.56865 -0.000041645 -0.000152599 +383 0.56961 -0.000041498 -0.000151320 +384 0.57058 -0.000041353 -0.000150053 +385 0.57154 -0.000041209 -0.000148798 +386 0.57251 -0.000041066 -0.000147555 +387 0.57347 -0.000040924 -0.000146323 +388 0.57444 -0.000040783 -0.000145104 +389 0.57540 -0.000040644 -0.000143896 +390 0.57637 -0.000040506 -0.000142699 +391 0.57733 -0.000040368 -0.000141513 +392 0.57830 -0.000040232 -0.000140339 +393 0.57926 -0.000040097 -0.000139175 +394 0.58023 -0.000039964 -0.000138022 +395 0.58119 -0.000039831 -0.000136880 +396 0.58216 -0.000039700 -0.000135749 +397 0.58312 -0.000039569 -0.000134628 +398 0.58409 -0.000039440 -0.000133517 +399 0.58505 -0.000039311 -0.000132417 +400 0.58602 -0.000039184 -0.000131326 +401 0.58698 -0.000039058 -0.000130246 +402 0.58795 -0.000038933 -0.000129175 +403 0.58891 -0.000038809 -0.000128114 +404 0.58988 -0.000038685 -0.000127063 +405 0.59084 -0.000038563 -0.000126021 +406 0.59181 -0.000038442 -0.000124989 +407 0.59277 -0.000038322 -0.000123966 +408 0.59374 -0.000038203 -0.000122952 +409 0.59470 -0.000038085 -0.000121947 +410 0.59567 -0.000037968 -0.000120951 +411 0.59663 -0.000037851 -0.000119964 +412 0.59760 -0.000037736 -0.000118986 +413 0.59856 -0.000037622 -0.000118016 +414 0.59953 -0.000037508 -0.000117055 +415 0.60050 -0.000037396 -0.000116103 +416 0.60146 -0.000037284 -0.000115159 +417 0.60243 -0.000037173 -0.000114223 +418 0.60339 -0.000037064 -0.000113295 +419 0.60436 -0.000036955 -0.000112375 +420 0.60532 -0.000036847 -0.000111464 +421 0.60629 -0.000036740 -0.000110560 +422 0.60725 -0.000036633 -0.000109664 +423 0.60822 -0.000036528 -0.000108776 +424 0.60918 -0.000036423 -0.000107895 +425 0.61015 -0.000036320 -0.000107022 +426 0.61111 -0.000036217 -0.000106157 +427 0.61208 -0.000036115 -0.000105299 +428 0.61304 -0.000036014 -0.000104448 +429 0.61401 -0.000035913 -0.000103604 +430 0.61497 -0.000035814 -0.000102768 +431 0.61594 -0.000035715 -0.000101938 +432 0.61690 -0.000035617 -0.000101116 +433 0.61787 -0.000035520 -0.000100300 +434 0.61883 -0.000035423 -0.000099492 +435 0.61980 -0.000035328 -0.000098690 +436 0.62076 -0.000035233 -0.000097895 +437 0.62173 -0.000035139 -0.000097106 +438 0.62269 -0.000035045 -0.000096324 +439 0.62366 -0.000034953 -0.000095548 +440 0.62462 -0.000034861 -0.000094779 +441 0.62559 -0.000034770 -0.000094016 +442 0.62655 -0.000034680 -0.000093259 +443 0.62752 -0.000034590 -0.000092509 +444 0.62848 -0.000034501 -0.000091765 +445 0.62945 -0.000034413 -0.000091026 +446 0.63041 -0.000034325 -0.000090294 +447 0.63138 -0.000034239 -0.000089568 +448 0.63234 -0.000034152 -0.000088847 +449 0.63331 -0.000034067 -0.000088132 +450 0.63427 -0.000033982 -0.000087423 +451 0.63524 -0.000033898 -0.000086720 +452 0.63620 -0.000033815 -0.000086022 +453 0.63717 -0.000033732 -0.000085330 +454 0.63813 -0.000033650 -0.000084644 +455 0.63910 -0.000033569 -0.000083962 +456 0.64006 -0.000033488 -0.000083286 +457 0.64103 -0.000033408 -0.000082616 +458 0.64199 -0.000033329 -0.000081951 +459 0.64296 -0.000033250 -0.000081290 +460 0.64392 -0.000033172 -0.000080636 +461 0.64489 -0.000033094 -0.000079986 +462 0.64585 -0.000033017 -0.000079341 +463 0.64682 -0.000032941 -0.000078701 +464 0.64778 -0.000032866 -0.000078066 +465 0.64875 -0.000032790 -0.000077436 +466 0.64971 -0.000032716 -0.000076811 +467 0.65068 -0.000032642 -0.000076190 +468 0.65164 -0.000032569 -0.000075575 +469 0.65261 -0.000032496 -0.000074964 +470 0.65357 -0.000032424 -0.000074357 +471 0.65454 -0.000032353 -0.000073756 +472 0.65550 -0.000032282 -0.000073158 +473 0.65647 -0.000032212 -0.000072566 +474 0.65743 -0.000032142 -0.000071977 +475 0.65840 -0.000032073 -0.000071393 +476 0.65936 -0.000032004 -0.000070814 +477 0.66033 -0.000031936 -0.000070238 +478 0.66129 -0.000031869 -0.000069667 +479 0.66226 -0.000031802 -0.000069100 +480 0.66322 -0.000031735 -0.000068538 +481 0.66419 -0.000031669 -0.000067979 +482 0.66515 -0.000031604 -0.000067425 +483 0.66612 -0.000031539 -0.000066874 +484 0.66708 -0.000031475 -0.000066328 +485 0.66805 -0.000031411 -0.000065785 +486 0.66901 -0.000031348 -0.000065247 +487 0.66998 -0.000031285 -0.000064712 +488 0.67094 -0.000031223 -0.000064181 +489 0.67191 -0.000031161 -0.000063654 +490 0.67287 -0.000031100 -0.000063131 +491 0.67384 -0.000031039 -0.000062612 +492 0.67480 -0.000030979 -0.000062096 +493 0.67577 -0.000030920 -0.000061583 +494 0.67673 -0.000030860 -0.000061075 +495 0.67770 -0.000030802 -0.000060570 +496 0.67866 -0.000030744 -0.000060068 +497 0.67963 -0.000030686 -0.000059570 +498 0.68059 -0.000030629 -0.000059076 +499 0.68156 -0.000030572 -0.000058585 +500 0.68252 -0.000030515 -0.000058097 +501 0.68349 -0.000030460 -0.000057612 +502 0.68445 -0.000030404 -0.000057131 +503 0.68542 -0.000030349 -0.000056654 +504 0.68638 -0.000030295 -0.000056179 +505 0.68735 -0.000030241 -0.000055708 +506 0.68831 -0.000030187 -0.000055240 +507 0.68928 -0.000030134 -0.000054775 +508 0.69024 -0.000030082 -0.000054313 +509 0.69121 -0.000030029 -0.000053854 +510 0.69217 -0.000029978 -0.000053399 +511 0.69314 -0.000029926 -0.000052946 +512 0.69410 -0.000029876 -0.000052497 +513 0.69507 -0.000029825 -0.000052050 +514 0.69603 -0.000029775 -0.000051606 +515 0.69700 -0.000029726 -0.000051166 +516 0.69796 -0.000029676 -0.000050728 +517 0.69893 -0.000029628 -0.000050293 +518 0.69989 -0.000029579 -0.000049861 +519 0.70086 -0.000029531 -0.000049431 +520 0.70183 -0.000029484 -0.000049005 +521 0.70279 -0.000029437 -0.000048581 +522 0.70376 -0.000029390 -0.000048160 +523 0.70472 -0.000029344 -0.000047741 +524 0.70569 -0.000029298 -0.000047326 +525 0.70665 -0.000029252 -0.000046913 +526 0.70762 -0.000029207 -0.000046502 +527 0.70858 -0.000029163 -0.000046094 +528 0.70955 -0.000029118 -0.000045689 +529 0.71051 -0.000029075 -0.000045286 +530 0.71148 -0.000029031 -0.000044886 +531 0.71244 -0.000028988 -0.000044488 +532 0.71341 -0.000028945 -0.000044092 +533 0.71437 -0.000028903 -0.000043700 +534 0.71534 -0.000028861 -0.000043309 +535 0.71630 -0.000028819 -0.000042921 +536 0.71727 -0.000028778 -0.000042535 +537 0.71823 -0.000028737 -0.000042152 +538 0.71920 -0.000028697 -0.000041771 +539 0.72016 -0.000028656 -0.000041392 +540 0.72113 -0.000028617 -0.000041015 +541 0.72209 -0.000028577 -0.000040641 +542 0.72306 -0.000028538 -0.000040269 +543 0.72402 -0.000028500 -0.000039899 +544 0.72499 -0.000028461 -0.000039532 +545 0.72595 -0.000028423 -0.000039166 +546 0.72692 -0.000028386 -0.000038803 +547 0.72788 -0.000028348 -0.000038442 +548 0.72885 -0.000028311 -0.000038083 +549 0.72981 -0.000028275 -0.000037726 +550 0.73078 -0.000028239 -0.000037371 +551 0.73174 -0.000028203 -0.000037018 +552 0.73271 -0.000028167 -0.000036667 +553 0.73367 -0.000028132 -0.000036318 +554 0.73464 -0.000028097 -0.000035971 +555 0.73560 -0.000028063 -0.000035626 +556 0.73657 -0.000028028 -0.000035283 +557 0.73753 -0.000027994 -0.000034942 +558 0.73850 -0.000027961 -0.000034603 +559 0.73946 -0.000027928 -0.000034266 +560 0.74043 -0.000027895 -0.000033931 +561 0.74139 -0.000027862 -0.000033597 +562 0.74236 -0.000027830 -0.000033265 +563 0.74332 -0.000027798 -0.000032936 +564 0.74429 -0.000027766 -0.000032608 +565 0.74525 -0.000027735 -0.000032281 +566 0.74622 -0.000027704 -0.000031957 +567 0.74718 -0.000027673 -0.000031634 +568 0.74815 -0.000027643 -0.000031313 +569 0.74911 -0.000027613 -0.000030994 +570 0.75008 -0.000027583 -0.000030676 +571 0.75104 -0.000027554 -0.000030360 +572 0.75201 -0.000027525 -0.000030046 +573 0.75297 -0.000027496 -0.000029733 +574 0.75394 -0.000027467 -0.000029422 +575 0.75490 -0.000027439 -0.000029113 +576 0.75587 -0.000027411 -0.000028805 +577 0.75683 -0.000027383 -0.000028499 +578 0.75780 -0.000027356 -0.000028194 +579 0.75876 -0.000027329 -0.000027891 +580 0.75973 -0.000027302 -0.000027589 +581 0.76069 -0.000027276 -0.000027289 +582 0.76166 -0.000027249 -0.000026991 +583 0.76262 -0.000027224 -0.000026694 +584 0.76359 -0.000027198 -0.000026398 +585 0.76455 -0.000027173 -0.000026104 +586 0.76552 -0.000027148 -0.000025811 +587 0.76648 -0.000027123 -0.000025520 +588 0.76745 -0.000027098 -0.000025230 +589 0.76841 -0.000027074 -0.000024942 +590 0.76938 -0.000027050 -0.000024655 +591 0.77034 -0.000027027 -0.000024369 +592 0.77131 -0.000027003 -0.000024085 +593 0.77227 -0.000026980 -0.000023802 +594 0.77324 -0.000026957 -0.000023521 +595 0.77420 -0.000026935 -0.000023241 +596 0.77517 -0.000026912 -0.000022962 +597 0.77613 -0.000026890 -0.000022684 +598 0.77710 -0.000026869 -0.000022408 +599 0.77806 -0.000026847 -0.000022133 +600 0.77903 -0.000026826 -0.000021859 +601 0.77999 -0.000026805 -0.000021586 +602 0.78096 -0.000026784 -0.000021315 +603 0.78192 -0.000026764 -0.000021045 +604 0.78289 -0.000026744 -0.000020776 +605 0.78385 -0.000026724 -0.000020509 +606 0.78482 -0.000026704 -0.000020242 +607 0.78578 -0.000026685 -0.000019977 +608 0.78675 -0.000026665 -0.000019713 +609 0.78771 -0.000026647 -0.000019450 +610 0.78868 -0.000026628 -0.000019188 +611 0.78964 -0.000026609 -0.000018928 +612 0.79061 -0.000026591 -0.000018668 +613 0.79157 -0.000026573 -0.000018410 +614 0.79254 -0.000026556 -0.000018152 +615 0.79350 -0.000026538 -0.000017896 +616 0.79447 -0.000026521 -0.000017641 +617 0.79543 -0.000026504 -0.000017387 +618 0.79640 -0.000026488 -0.000017134 +619 0.79736 -0.000026471 -0.000016882 +620 0.79833 -0.000026455 -0.000016631 +621 0.79929 -0.000026439 -0.000016381 +622 0.80026 -0.000026424 -0.000016132 +623 0.80123 -0.000026408 -0.000015884 +624 0.80219 -0.000026393 -0.000015637 +625 0.80316 -0.000026378 -0.000015391 +626 0.80412 -0.000026363 -0.000015146 +627 0.80509 -0.000026349 -0.000014902 +628 0.80605 -0.000026334 -0.000014659 +629 0.80702 -0.000026320 -0.000014417 +630 0.80798 -0.000026307 -0.000014176 +631 0.80895 -0.000026293 -0.000013936 +632 0.80991 -0.000026280 -0.000013696 +633 0.81088 -0.000026267 -0.000013458 +634 0.81184 -0.000026254 -0.000013220 +635 0.81281 -0.000026241 -0.000012984 +636 0.81377 -0.000026229 -0.000012748 +637 0.81474 -0.000026216 -0.000012513 +638 0.81570 -0.000026204 -0.000012279 +639 0.81667 -0.000026193 -0.000012046 +640 0.81763 -0.000026181 -0.000011814 +641 0.81860 -0.000026170 -0.000011582 +642 0.81956 -0.000026159 -0.000011351 +643 0.82053 -0.000026148 -0.000011122 +644 0.82149 -0.000026137 -0.000010893 +645 0.82246 -0.000026127 -0.000010664 +646 0.82342 -0.000026117 -0.000010437 +647 0.82439 -0.000026107 -0.000010210 +648 0.82535 -0.000026097 -0.000009984 +649 0.82632 -0.000026088 -0.000009759 +650 0.82728 -0.000026078 -0.000009535 +651 0.82825 -0.000026069 -0.000009311 +652 0.82921 -0.000026060 -0.000009088 +653 0.83018 -0.000026052 -0.000008866 +654 0.83114 -0.000026043 -0.000008645 +655 0.83211 -0.000026035 -0.000008424 +656 0.83307 -0.000026027 -0.000008204 +657 0.83404 -0.000026019 -0.000007985 +658 0.83500 -0.000026012 -0.000007766 +659 0.83597 -0.000026004 -0.000007548 +660 0.83693 -0.000025997 -0.000007331 +661 0.83790 -0.000025990 -0.000007114 +662 0.83886 -0.000025983 -0.000006899 +663 0.83983 -0.000025977 -0.000006683 +664 0.84079 -0.000025970 -0.000006469 +665 0.84176 -0.000025964 -0.000006255 +666 0.84272 -0.000025958 -0.000006041 +667 0.84369 -0.000025953 -0.000005829 +668 0.84465 -0.000025947 -0.000005617 +669 0.84562 -0.000025942 -0.000005405 +670 0.84658 -0.000025937 -0.000005194 +671 0.84755 -0.000025932 -0.000004984 +672 0.84851 -0.000025927 -0.000004774 +673 0.84948 -0.000025922 -0.000004565 +674 0.85044 -0.000025918 -0.000004357 +675 0.85141 -0.000025914 -0.000004149 +676 0.85237 -0.000025910 -0.000003942 +677 0.85334 -0.000025906 -0.000003735 +678 0.85430 -0.000025903 -0.000003529 +679 0.85527 -0.000025900 -0.000003323 +680 0.85623 -0.000025896 -0.000003118 +681 0.85720 -0.000025894 -0.000002913 +682 0.85816 -0.000025891 -0.000002709 +683 0.85913 -0.000025888 -0.000002505 +684 0.86009 -0.000025886 -0.000002302 +685 0.86106 -0.000025884 -0.000002100 +686 0.86202 -0.000025882 -0.000001898 +687 0.86299 -0.000025880 -0.000001696 +688 0.86395 -0.000025879 -0.000001495 +689 0.86492 -0.000025877 -0.000001295 +690 0.86588 -0.000025876 -0.000001094 +691 0.86685 -0.000025875 -0.000000895 +692 0.86781 -0.000025874 -0.000000696 +693 0.86878 -0.000025874 -0.000000497 +694 0.86974 -0.000025874 -0.000000299 +695 0.87071 -0.000025873 -0.000000101 +696 0.87167 -0.000025873 0.000000097 +697 0.87264 -0.000025874 0.000000294 +698 0.87360 -0.000025874 0.000000490 +699 0.87457 -0.000025874 0.000000686 +700 0.87553 -0.000025875 0.000000882 +701 0.87650 -0.000025876 0.000001077 +702 0.87746 -0.000025877 0.000001272 +703 0.87843 -0.000025879 0.000001467 +704 0.87939 -0.000025880 0.000001661 +705 0.88036 -0.000025882 0.000001854 +706 0.88132 -0.000025884 0.000002048 +707 0.88229 -0.000025886 0.000002241 +708 0.88325 -0.000025888 0.000002433 +709 0.88422 -0.000025890 0.000002625 +710 0.88518 -0.000025893 0.000002817 +711 0.88615 -0.000025896 0.000003009 +712 0.88711 -0.000025899 0.000003200 +713 0.88808 -0.000025902 0.000003390 +714 0.88904 -0.000025905 0.000003581 +715 0.89001 -0.000025909 0.000003771 +716 0.89097 -0.000025913 0.000003961 +717 0.89194 -0.000025917 0.000004150 +718 0.89290 -0.000025921 0.000004339 +719 0.89387 -0.000025925 0.000004528 +720 0.89483 -0.000025929 0.000004716 +721 0.89580 -0.000025934 0.000004905 +722 0.89676 -0.000025939 0.000005093 +723 0.89773 -0.000025944 0.000005280 +724 0.89869 -0.000025949 0.000005467 +725 0.89966 -0.000025955 0.000005654 +726 0.90063 -0.000025960 0.000005841 +727 0.90159 -0.000025966 0.000006028 +728 0.90256 -0.000025972 0.000006214 +729 0.90352 -0.000025978 0.000006400 +730 0.90449 -0.000025984 0.000006585 +731 0.90545 -0.000025990 0.000006771 +732 0.90642 -0.000025997 0.000006956 +733 0.90738 -0.000026004 0.000007141 +734 0.90835 -0.000026011 0.000007325 +735 0.90931 -0.000026018 0.000007510 +736 0.91028 -0.000026025 0.000007694 +737 0.91124 -0.000026033 0.000007878 +738 0.91221 -0.000026041 0.000008062 +739 0.91317 -0.000026048 0.000008245 +740 0.91414 -0.000026057 0.000008428 +741 0.91510 -0.000026065 0.000008612 +742 0.91607 -0.000026073 0.000008794 +743 0.91703 -0.000026082 0.000008977 +744 0.91800 -0.000026090 0.000009160 +745 0.91896 -0.000026099 0.000009342 +746 0.91993 -0.000026108 0.000009524 +747 0.92089 -0.000026118 0.000009706 +748 0.92186 -0.000026127 0.000009887 +749 0.92282 -0.000026137 0.000010069 +750 0.92379 -0.000026147 0.000010250 +751 0.92475 -0.000026157 0.000010432 +752 0.92572 -0.000026167 0.000010613 +753 0.92668 -0.000026177 0.000010794 +754 0.92765 -0.000026188 0.000010974 +755 0.92861 -0.000026198 0.000011155 +756 0.92958 -0.000026209 0.000011335 +757 0.93054 -0.000026220 0.000011516 +758 0.93151 -0.000026231 0.000011696 +759 0.93247 -0.000026243 0.000011876 +760 0.93344 -0.000026254 0.000012056 +761 0.93440 -0.000026266 0.000012235 +762 0.93537 -0.000026278 0.000012415 +763 0.93633 -0.000026290 0.000012594 +764 0.93730 -0.000026302 0.000012774 +765 0.93826 -0.000026315 0.000012953 +766 0.93923 -0.000026327 0.000013132 +767 0.94019 -0.000026340 0.000013311 +768 0.94116 -0.000026353 0.000013490 +769 0.94212 -0.000026366 0.000013669 +770 0.94309 -0.000026379 0.000013848 +771 0.94405 -0.000026393 0.000014027 +772 0.94502 -0.000026406 0.000014205 +773 0.94598 -0.000026420 0.000014384 +774 0.94695 -0.000026434 0.000014562 +775 0.94791 -0.000026448 0.000014741 +776 0.94888 -0.000026463 0.000014919 +777 0.94984 -0.000026477 0.000015097 +778 0.95081 -0.000026492 0.000015276 +779 0.95177 -0.000026507 0.000015454 +780 0.95274 -0.000026522 0.000015632 +781 0.95370 -0.000026537 0.000015810 +782 0.95467 -0.000026552 0.000015988 +783 0.95563 -0.000026568 0.000016166 +784 0.95660 -0.000026583 0.000016344 +785 0.95756 -0.000026599 0.000016522 +786 0.95853 -0.000026615 0.000016699 +787 0.95949 -0.000026631 0.000016877 +788 0.96046 -0.000026648 0.000017055 +789 0.96142 -0.000026664 0.000017233 +790 0.96239 -0.000026681 0.000017411 +791 0.96335 -0.000026698 0.000017588 +792 0.96432 -0.000026715 0.000017766 +793 0.96528 -0.000026732 0.000017944 +794 0.96625 -0.000026750 0.000018122 +795 0.96721 -0.000026767 0.000018299 +796 0.96818 -0.000026785 0.000018477 +797 0.96914 -0.000026803 0.000018655 +798 0.97011 -0.000026821 0.000018833 +799 0.97107 -0.000026839 0.000019011 +800 0.97204 -0.000026858 0.000019188 +801 0.97300 -0.000026876 0.000019366 +802 0.97397 -0.000026895 0.000019544 +803 0.97493 -0.000026914 0.000019722 +804 0.97590 -0.000026933 0.000019900 +805 0.97686 -0.000026952 0.000020078 +806 0.97783 -0.000026972 0.000020256 +807 0.97879 -0.000026991 0.000020434 +808 0.97976 -0.000027011 0.000020612 +809 0.98072 -0.000027031 0.000020790 +810 0.98169 -0.000027051 0.000020968 +811 0.98265 -0.000027072 0.000021146 +812 0.98362 -0.000027092 0.000021325 +813 0.98458 -0.000027113 0.000021503 +814 0.98555 -0.000027134 0.000021682 +815 0.98651 -0.000027155 0.000021860 +816 0.98748 -0.000027176 0.000022039 +817 0.98844 -0.000027197 0.000022217 +818 0.98941 -0.000027219 0.000022396 +819 0.99037 -0.000027240 0.000022575 +820 0.99134 -0.000027262 0.000022754 +821 0.99230 -0.000027284 0.000022933 +822 0.99327 -0.000027307 0.000023112 +823 0.99423 -0.000027329 0.000023291 +824 0.99520 -0.000027352 0.000023470 +825 0.99616 -0.000027374 0.000023650 +826 0.99713 -0.000027397 0.000023829 +827 0.99809 -0.000027420 0.000024009 +828 0.99906 -0.000027444 0.000024189 +829 1.00003 -0.000027467 0.000024368 +830 1.00099 -0.000027491 0.000024548 +831 1.00196 -0.000027514 0.000024728 +832 1.00292 -0.000027538 0.000024909 +833 1.00389 -0.000027562 0.000025089 +834 1.00485 -0.000027587 0.000025269 +835 1.00582 -0.000027611 0.000025450 +836 1.00678 -0.000027636 0.000025631 +837 1.00775 -0.000027661 0.000025812 +838 1.00871 -0.000027686 0.000025993 +839 1.00968 -0.000027711 0.000026174 +840 1.01064 -0.000027736 0.000026355 +841 1.01161 -0.000027762 0.000026537 +842 1.01257 -0.000027787 0.000026718 +843 1.01354 -0.000027813 0.000026900 +844 1.01450 -0.000027839 0.000027082 +845 1.01547 -0.000027866 0.000027264 +846 1.01643 -0.000027892 0.000027446 +847 1.01740 -0.000027918 0.000027629 +848 1.01836 -0.000027945 0.000027812 +849 1.01933 -0.000027972 0.000027994 +850 1.02029 -0.000027999 0.000028177 +851 1.02126 -0.000028027 0.000028361 +852 1.02222 -0.000028054 0.000028544 +853 1.02319 -0.000028082 0.000028728 +854 1.02415 -0.000028109 0.000028911 +855 1.02512 -0.000028137 0.000029095 +856 1.02608 -0.000028166 0.000029279 +857 1.02705 -0.000028194 0.000029464 +858 1.02801 -0.000028222 0.000029648 +859 1.02898 -0.000028251 0.000029833 +860 1.02994 -0.000028280 0.000030018 +861 1.03091 -0.000028309 0.000030203 +862 1.03187 -0.000028338 0.000030389 +863 1.03284 -0.000028368 0.000030575 +864 1.03380 -0.000028397 0.000030760 +865 1.03477 -0.000028427 0.000030947 +866 1.03573 -0.000028457 0.000031133 +867 1.03670 -0.000028487 0.000031319 +868 1.03766 -0.000028518 0.000031506 +869 1.03863 -0.000028548 0.000031693 +870 1.03959 -0.000028579 0.000031881 +871 1.04056 -0.000028610 0.000032068 +872 1.04152 -0.000028641 0.000032256 +873 1.04249 -0.000028672 0.000032444 +874 1.04345 -0.000028703 0.000032632 +875 1.04442 -0.000028735 0.000032821 +876 1.04538 -0.000028767 0.000033010 +877 1.04635 -0.000028799 0.000033199 +878 1.04731 -0.000028831 0.000033388 +879 1.04828 -0.000028863 0.000033578 +880 1.04924 -0.000028895 0.000033768 +881 1.05021 -0.000028928 0.000033958 +882 1.05117 -0.000028961 0.000034149 +883 1.05214 -0.000028994 0.000034339 +884 1.05310 -0.000029027 0.000034530 +885 1.05407 -0.000029061 0.000034722 +886 1.05503 -0.000029094 0.000034913 +887 1.05600 -0.000029128 0.000035105 +888 1.05696 -0.000029162 0.000035297 +889 1.05793 -0.000029196 0.000035490 +890 1.05889 -0.000029231 0.000035683 +891 1.05986 -0.000029265 0.000035876 +892 1.06082 -0.000029300 0.000036069 +893 1.06179 -0.000029335 0.000036263 +894 1.06275 -0.000029370 0.000036457 +895 1.06372 -0.000029405 0.000036652 +896 1.06468 -0.000029441 0.000036846 +897 1.06565 -0.000029476 0.000037041 +898 1.06661 -0.000029512 0.000037237 +899 1.06758 -0.000029548 0.000037432 +900 1.06854 -0.000029584 0.000037629 +901 1.06951 -0.000029621 0.000037825 +902 1.07047 -0.000029657 0.000038022 +903 1.07144 -0.000029694 0.000038219 +904 1.07240 -0.000029731 0.000038416 +905 1.07337 -0.000029768 0.000038614 +906 1.07433 -0.000029806 0.000038812 +907 1.07530 -0.000029843 0.000039010 +908 1.07626 -0.000029881 0.000039209 +909 1.07723 -0.000029919 0.000039409 +910 1.07819 -0.000029957 0.000039608 +911 1.07916 -0.000029995 0.000039808 +912 1.08012 -0.000030034 0.000040008 +913 1.08109 -0.000030072 0.000040209 +914 1.08205 -0.000030111 0.000040410 +915 1.08302 -0.000030150 0.000040611 +916 1.08398 -0.000030190 0.000040813 +917 1.08495 -0.000030229 0.000041015 +918 1.08591 -0.000030269 0.000041218 +919 1.08688 -0.000030309 0.000041421 +920 1.08784 -0.000030349 0.000041624 +921 1.08881 -0.000030389 0.000041828 +922 1.08977 -0.000030430 0.000042032 +923 1.09074 -0.000030470 0.000042237 +924 1.09170 -0.000030511 0.000042442 +925 1.09267 -0.000030552 0.000042647 +926 1.09363 -0.000030593 0.000042853 +927 1.09460 -0.000030635 0.000043059 +928 1.09556 -0.000030677 0.000043266 +929 1.09653 -0.000030718 0.000043473 +930 1.09749 -0.000030760 0.000043681 +931 1.09846 -0.000030803 0.000043889 +932 1.09942 -0.000030845 0.000044097 +933 1.10039 -0.000030888 0.000044306 +934 1.10136 -0.000030931 0.000044515 +935 1.10232 -0.000030974 0.000044725 +936 1.10329 -0.000031017 0.000044935 +937 1.10425 -0.000031060 0.000045146 +938 1.10522 -0.000031104 0.000045357 +939 1.10618 -0.000031148 0.000045568 +940 1.10715 -0.000031192 0.000045780 +941 1.10811 -0.000031236 0.000045992 +942 1.10908 -0.000031281 0.000046205 +943 1.11004 -0.000031326 0.000046419 +944 1.11101 -0.000031370 0.000046632 +945 1.11197 -0.000031416 0.000046847 +946 1.11294 -0.000031461 0.000047062 +947 1.11390 -0.000031506 0.000047277 +948 1.11487 -0.000031552 0.000047493 +949 1.11583 -0.000031598 0.000047709 +950 1.11680 -0.000031644 0.000047925 +951 1.11776 -0.000031691 0.000048143 +952 1.11873 -0.000031737 0.000048360 +953 1.11969 -0.000031784 0.000048578 +954 1.12066 -0.000031831 0.000048797 +955 1.12162 -0.000031878 0.000049016 +956 1.12259 -0.000031925 0.000049236 +957 1.12355 -0.000031973 0.000049456 +958 1.12452 -0.000032021 0.000049677 +959 1.12548 -0.000032069 0.000049898 +960 1.12645 -0.000032117 0.000050120 +961 1.12741 -0.000032166 0.000050342 +962 1.12838 -0.000032214 0.000050565 +963 1.12934 -0.000032263 0.000050789 +964 1.13031 -0.000032312 0.000051012 +965 1.13127 -0.000032362 0.000051237 +966 1.13224 -0.000032411 0.000051462 +967 1.13320 -0.000032461 0.000051687 +968 1.13417 -0.000032511 0.000051913 +969 1.13513 -0.000032561 0.000052140 +970 1.13610 -0.000032612 0.000052367 +971 1.13706 -0.000032662 0.000052595 +972 1.13803 -0.000032713 0.000052823 +973 1.13899 -0.000032764 0.000053052 +974 1.13996 -0.000032816 0.000053281 +975 1.14092 -0.000032867 0.000053511 +976 1.14189 -0.000032919 0.000053742 +977 1.14285 -0.000032971 0.000053973 +978 1.14382 -0.000033023 0.000054205 +979 1.14478 -0.000033076 0.000054437 +980 1.14575 -0.000033128 0.000054670 +981 1.14671 -0.000033181 0.000054903 +982 1.14768 -0.000033234 0.000055138 +983 1.14864 -0.000033287 0.000055372 +984 1.14961 -0.000033341 0.000055608 +985 1.15057 -0.000033395 0.000055843 +986 1.15154 -0.000033449 0.000056080 +987 1.15250 -0.000033503 0.000056317 +988 1.15347 -0.000033557 0.000056555 +989 1.15443 -0.000033612 0.000056793 +990 1.15540 -0.000033667 0.000057032 +991 1.15636 -0.000033722 0.000057271 +992 1.15733 -0.000033778 0.000057512 +993 1.15829 -0.000033833 0.000057753 +994 1.15926 -0.000033889 0.000057994 +995 1.16022 -0.000033945 0.000058236 +996 1.16119 -0.000034002 0.000058479 +997 1.16215 -0.000034058 0.000058722 +998 1.16312 -0.000034115 0.000058966 +999 1.16408 -0.000034172 0.000059211 +1000 1.16505 -0.000034229 0.000059456 +1001 1.16601 -0.000034287 0.000059702 +1002 1.16698 -0.000034344 0.000059949 +1003 1.16794 -0.000034402 0.000060196 +1004 1.16891 -0.000034461 0.000060444 +1005 1.16987 -0.000034519 0.000060693 +1006 1.17084 -0.000034578 0.000060942 +1007 1.17180 -0.000034637 0.000061192 +1008 1.17277 -0.000034696 0.000061443 +1009 1.17373 -0.000034755 0.000061695 +1010 1.17470 -0.000034815 0.000061947 +1011 1.17566 -0.000034875 0.000062199 +1012 1.17663 -0.000034935 0.000062453 +1013 1.17759 -0.000034995 0.000062707 +1014 1.17856 -0.000035056 0.000062962 +1015 1.17952 -0.000035117 0.000063218 +1016 1.18049 -0.000035178 0.000063474 +1017 1.18145 -0.000035239 0.000063731 +1018 1.18242 -0.000035301 0.000063989 +1019 1.18338 -0.000035363 0.000064247 +1020 1.18435 -0.000035425 0.000064506 +1021 1.18531 -0.000035487 0.000064766 +1022 1.18628 -0.000035550 0.000065027 +1023 1.18724 -0.000035613 0.000065288 +1024 1.18821 -0.000035676 0.000065551 +1025 1.18917 -0.000035739 0.000065814 +1026 1.19014 -0.000035803 0.000066077 +1027 1.19110 -0.000035867 0.000066342 +1028 1.19207 -0.000035931 0.000066607 +1029 1.19303 -0.000035995 0.000066873 +1030 1.19400 -0.000036060 0.000067139 +1031 1.19496 -0.000036125 0.000067407 +1032 1.19593 -0.000036190 0.000067675 +1033 1.19689 -0.000036256 0.000067944 +1034 1.19786 -0.000036321 0.000068214 +1035 1.19882 -0.000036387 0.000068485 +1036 1.19979 -0.000036454 0.000068756 +1037 1.20076 -0.000036520 0.000069028 +1038 1.20172 -0.000036587 0.000069301 +1039 1.20269 -0.000036654 0.000069575 +1040 1.20365 -0.000036721 0.000069849 +1041 1.20462 -0.000036789 0.000070125 +1042 1.20558 -0.000036856 0.000070401 +1043 1.20655 -0.000036925 0.000070678 +1044 1.20751 -0.000036993 0.000070956 +1045 1.20848 -0.000037061 0.000071234 +1046 1.20944 -0.000037130 0.000071514 +1047 1.21041 -0.000037200 0.000071794 +1048 1.21137 -0.000037269 0.000072075 +1049 1.21234 -0.000037339 0.000072357 +1050 1.21330 -0.000037409 0.000072640 +1051 1.21427 -0.000037479 0.000072924 +1052 1.21523 -0.000037549 0.000073208 +1053 1.21620 -0.000037620 0.000073494 +1054 1.21716 -0.000037691 0.000073780 +1055 1.21813 -0.000037763 0.000074067 +1056 1.21909 -0.000037834 0.000074355 +1057 1.22006 -0.000037906 0.000074644 +1058 1.22102 -0.000037978 0.000074934 +1059 1.22199 -0.000038051 0.000075224 +1060 1.22295 -0.000038123 0.000075516 +1061 1.22392 -0.000038196 0.000075808 +1062 1.22488 -0.000038270 0.000076101 +1063 1.22585 -0.000038343 0.000076396 +1064 1.22681 -0.000038417 0.000076691 +1065 1.22778 -0.000038491 0.000076987 +1066 1.22874 -0.000038566 0.000077283 +1067 1.22971 -0.000038640 0.000077581 +1068 1.23067 -0.000038716 0.000077880 +1069 1.23164 -0.000038791 0.000078180 +1070 1.23260 -0.000038866 0.000078480 +1071 1.23357 -0.000038942 0.000078782 +1072 1.23453 -0.000039018 0.000079084 +1073 1.23550 -0.000039095 0.000079388 +1074 1.23646 -0.000039172 0.000079692 +1075 1.23743 -0.000039249 0.000079997 +1076 1.23839 -0.000039326 0.000080303 +1077 1.23936 -0.000039404 0.000080611 +1078 1.24032 -0.000039482 0.000080919 +1079 1.24129 -0.000039560 0.000081228 +1080 1.24225 -0.000039638 0.000081538 +1081 1.24322 -0.000039717 0.000081849 +1082 1.24418 -0.000039796 0.000082161 +1083 1.24515 -0.000039876 0.000082474 +1084 1.24611 -0.000039956 0.000082788 +1085 1.24708 -0.000040036 0.000083103 +1086 1.24804 -0.000040116 0.000083419 +1087 1.24901 -0.000040197 0.000083736 +1088 1.24997 -0.000040278 0.000084054 +1089 1.25094 -0.000040359 0.000084373 +1090 1.25190 -0.000040440 0.000084693 +1091 1.25287 -0.000040522 0.000085014 +1092 1.25383 -0.000040605 0.000085336 +1093 1.25480 -0.000040687 0.000085659 +1094 1.25576 -0.000040770 0.000085984 +1095 1.25673 -0.000040853 0.000086309 +1096 1.25769 -0.000040936 0.000086635 +1097 1.25866 -0.000041020 0.000086962 +1098 1.25962 -0.000041104 0.000087291 +1099 1.26059 -0.000041189 0.000087620 +1100 1.26155 -0.000041273 0.000087951 +1101 1.26252 -0.000041358 0.000088282 +1102 1.26348 -0.000041444 0.000088615 +1103 1.26445 -0.000041530 0.000088948 +1104 1.26541 -0.000041616 0.000089283 +1105 1.26638 -0.000041702 0.000089619 +1106 1.26734 -0.000041788 0.000089956 +1107 1.26831 -0.000041875 0.000090294 +1108 1.26927 -0.000041963 0.000090633 +1109 1.27024 -0.000042050 0.000090974 +1110 1.27120 -0.000042138 0.000091315 +1111 1.27217 -0.000042227 0.000091657 +1112 1.27313 -0.000042315 0.000092001 +1113 1.27410 -0.000042404 0.000092346 +1114 1.27506 -0.000042493 0.000092692 +1115 1.27603 -0.000042583 0.000093039 +1116 1.27699 -0.000042673 0.000093387 +1117 1.27796 -0.000042763 0.000093736 +1118 1.27892 -0.000042854 0.000094086 +1119 1.27989 -0.000042945 0.000094438 +1120 1.28085 -0.000043036 0.000094791 +1121 1.28182 -0.000043128 0.000095145 +1122 1.28278 -0.000043220 0.000095500 +1123 1.28375 -0.000043312 0.000095856 +1124 1.28471 -0.000043405 0.000096214 +1125 1.28568 -0.000043498 0.000096572 +1126 1.28664 -0.000043591 0.000096932 +1127 1.28761 -0.000043685 0.000097293 +1128 1.28857 -0.000043779 0.000097655 +1129 1.28954 -0.000043874 0.000098019 +1130 1.29050 -0.000043968 0.000098384 +1131 1.29147 -0.000044063 0.000098750 +1132 1.29243 -0.000044159 0.000099117 +1133 1.29340 -0.000044255 0.000099485 +1134 1.29436 -0.000044351 0.000099855 +1135 1.29533 -0.000044447 0.000100225 +1136 1.29629 -0.000044544 0.000100598 +1137 1.29726 -0.000044642 0.000100971 +1138 1.29822 -0.000044739 0.000101345 +1139 1.29919 -0.000044837 0.000101721 +1140 1.30016 -0.000044936 0.000102098 +1141 1.30112 -0.000045034 0.000102477 +1142 1.30209 -0.000045133 0.000102857 +1143 1.30305 -0.000045233 0.000103237 +1144 1.30402 -0.000045333 0.000103620 +1145 1.30498 -0.000045433 0.000104003 +1146 1.30595 -0.000045533 0.000104388 +1147 1.30691 -0.000045634 0.000104774 +1148 1.30788 -0.000045736 0.000105162 +1149 1.30884 -0.000045837 0.000105551 +1150 1.30981 -0.000045939 0.000105941 +1151 1.31077 -0.000046042 0.000106332 +1152 1.31174 -0.000046145 0.000106725 +1153 1.31270 -0.000046248 0.000107119 +1154 1.31367 -0.000046351 0.000107515 +1155 1.31463 -0.000046455 0.000107911 +1156 1.31560 -0.000046560 0.000108310 +1157 1.31656 -0.000046664 0.000108709 +1158 1.31753 -0.000046769 0.000109110 +1159 1.31849 -0.000046875 0.000109512 +1160 1.31946 -0.000046981 0.000109916 +1161 1.32042 -0.000047087 0.000110321 +1162 1.32139 -0.000047194 0.000110728 +1163 1.32235 -0.000047301 0.000111135 +1164 1.32332 -0.000047408 0.000111545 +1165 1.32428 -0.000047516 0.000111955 +1166 1.32525 -0.000047624 0.000112368 +1167 1.32621 -0.000047733 0.000112781 +1168 1.32718 -0.000047842 0.000113196 +1169 1.32814 -0.000047951 0.000113613 +1170 1.32911 -0.000048061 0.000114030 +1171 1.33007 -0.000048171 0.000114450 +1172 1.33104 -0.000048282 0.000114870 +1173 1.33200 -0.000048393 0.000115293 +1174 1.33297 -0.000048505 0.000115716 +1175 1.33393 -0.000048617 0.000116142 +1176 1.33490 -0.000048729 0.000116568 +1177 1.33586 -0.000048842 0.000116996 +1178 1.33683 -0.000048955 0.000117426 +1179 1.33779 -0.000049068 0.000117857 +1180 1.33876 -0.000049182 0.000118290 +1181 1.33972 -0.000049296 0.000118724 +1182 1.34069 -0.000049411 0.000119160 +1183 1.34165 -0.000049526 0.000119597 +1184 1.34262 -0.000049642 0.000120036 +1185 1.34358 -0.000049758 0.000120476 +1186 1.34455 -0.000049875 0.000120918 +1187 1.34551 -0.000049992 0.000121361 +1188 1.34648 -0.000050109 0.000121806 +1189 1.34744 -0.000050227 0.000122253 +1190 1.34841 -0.000050345 0.000122701 +1191 1.34937 -0.000050463 0.000123150 +1192 1.35034 -0.000050583 0.000123602 +1193 1.35130 -0.000050702 0.000124054 +1194 1.35227 -0.000050822 0.000124509 +1195 1.35323 -0.000050942 0.000124965 +1196 1.35420 -0.000051063 0.000125423 +1197 1.35516 -0.000051184 0.000125882 +1198 1.35613 -0.000051306 0.000126343 +1199 1.35709 -0.000051428 0.000126805 +1200 1.35806 -0.000051551 0.000127270 +1201 1.35902 -0.000051674 0.000127735 +1202 1.35999 -0.000051797 0.000128203 +1203 1.36095 -0.000051921 0.000128672 +1204 1.36192 -0.000052046 0.000129143 +1205 1.36288 -0.000052171 0.000129615 +1206 1.36385 -0.000052296 0.000130089 +1207 1.36481 -0.000052422 0.000130565 +1208 1.36578 -0.000052548 0.000131043 +1209 1.36674 -0.000052675 0.000131522 +1210 1.36771 -0.000052802 0.000132003 +1211 1.36867 -0.000052929 0.000132486 +1212 1.36964 -0.000053057 0.000132970 +1213 1.37060 -0.000053186 0.000133457 +1214 1.37157 -0.000053315 0.000133944 +1215 1.37253 -0.000053445 0.000134434 +1216 1.37350 -0.000053575 0.000134926 +1217 1.37446 -0.000053705 0.000135419 +1218 1.37543 -0.000053836 0.000135914 +1219 1.37639 -0.000053967 0.000136410 +1220 1.37736 -0.000054099 0.000136909 +1221 1.37832 -0.000054232 0.000137409 +1222 1.37929 -0.000054364 0.000137911 +1223 1.38025 -0.000054498 0.000138415 +1224 1.38122 -0.000054632 0.000138921 +1225 1.38218 -0.000054766 0.000139429 +1226 1.38315 -0.000054901 0.000139938 +1227 1.38411 -0.000055036 0.000140449 +1228 1.38508 -0.000055172 0.000140962 +1229 1.38604 -0.000055308 0.000141477 +1230 1.38701 -0.000055445 0.000141994 +1231 1.38797 -0.000055582 0.000142513 +1232 1.38894 -0.000055720 0.000143033 +1233 1.38990 -0.000055858 0.000143556 +1234 1.39087 -0.000055997 0.000144080 +1235 1.39183 -0.000056136 0.000144606 +1236 1.39280 -0.000056276 0.000145135 +1237 1.39376 -0.000056416 0.000145665 +1238 1.39473 -0.000056557 0.000146197 +1239 1.39569 -0.000056699 0.000146731 +1240 1.39666 -0.000056840 0.000147266 +1241 1.39762 -0.000056983 0.000147804 +1242 1.39859 -0.000057126 0.000148344 +1243 1.39955 -0.000057269 0.000148886 +1244 1.40052 -0.000057413 0.000149430 +1245 1.40149 -0.000057558 0.000149975 +1246 1.40245 -0.000057703 0.000150523 +1247 1.40342 -0.000057848 0.000151073 +1248 1.40438 -0.000057994 0.000151624 +1249 1.40535 -0.000058141 0.000152178 +1250 1.40631 -0.000058288 0.000152734 +1251 1.40728 -0.000058435 0.000153292 +1252 1.40824 -0.000058584 0.000153852 +1253 1.40921 -0.000058732 0.000154414 +1254 1.41017 -0.000058882 0.000154978 +1255 1.41114 -0.000059032 0.000155544 +1256 1.41210 -0.000059182 0.000156112 +1257 1.41307 -0.000059333 0.000156682 +1258 1.41403 -0.000059484 0.000157254 +1259 1.41500 -0.000059636 0.000157829 +1260 1.41596 -0.000059789 0.000158405 +1261 1.41693 -0.000059942 0.000158984 +1262 1.41789 -0.000060096 0.000159565 +1263 1.41886 -0.000060250 0.000160147 +1264 1.41982 -0.000060405 0.000160733 +1265 1.42079 -0.000060560 0.000161320 +1266 1.42175 -0.000060716 0.000161909 +1267 1.42272 -0.000060873 0.000162501 +1268 1.42368 -0.000061030 0.000163094 +1269 1.42465 -0.000061188 0.000163690 +1270 1.42561 -0.000061346 0.000164289 +1271 1.42658 -0.000061505 0.000164889 +1272 1.42754 -0.000061664 0.000165492 +1273 1.42851 -0.000061824 0.000166096 +1274 1.42947 -0.000061985 0.000166704 +1275 1.43044 -0.000062146 0.000167313 +1276 1.43140 -0.000062308 0.000167924 +1277 1.43237 -0.000062470 0.000168538 +1278 1.43333 -0.000062633 0.000169154 +1279 1.43430 -0.000062796 0.000169773 +1280 1.43526 -0.000062961 0.000170394 +1281 1.43623 -0.000063125 0.000171017 +1282 1.43719 -0.000063291 0.000171642 +1283 1.43816 -0.000063457 0.000172270 +1284 1.43912 -0.000063623 0.000172900 +1285 1.44009 -0.000063790 0.000173532 +1286 1.44105 -0.000063958 0.000174167 +1287 1.44202 -0.000064127 0.000174804 +1288 1.44298 -0.000064296 0.000175443 +1289 1.44395 -0.000064465 0.000176085 +1290 1.44491 -0.000064635 0.000176730 +1291 1.44588 -0.000064806 0.000177376 +1292 1.44684 -0.000064978 0.000178025 +1293 1.44781 -0.000065150 0.000178677 +1294 1.44877 -0.000065323 0.000179331 +1295 1.44974 -0.000065496 0.000179987 +1296 1.45070 -0.000065670 0.000180646 +1297 1.45167 -0.000065845 0.000181307 +1298 1.45263 -0.000066020 0.000181971 +1299 1.45360 -0.000066196 0.000182638 +1300 1.45456 -0.000066372 0.000183306 +1301 1.45553 -0.000066550 0.000183978 +1302 1.45649 -0.000066728 0.000184651 +1303 1.45746 -0.000066906 0.000185328 +1304 1.45842 -0.000067085 0.000186007 +1305 1.45939 -0.000067265 0.000186688 +1306 1.46035 -0.000067446 0.000187372 +1307 1.46132 -0.000067627 0.000188059 +1308 1.46228 -0.000067809 0.000188748 +1309 1.46325 -0.000067991 0.000189440 +1310 1.46421 -0.000068174 0.000190134 +1311 1.46518 -0.000068358 0.000190831 +1312 1.46614 -0.000068543 0.000191531 +1313 1.46711 -0.000068728 0.000192233 +1314 1.46807 -0.000068914 0.000192938 +1315 1.46904 -0.000069100 0.000193646 +1316 1.47000 -0.000069287 0.000194356 +1317 1.47097 -0.000069475 0.000195069 +1318 1.47193 -0.000069664 0.000195785 +1319 1.47290 -0.000069853 0.000196503 +1320 1.47386 -0.000070043 0.000197224 +1321 1.47483 -0.000070234 0.000197948 +1322 1.47579 -0.000070425 0.000198675 +1323 1.47676 -0.000070617 0.000199404 +1324 1.47772 -0.000070810 0.000200136 +1325 1.47869 -0.000071003 0.000200871 +1326 1.47965 -0.000071198 0.000201609 +1327 1.48062 -0.000071393 0.000202349 +1328 1.48158 -0.000071588 0.000203092 +1329 1.48255 -0.000071785 0.000203838 +1330 1.48351 -0.000071982 0.000204587 +1331 1.48448 -0.000072179 0.000205339 +1332 1.48544 -0.000072378 0.000206094 +1333 1.48641 -0.000072577 0.000206851 +1334 1.48737 -0.000072777 0.000207612 +1335 1.48834 -0.000072978 0.000208375 +1336 1.48930 -0.000073179 0.000209141 +1337 1.49027 -0.000073382 0.000209910 +1338 1.49123 -0.000073585 0.000210682 +1339 1.49220 -0.000073788 0.000211457 +1340 1.49316 -0.000073993 0.000212235 +1341 1.49413 -0.000074198 0.000213016 +1342 1.49509 -0.000074404 0.000213800 +1343 1.49606 -0.000074611 0.000214587 +1344 1.49702 -0.000074818 0.000215377 +1345 1.49799 -0.000075026 0.000216170 +1346 1.49895 -0.000075235 0.000216966 +1347 1.49992 -0.000075445 0.000217765 +1348 1.50089 -0.000075656 0.000218567 +1349 1.50185 -0.000075867 0.000219372 +1350 1.50282 -0.000076079 0.000220180 +1351 1.50378 -0.000076292 0.000220992 +1352 1.50475 -0.000076506 0.000221806 +1353 1.50571 -0.000076720 0.000222624 +1354 1.50668 -0.000076935 0.000223444 +1355 1.50764 -0.000077151 0.000224268 +1356 1.50861 -0.000077368 0.000225095 +1357 1.50957 -0.000077586 0.000225925 +1358 1.51054 -0.000077804 0.000226759 +1359 1.51150 -0.000078023 0.000227595 +1360 1.51247 -0.000078243 0.000228435 +1361 1.51343 -0.000078464 0.000229278 +1362 1.51440 -0.000078686 0.000230125 +1363 1.51536 -0.000078908 0.000230974 +1364 1.51633 -0.000079132 0.000231827 +1365 1.51729 -0.000079356 0.000232683 +1366 1.51826 -0.000079581 0.000233542 +1367 1.51922 -0.000079807 0.000234405 +1368 1.52019 -0.000080033 0.000235271 +1369 1.52115 -0.000080261 0.000236141 +1370 1.52212 -0.000080489 0.000237013 +1371 1.52308 -0.000080718 0.000237889 +1372 1.52405 -0.000080948 0.000238769 +1373 1.52501 -0.000081179 0.000239652 +1374 1.52598 -0.000081411 0.000240538 +1375 1.52694 -0.000081643 0.000241428 +1376 1.52791 -0.000081877 0.000242321 +1377 1.52887 -0.000082111 0.000243218 +1378 1.52984 -0.000082346 0.000244118 +1379 1.53080 -0.000082582 0.000245021 +1380 1.53177 -0.000082819 0.000245928 +1381 1.53273 -0.000083057 0.000246839 +1382 1.53370 -0.000083295 0.000247753 +1383 1.53466 -0.000083535 0.000248671 +1384 1.53563 -0.000083775 0.000249592 +1385 1.53659 -0.000084017 0.000250517 +1386 1.53756 -0.000084259 0.000251445 +1387 1.53852 -0.000084502 0.000252377 +1388 1.53949 -0.000084746 0.000253313 +1389 1.54045 -0.000084991 0.000254252 +1390 1.54142 -0.000085237 0.000255195 +1391 1.54238 -0.000085484 0.000256141 +1392 1.54335 -0.000085731 0.000257092 +1393 1.54431 -0.000085980 0.000258046 +1394 1.54528 -0.000086229 0.000259003 +1395 1.54624 -0.000086480 0.000259965 +1396 1.54721 -0.000086731 0.000260930 +1397 1.54817 -0.000086983 0.000261899 +1398 1.54914 -0.000087237 0.000262871 +1399 1.55010 -0.000087491 0.000263848 +1400 1.55107 -0.000087746 0.000264828 +1401 1.55203 -0.000088002 0.000265812 +1402 1.55300 -0.000088259 0.000266800 +1403 1.55396 -0.000088517 0.000267792 +1404 1.55493 -0.000088776 0.000268788 +1405 1.55589 -0.000089036 0.000269787 +1406 1.55686 -0.000089296 0.000270791 +1407 1.55782 -0.000089558 0.000271798 +1408 1.55879 -0.000089821 0.000272809 +1409 1.55975 -0.000090085 0.000273825 +1410 1.56072 -0.000090349 0.000274844 +1411 1.56168 -0.000090615 0.000275867 +1412 1.56265 -0.000090882 0.000276894 +1413 1.56361 -0.000091150 0.000277926 +1414 1.56458 -0.000091418 0.000278961 +1415 1.56554 -0.000091688 0.000280000 +1416 1.56651 -0.000091959 0.000281044 +1417 1.56747 -0.000092231 0.000282091 +1418 1.56844 -0.000092503 0.000283143 +1419 1.56940 -0.000092777 0.000284198 +1420 1.57037 -0.000093052 0.000285258 +1421 1.57133 -0.000093328 0.000286322 +1422 1.57230 -0.000093604 0.000287391 +1423 1.57326 -0.000093882 0.000288463 +1424 1.57423 -0.000094161 0.000289540 +1425 1.57519 -0.000094441 0.000290621 +1426 1.57616 -0.000094722 0.000291706 +1427 1.57712 -0.000095004 0.000292795 +1428 1.57809 -0.000095287 0.000293889 +1429 1.57905 -0.000095571 0.000294987 +1430 1.58002 -0.000095857 0.000296089 +1431 1.58098 -0.000096143 0.000297196 +1432 1.58195 -0.000096430 0.000298307 +1433 1.58291 -0.000096719 0.000299422 +1434 1.58388 -0.000097008 0.000300542 +1435 1.58484 -0.000097299 0.000301666 +1436 1.58581 -0.000097590 0.000302795 +1437 1.58677 -0.000097883 0.000303928 +1438 1.58774 -0.000098177 0.000305065 +1439 1.58870 -0.000098472 0.000306207 +1440 1.58967 -0.000098768 0.000307354 +1441 1.59063 -0.000099065 0.000308505 +1442 1.59160 -0.000099363 0.000309660 +1443 1.59256 -0.000099663 0.000310821 +1444 1.59353 -0.000099963 0.000311985 +1445 1.59449 -0.000100265 0.000313155 +1446 1.59546 -0.000100568 0.000314329 +1447 1.59642 -0.000100872 0.000315507 +1448 1.59739 -0.000101177 0.000316691 +1449 1.59835 -0.000101483 0.000317879 +1450 1.59932 -0.000101790 0.000319071 +1451 1.60029 -0.000102099 0.000320269 +1452 1.60125 -0.000102408 0.000321471 +1453 1.60222 -0.000102719 0.000322678 +1454 1.60318 -0.000103031 0.000323890 +1455 1.60415 -0.000103344 0.000325106 +1456 1.60511 -0.000103659 0.000326327 +1457 1.60608 -0.000103974 0.000327554 +1458 1.60704 -0.000104291 0.000328785 +1459 1.60801 -0.000104609 0.000330021 +1460 1.60897 -0.000104928 0.000331261 +1461 1.60994 -0.000105248 0.000332507 +1462 1.61090 -0.000105570 0.000333758 +1463 1.61187 -0.000105892 0.000335014 +1464 1.61283 -0.000106216 0.000336274 +1465 1.61380 -0.000106541 0.000337540 +1466 1.61476 -0.000106868 0.000338811 +1467 1.61573 -0.000107195 0.000340086 +1468 1.61669 -0.000107524 0.000341367 +1469 1.61766 -0.000107854 0.000342653 +1470 1.61862 -0.000108186 0.000343944 +1471 1.61959 -0.000108518 0.000345241 +1472 1.62055 -0.000108852 0.000346542 +1473 1.62152 -0.000109187 0.000347848 +1474 1.62248 -0.000109523 0.000349160 +1475 1.62345 -0.000109861 0.000350477 +1476 1.62441 -0.000110200 0.000351800 +1477 1.62538 -0.000110540 0.000353127 +1478 1.62634 -0.000110881 0.000354460 +1479 1.62731 -0.000111224 0.000355798 +1480 1.62827 -0.000111568 0.000357142 +1481 1.62924 -0.000111913 0.000358490 +1482 1.63020 -0.000112260 0.000359845 +1483 1.63117 -0.000112608 0.000361204 +1484 1.63213 -0.000112957 0.000362569 +1485 1.63310 -0.000113308 0.000363940 +1486 1.63406 -0.000113660 0.000365316 +1487 1.63503 -0.000114013 0.000366697 +1488 1.63599 -0.000114367 0.000368085 +1489 1.63696 -0.000114723 0.000369477 +1490 1.63792 -0.000115080 0.000370875 +1491 1.63889 -0.000115439 0.000372279 +1492 1.63985 -0.000115799 0.000373689 +1493 1.64082 -0.000116160 0.000375104 +1494 1.64178 -0.000116523 0.000376524 +1495 1.64275 -0.000116887 0.000377951 +1496 1.64371 -0.000117252 0.000379383 +1497 1.64468 -0.000117619 0.000380821 +1498 1.64564 -0.000117987 0.000382264 +1499 1.64661 -0.000118357 0.000383714 +1500 1.64757 -0.000118728 0.000385169 +1501 1.64854 -0.000119100 0.000386630 +1502 1.64950 -0.000119474 0.000388097 +1503 1.65047 -0.000119850 0.000389570 +1504 1.65143 -0.000120226 0.000391049 +1505 1.65240 -0.000120604 0.000392534 +1506 1.65336 -0.000120984 0.000394024 +1507 1.65433 -0.000121365 0.000395521 +1508 1.65529 -0.000121747 0.000397024 +1509 1.65626 -0.000122131 0.000398532 +1510 1.65722 -0.000122516 0.000400047 +1511 1.65819 -0.000122903 0.000401568 +1512 1.65915 -0.000123292 0.000403095 +1513 1.66012 -0.000123681 0.000404628 +1514 1.66108 -0.000124072 0.000406168 +1515 1.66205 -0.000124465 0.000407713 +1516 1.66301 -0.000124859 0.000409265 +1517 1.66398 -0.000125255 0.000410823 +1518 1.66494 -0.000125652 0.000412387 +1519 1.66591 -0.000126051 0.000413958 +1520 1.66687 -0.000126451 0.000415535 +1521 1.66784 -0.000126853 0.000417118 +1522 1.66880 -0.000127256 0.000418708 +1523 1.66977 -0.000127661 0.000420304 +1524 1.67073 -0.000128068 0.000421907 +1525 1.67170 -0.000128476 0.000423516 +1526 1.67266 -0.000128885 0.000425131 +1527 1.67363 -0.000129296 0.000426753 +1528 1.67459 -0.000129709 0.000428382 +1529 1.67556 -0.000130123 0.000430017 +1530 1.67652 -0.000130539 0.000431659 +1531 1.67749 -0.000130956 0.000433308 +1532 1.67845 -0.000131375 0.000434963 +1533 1.67942 -0.000131796 0.000436624 +1534 1.68038 -0.000132218 0.000438293 +1535 1.68135 -0.000132642 0.000439968 +1536 1.68231 -0.000133067 0.000441650 +1537 1.68328 -0.000133494 0.000443339 +1538 1.68424 -0.000133923 0.000445035 +1539 1.68521 -0.000134353 0.000446738 +1540 1.68617 -0.000134785 0.000448447 +1541 1.68714 -0.000135218 0.000450164 +1542 1.68810 -0.000135654 0.000451887 +1543 1.68907 -0.000136091 0.000453618 +1544 1.69003 -0.000136529 0.000455355 +1545 1.69100 -0.000136970 0.000457099 +1546 1.69196 -0.000137412 0.000458851 +1547 1.69293 -0.000137855 0.000460610 +1548 1.69389 -0.000138301 0.000462375 +1549 1.69486 -0.000138748 0.000464148 +1550 1.69582 -0.000139196 0.000465929 +1551 1.69679 -0.000139647 0.000467716 +1552 1.69775 -0.000140099 0.000469511 +1553 1.69872 -0.000140553 0.000471313 +1554 1.69968 -0.000141009 0.000473122 +1555 1.70065 -0.000141466 0.000474939 +1556 1.70162 -0.000141926 0.000476763 +1557 1.70258 -0.000142386 0.000478594 +1558 1.70355 -0.000142849 0.000480433 +1559 1.70451 -0.000143314 0.000482280 +1560 1.70548 -0.000143780 0.000484134 +1561 1.70644 -0.000144248 0.000485995 +1562 1.70741 -0.000144718 0.000487864 +1563 1.70837 -0.000145190 0.000489741 +1564 1.70934 -0.000145663 0.000491625 +1565 1.71030 -0.000146139 0.000493517 +1566 1.71127 -0.000146616 0.000495417 +1567 1.71223 -0.000147095 0.000497324 +1568 1.71320 -0.000147576 0.000499240 +1569 1.71416 -0.000148059 0.000501163 +1570 1.71513 -0.000148543 0.000503093 +1571 1.71609 -0.000149030 0.000505032 +1572 1.71706 -0.000149518 0.000506979 +1573 1.71802 -0.000150008 0.000508934 +1574 1.71899 -0.000150500 0.000510896 +1575 1.71995 -0.000150994 0.000512867 +1576 1.72092 -0.000151490 0.000514846 +1577 1.72188 -0.000151988 0.000516832 +1578 1.72285 -0.000152488 0.000518827 +1579 1.72381 -0.000152989 0.000520830 +1580 1.72478 -0.000153493 0.000522842 +1581 1.72574 -0.000153998 0.000524861 +1582 1.72671 -0.000154506 0.000526889 +1583 1.72767 -0.000155015 0.000528925 +1584 1.72864 -0.000155527 0.000530969 +1585 1.72960 -0.000156040 0.000533022 +1586 1.73057 -0.000156556 0.000535083 +1587 1.73153 -0.000157073 0.000537153 +1588 1.73250 -0.000157592 0.000539231 +1589 1.73346 -0.000158114 0.000541318 +1590 1.73443 -0.000158637 0.000543413 +1591 1.73539 -0.000159163 0.000545517 +1592 1.73636 -0.000159690 0.000547629 +1593 1.73732 -0.000160219 0.000549750 +1594 1.73829 -0.000160751 0.000551880 +1595 1.73925 -0.000161285 0.000554018 +1596 1.74022 -0.000161820 0.000556165 +1597 1.74118 -0.000162358 0.000558322 +1598 1.74215 -0.000162898 0.000560486 +1599 1.74311 -0.000163440 0.000562660 +1600 1.74408 -0.000163984 0.000564843 +1601 1.74504 -0.000164530 0.000567035 +1602 1.74601 -0.000165078 0.000569235 +1603 1.74697 -0.000165629 0.000571445 +1604 1.74794 -0.000166181 0.000573664 +1605 1.74890 -0.000166736 0.000575892 +1606 1.74987 -0.000167293 0.000578129 +1607 1.75083 -0.000167852 0.000580375 +1608 1.75180 -0.000168413 0.000582631 +1609 1.75276 -0.000168976 0.000584896 +1610 1.75373 -0.000169542 0.000587170 +1611 1.75469 -0.000170110 0.000589453 +1612 1.75566 -0.000170680 0.000591746 +1613 1.75662 -0.000171252 0.000594049 +1614 1.75759 -0.000171826 0.000596361 +1615 1.75855 -0.000172403 0.000598682 +1616 1.75952 -0.000172982 0.000601013 +1617 1.76048 -0.000173563 0.000603353 +1618 1.76145 -0.000174146 0.000605704 +1619 1.76241 -0.000174732 0.000608064 +1620 1.76338 -0.000175320 0.000610433 +1621 1.76434 -0.000175910 0.000612813 +1622 1.76531 -0.000176503 0.000615202 +1623 1.76627 -0.000177098 0.000617601 +1624 1.76724 -0.000177695 0.000620010 +1625 1.76820 -0.000178294 0.000622429 +1626 1.76917 -0.000178896 0.000624858 +1627 1.77013 -0.000179500 0.000627296 +1628 1.77110 -0.000180107 0.000629745 +1629 1.77206 -0.000180716 0.000632205 +1630 1.77303 -0.000181327 0.000634674 +1631 1.77399 -0.000181941 0.000637153 +1632 1.77496 -0.000182557 0.000639643 +1633 1.77592 -0.000183175 0.000642143 +1634 1.77689 -0.000183796 0.000644653 +1635 1.77785 -0.000184420 0.000647174 +1636 1.77882 -0.000185045 0.000649705 +1637 1.77978 -0.000185674 0.000652247 +1638 1.78075 -0.000186304 0.000654799 +1639 1.78171 -0.000186937 0.000657362 +1640 1.78268 -0.000187573 0.000659935 +1641 1.78364 -0.000188211 0.000662519 +1642 1.78461 -0.000188852 0.000665114 +1643 1.78557 -0.000189495 0.000667719 +1644 1.78654 -0.000190141 0.000670336 +1645 1.78750 -0.000190789 0.000672963 +1646 1.78847 -0.000191439 0.000675601 +1647 1.78943 -0.000192093 0.000678249 +1648 1.79040 -0.000192749 0.000680909 +1649 1.79136 -0.000193407 0.000683580 +1650 1.79233 -0.000194068 0.000686262 +1651 1.79329 -0.000194731 0.000688955 +1652 1.79426 -0.000195398 0.000691660 +1653 1.79522 -0.000196066 0.000694375 +1654 1.79619 -0.000196738 0.000697102 +1655 1.79715 -0.000197412 0.000699840 +1656 1.79812 -0.000198089 0.000702589 +1657 1.79908 -0.000198768 0.000705350 +1658 1.80005 -0.000199450 0.000708123 +1659 1.80102 -0.000200135 0.000710907 +1660 1.80198 -0.000200822 0.000713702 +1661 1.80295 -0.000201512 0.000716509 +1662 1.80391 -0.000202205 0.000719328 +1663 1.80488 -0.000202901 0.000722158 +1664 1.80584 -0.000203599 0.000725000 +1665 1.80681 -0.000204300 0.000727854 +1666 1.80777 -0.000205004 0.000730720 +1667 1.80874 -0.000205710 0.000733598 +1668 1.80970 -0.000206420 0.000736488 +1669 1.81067 -0.000207132 0.000739390 +1670 1.81163 -0.000207847 0.000742304 +1671 1.81260 -0.000208565 0.000745230 +1672 1.81356 -0.000209285 0.000748168 +1673 1.81453 -0.000210009 0.000751118 +1674 1.81549 -0.000210735 0.000754081 +1675 1.81646 -0.000211464 0.000757056 +1676 1.81742 -0.000212196 0.000760044 +1677 1.81839 -0.000212931 0.000763044 +1678 1.81935 -0.000213669 0.000766056 +1679 1.82032 -0.000214410 0.000769081 +1680 1.82128 -0.000215153 0.000772119 +1681 1.82225 -0.000215900 0.000775169 +1682 1.82321 -0.000216649 0.000778232 +1683 1.82418 -0.000217402 0.000781308 +1684 1.82514 -0.000218157 0.000784396 +1685 1.82611 -0.000218916 0.000787498 +1686 1.82707 -0.000219677 0.000790612 +1687 1.82804 -0.000220442 0.000793740 +1688 1.82900 -0.000221209 0.000796880 +1689 1.82997 -0.000221980 0.000800034 +1690 1.83093 -0.000222753 0.000803201 +1691 1.83190 -0.000223530 0.000806381 +1692 1.83286 -0.000224310 0.000809574 +1693 1.83383 -0.000225093 0.000812781 +1694 1.83479 -0.000225879 0.000816001 +1695 1.83576 -0.000226668 0.000819234 +1696 1.83672 -0.000227460 0.000822482 +1697 1.83769 -0.000228255 0.000825742 +1698 1.83865 -0.000229054 0.000829016 +1699 1.83962 -0.000229855 0.000832304 +1700 1.84058 -0.000230660 0.000835606 +1701 1.84155 -0.000231468 0.000838922 +1702 1.84251 -0.000232279 0.000842251 +1703 1.84348 -0.000233094 0.000845594 +1704 1.84444 -0.000233911 0.000848952 +1705 1.84541 -0.000234732 0.000852323 +1706 1.84637 -0.000235556 0.000855709 +1707 1.84734 -0.000236384 0.000859108 +1708 1.84830 -0.000237215 0.000862522 +1709 1.84927 -0.000238049 0.000865950 +1710 1.85023 -0.000238886 0.000869393 +1711 1.85120 -0.000239727 0.000872850 +1712 1.85216 -0.000240571 0.000876322 +1713 1.85313 -0.000241418 0.000879808 +1714 1.85409 -0.000242269 0.000883308 +1715 1.85506 -0.000243123 0.000886824 +1716 1.85602 -0.000243980 0.000890354 +1717 1.85699 -0.000244841 0.000893899 +1718 1.85795 -0.000245706 0.000897459 +1719 1.85892 -0.000246574 0.000901033 +1720 1.85988 -0.000247445 0.000904623 +1721 1.86085 -0.000248320 0.000908228 +1722 1.86181 -0.000249198 0.000911848 +1723 1.86278 -0.000250079 0.000915483 +1724 1.86374 -0.000250965 0.000919133 +1725 1.86471 -0.000251854 0.000922799 +1726 1.86567 -0.000252746 0.000926480 +1727 1.86664 -0.000253642 0.000930177 +1728 1.86760 -0.000254541 0.000933889 +1729 1.86857 -0.000255444 0.000937617 +1730 1.86953 -0.000256351 0.000941360 +1731 1.87050 -0.000257261 0.000945119 +1732 1.87146 -0.000258175 0.000948894 +1733 1.87243 -0.000259093 0.000952685 +1734 1.87339 -0.000260014 0.000956492 +1735 1.87436 -0.000260939 0.000960314 +1736 1.87532 -0.000261867 0.000964153 +1737 1.87629 -0.000262800 0.000968008 +1738 1.87725 -0.000263736 0.000971879 +1739 1.87822 -0.000264675 0.000975767 +1740 1.87918 -0.000265619 0.000979671 +1741 1.88015 -0.000266566 0.000983591 +1742 1.88111 -0.000267517 0.000987528 +1743 1.88208 -0.000268472 0.000991481 +1744 1.88304 -0.000269431 0.000995451 +1745 1.88401 -0.000270394 0.000999438 +1746 1.88497 -0.000271360 0.001003442 +1747 1.88594 -0.000272330 0.001007462 +1748 1.88690 -0.000273305 0.001011499 +1749 1.88787 -0.000274283 0.001015554 +1750 1.88883 -0.000275265 0.001019625 +1751 1.88980 -0.000276251 0.001023714 +1752 1.89076 -0.000277241 0.001027820 +1753 1.89173 -0.000278235 0.001031943 +1754 1.89269 -0.000279232 0.001036084 +1755 1.89366 -0.000280234 0.001040242 +1756 1.89462 -0.000281240 0.001044417 +1757 1.89559 -0.000282250 0.001048610 +1758 1.89655 -0.000283264 0.001052821 +1759 1.89752 -0.000284282 0.001057050 +1760 1.89848 -0.000285304 0.001061296 +1761 1.89945 -0.000286331 0.001065561 +1762 1.90042 -0.000287361 0.001069843 +1763 1.90138 -0.000288395 0.001074144 +1764 1.90235 -0.000289434 0.001078462 +1765 1.90331 -0.000290477 0.001082799 +1766 1.90428 -0.000291524 0.001087154 +1767 1.90524 -0.000292575 0.001091528 +1768 1.90621 -0.000293631 0.001095920 +1769 1.90717 -0.000294691 0.001100331 +1770 1.90814 -0.000295755 0.001104760 +1771 1.90910 -0.000296823 0.001109208 +1772 1.91007 -0.000297895 0.001113675 +1773 1.91103 -0.000298972 0.001118161 +1774 1.91200 -0.000300054 0.001122665 +1775 1.91296 -0.000301139 0.001127189 +1776 1.91393 -0.000302229 0.001131732 +1777 1.91489 -0.000303324 0.001136294 +1778 1.91586 -0.000304422 0.001140875 +1779 1.91682 -0.000305526 0.001145476 +1780 1.91779 -0.000306633 0.001150096 +1781 1.91875 -0.000307745 0.001154736 +1782 1.91972 -0.000308862 0.001159395 +1783 1.92068 -0.000309983 0.001164074 +1784 1.92165 -0.000311109 0.001168773 +1785 1.92261 -0.000312239 0.001173492 +1786 1.92358 -0.000313374 0.001178231 +1787 1.92454 -0.000314513 0.001182989 +1788 1.92551 -0.000315657 0.001187768 +1789 1.92647 -0.000316806 0.001192568 +1790 1.92744 -0.000317959 0.001197387 +1791 1.92840 -0.000319117 0.001202227 +1792 1.92937 -0.000320279 0.001207088 +1793 1.93033 -0.000321446 0.001211969 +1794 1.93130 -0.000322618 0.001216870 +1795 1.93226 -0.000323795 0.001221793 +1796 1.93323 -0.000324977 0.001226736 +1797 1.93419 -0.000326163 0.001231701 +1798 1.93516 -0.000327354 0.001236686 +1799 1.93612 -0.000328550 0.001241693 +1800 1.93709 -0.000329751 0.001246720 +1801 1.93805 -0.000330956 0.001251769 +1802 1.93902 -0.000332167 0.001256840 +1803 1.93998 -0.000333382 0.001261932 +1804 1.94095 -0.000334602 0.001267045 +1805 1.94191 -0.000335827 0.001272181 +1806 1.94288 -0.000337058 0.001277338 +1807 1.94384 -0.000338293 0.001282517 +1808 1.94481 -0.000339533 0.001287718 +1809 1.94577 -0.000340778 0.001292941 +1810 1.94674 -0.000342029 0.001298186 +1811 1.94770 -0.000343284 0.001303453 +1812 1.94867 -0.000344544 0.001308743 +1813 1.94963 -0.000345810 0.001314055 +1814 1.95060 -0.000347081 0.001319390 +1815 1.95156 -0.000348356 0.001324748 +1816 1.95253 -0.000349638 0.001330128 +1817 1.95349 -0.000350924 0.001335531 +1818 1.95446 -0.000352215 0.001340957 +1819 1.95542 -0.000353512 0.001346406 +1820 1.95639 -0.000354814 0.001351878 +1821 1.95735 -0.000356121 0.001357374 +1822 1.95832 -0.000357434 0.001362892 +1823 1.95928 -0.000358752 0.001368435 +1824 1.96025 -0.000360075 0.001374001 +1825 1.96121 -0.000361404 0.001379590 +1826 1.96218 -0.000362738 0.001385203 +1827 1.96314 -0.000364077 0.001390840 +1828 1.96411 -0.000365422 0.001396501 +1829 1.96507 -0.000366773 0.001402187 +1830 1.96604 -0.000368129 0.001407896 +1831 1.96700 -0.000369490 0.001413629 +1832 1.96797 -0.000370857 0.001419387 +1833 1.96893 -0.000372230 0.001425170 +1834 1.96990 -0.000373608 0.001430977 +1835 1.97086 -0.000374992 0.001436809 +1836 1.97183 -0.000376381 0.001442665 +1837 1.97279 -0.000377776 0.001448547 +1838 1.97376 -0.000379177 0.001454453 +1839 1.97472 -0.000380583 0.001460385 +1840 1.97569 -0.000381996 0.001466342 +1841 1.97665 -0.000383413 0.001472324 +1842 1.97762 -0.000384837 0.001478332 +1843 1.97858 -0.000386267 0.001484365 +1844 1.97955 -0.000387702 0.001490424 +1845 1.98051 -0.000389143 0.001496509 +1846 1.98148 -0.000390591 0.001502619 +1847 1.98244 -0.000392044 0.001508756 +1848 1.98341 -0.000393503 0.001514919 +1849 1.98437 -0.000394968 0.001521108 +1850 1.98534 -0.000396439 0.001527323 +1851 1.98630 -0.000397916 0.001533565 +1852 1.98727 -0.000399399 0.001539833 +1853 1.98823 -0.000400888 0.001546128 +1854 1.98920 -0.000402383 0.001552450 +1855 1.99016 -0.000403884 0.001558799 +1856 1.99113 -0.000405391 0.001565175 +1857 1.99209 -0.000406905 0.001571578 +1858 1.99306 -0.000408425 0.001578008 +1859 1.99402 -0.000409951 0.001584466 +1860 1.99499 -0.000411483 0.001590951 +1861 1.99595 -0.000413021 0.001597464 +1862 1.99692 -0.000414566 0.001604005 +1863 1.99788 -0.000416117 0.001610573 +1864 1.99885 -0.000417675 0.001617170 +1865 1.99981 -0.000419239 0.001623794 +1866 2.00078 -0.000420809 0.001630447 +1867 2.00175 -0.000422385 0.001637128 +1868 2.00271 -0.000423969 0.001643838 +1869 2.00368 -0.000425558 0.001650576 +1870 2.00464 -0.000427154 0.001657343 +1871 2.00561 -0.000428757 0.001664139 +1872 2.00657 -0.000430366 0.001670964 +1873 2.00754 -0.000431982 0.001677818 +1874 2.00850 -0.000433605 0.001684701 +1875 2.00947 -0.000435234 0.001691613 +1876 2.01043 -0.000436870 0.001698555 +1877 2.01140 -0.000438512 0.001705527 +1878 2.01236 -0.000440162 0.001712528 +1879 2.01333 -0.000441818 0.001719559 +1880 2.01429 -0.000443480 0.001726620 +1881 2.01526 -0.000445150 0.001733712 +1882 2.01622 -0.000446827 0.001740833 +1883 2.01719 -0.000448510 0.001747985 +1884 2.01815 -0.000450200 0.001755167 +1885 2.01912 -0.000451898 0.001762380 +1886 2.02008 -0.000453602 0.001769624 +1887 2.02105 -0.000455313 0.001776899 +1888 2.02201 -0.000457032 0.001784204 +1889 2.02298 -0.000458757 0.001791541 +1890 2.02394 -0.000460489 0.001798909 +1891 2.02491 -0.000462229 0.001806309 +1892 2.02587 -0.000463976 0.001813740 +1893 2.02684 -0.000465730 0.001821203 +1894 2.02780 -0.000467491 0.001828698 +1895 2.02877 -0.000469259 0.001836224 +1896 2.02973 -0.000471035 0.001843783 +1897 2.03070 -0.000472818 0.001851374 +1898 2.03166 -0.000474608 0.001858998 +1899 2.03263 -0.000476406 0.001866653 +1900 2.03359 -0.000478211 0.001874342 +1901 2.03456 -0.000480024 0.001882063 +1902 2.03552 -0.000481844 0.001889818 +1903 2.03649 -0.000483671 0.001897605 +1904 2.03745 -0.000485506 0.001905426 +1905 2.03842 -0.000487349 0.001913280 +1906 2.03938 -0.000489199 0.001921167 +1907 2.04035 -0.000491057 0.001929089 +1908 2.04131 -0.000492923 0.001937044 +1909 2.04228 -0.000494796 0.001945032 +1910 2.04324 -0.000496677 0.001953055 +1911 2.04421 -0.000498565 0.001961113 +1912 2.04517 -0.000500462 0.001969204 +1913 2.04614 -0.000502366 0.001977330 +1914 2.04710 -0.000504278 0.001985491 +1915 2.04807 -0.000506198 0.001993687 +1916 2.04903 -0.000508126 0.002001918 +1917 2.05000 -0.000510062 0.002010183 +1918 2.05096 -0.000512006 0.002018484 +1919 2.05193 -0.000513958 0.002026821 +1920 2.05289 -0.000515918 0.002035193 +1921 2.05386 -0.000517886 0.002043600 +1922 2.05482 -0.000519862 0.002052044 +1923 2.05579 -0.000521847 0.002060524 +1924 2.05675 -0.000523839 0.002069039 +1925 2.05772 -0.000525840 0.002077591 +1926 2.05868 -0.000527849 0.002086180 +1927 2.05965 -0.000529867 0.002094805 +1928 2.06061 -0.000531893 0.002103467 +1929 2.06158 -0.000533927 0.002112166 +1930 2.06254 -0.000535969 0.002120902 +1931 2.06351 -0.000538020 0.002129676 +1932 2.06447 -0.000540080 0.002138486 +1933 2.06544 -0.000542148 0.002147335 +1934 2.06640 -0.000544224 0.002156221 +1935 2.06737 -0.000546310 0.002165145 +1936 2.06833 -0.000548403 0.002174107 +1937 2.06930 -0.000550506 0.002183107 +1938 2.07026 -0.000552617 0.002192146 +1939 2.07123 -0.000554737 0.002201223 +1940 2.07219 -0.000556866 0.002210339 +1941 2.07316 -0.000559003 0.002219494 +1942 2.07412 -0.000561149 0.002228687 +1943 2.07509 -0.000563305 0.002237920 +1944 2.07605 -0.000565469 0.002247193 +1945 2.07702 -0.000567642 0.002256505 +1946 2.07798 -0.000569824 0.002265856 +1947 2.07895 -0.000572015 0.002275248 +1948 2.07991 -0.000574216 0.002284679 +1949 2.08088 -0.000576425 0.002294151 +1950 2.08184 -0.000578644 0.002303663 +1951 2.08281 -0.000580871 0.002313216 +1952 2.08377 -0.000583108 0.002322809 +1953 2.08474 -0.000585355 0.002332443 +1954 2.08570 -0.000587610 0.002342119 +1955 2.08667 -0.000589875 0.002351835 +1956 2.08763 -0.000592149 0.002361593 +1957 2.08860 -0.000594433 0.002371393 +1958 2.08956 -0.000596726 0.002381234 +1959 2.09053 -0.000599029 0.002391117 +1960 2.09149 -0.000601342 0.002401042 +1961 2.09246 -0.000603663 0.002411010 +1962 2.09342 -0.000605995 0.002421020 +1963 2.09439 -0.000608336 0.002431072 +1964 2.09535 -0.000610687 0.002441168 +1965 2.09632 -0.000613048 0.002451306 +1966 2.09728 -0.000615418 0.002461488 +1967 2.09825 -0.000617799 0.002471713 +1968 2.09921 -0.000620189 0.002481981 +1969 2.10018 -0.000622589 0.002492293 +1970 2.10115 -0.000625000 0.002502649 +1971 2.10211 -0.000627420 0.002513049 +1972 2.10308 -0.000629850 0.002523494 +1973 2.10404 -0.000632290 0.002533982 +1974 2.10501 -0.000634741 0.002544516 +1975 2.10597 -0.000637202 0.002555094 +1976 2.10694 -0.000639672 0.002565717 +1977 2.10790 -0.000642154 0.002576386 +1978 2.10887 -0.000644645 0.002587099 +1979 2.10983 -0.000647147 0.002597859 +1980 2.11080 -0.000649659 0.002608664 +1981 2.11176 -0.000652182 0.002619515 +1982 2.11273 -0.000654715 0.002630412 +1983 2.11369 -0.000657259 0.002641356 +1984 2.11466 -0.000659813 0.002652346 +1985 2.11562 -0.000662378 0.002663382 +1986 2.11659 -0.000664954 0.002674466 +1987 2.11755 -0.000667540 0.002685596 +1988 2.11852 -0.000670137 0.002696774 +1989 2.11948 -0.000672745 0.002708000 +1990 2.12045 -0.000675364 0.002719273 +1991 2.12141 -0.000677994 0.002730594 +1992 2.12238 -0.000680634 0.002741963 +1993 2.12334 -0.000683286 0.002753380 +1994 2.12431 -0.000685949 0.002764846 +1995 2.12527 -0.000688622 0.002776360 +1996 2.12624 -0.000691307 0.002787924 +1997 2.12720 -0.000694003 0.002799536 +1998 2.12817 -0.000696711 0.002811198 +1999 2.12913 -0.000699429 0.002822909 +2000 2.13010 -0.000702159 0.002834670 +2001 2.13106 -0.000704901 0.002846480 +2002 2.13203 -0.000707653 0.002858341 +2003 2.13299 -0.000710417 0.002870252 +2004 2.13396 -0.000713193 0.002882213 +2005 2.13492 -0.000715980 0.002894225 +2006 2.13589 -0.000718779 0.002906288 +2007 2.13685 -0.000721590 0.002918403 +2008 2.13782 -0.000724412 0.002930568 +2009 2.13878 -0.000727246 0.002942785 +2010 2.13975 -0.000730092 0.002955054 +2011 2.14071 -0.000732950 0.002967374 +2012 2.14168 -0.000735819 0.002979747 +2013 2.14264 -0.000738701 0.002992172 +2014 2.14361 -0.000741594 0.003004650 +2015 2.14457 -0.000744500 0.003017181 +2016 2.14554 -0.000747418 0.003029764 +2017 2.14650 -0.000750348 0.003042401 +2018 2.14747 -0.000753290 0.003055091 +2019 2.14843 -0.000756245 0.003067835 +2020 2.14940 -0.000759211 0.003080632 +2021 2.15036 -0.000762190 0.003093484 +2022 2.15133 -0.000765182 0.003106390 +2023 2.15229 -0.000768186 0.003119351 +2024 2.15326 -0.000771203 0.003132366 +2025 2.15422 -0.000774232 0.003145436 +2026 2.15519 -0.000777274 0.003158562 +2027 2.15615 -0.000780328 0.003171743 +2028 2.15712 -0.000783396 0.003184979 +2029 2.15808 -0.000786476 0.003198271 +2030 2.15905 -0.000789569 0.003211620 +2031 2.16001 -0.000792674 0.003225024 +2032 2.16098 -0.000795793 0.003238485 +2033 2.16194 -0.000798925 0.003252003 +2034 2.16291 -0.000802070 0.003265578 +2035 2.16387 -0.000805228 0.003279210 +2036 2.16484 -0.000808399 0.003292899 +2037 2.16580 -0.000811583 0.003306646 +2038 2.16677 -0.000814781 0.003320451 +2039 2.16773 -0.000817992 0.003334314 +2040 2.16870 -0.000821217 0.003348236 +2041 2.16966 -0.000824455 0.003362216 +2042 2.17063 -0.000827706 0.003376254 +2043 2.17159 -0.000830971 0.003390352 +2044 2.17256 -0.000834250 0.003404509 +2045 2.17352 -0.000837542 0.003418726 +2046 2.17449 -0.000840848 0.003433002 +2047 2.17545 -0.000844168 0.003447338 +2048 2.17642 -0.000847502 0.003461734 +2049 2.17738 -0.000850850 0.003476191 +2050 2.17835 -0.000854212 0.003490709 +2051 2.17931 -0.000857587 0.003505287 +2052 2.18028 -0.000860977 0.003519926 +2053 2.18124 -0.000864381 0.003534627 +2054 2.18221 -0.000867799 0.003549390 +2055 2.18317 -0.000871232 0.003564214 +2056 2.18414 -0.000874679 0.003579101 +2057 2.18510 -0.000878140 0.003594050 +2058 2.18607 -0.000881615 0.003609061 +2059 2.18703 -0.000885106 0.003624135 +2060 2.18800 -0.000888610 0.003639273 +2061 2.18896 -0.000892130 0.003654473 +2062 2.18993 -0.000895664 0.003669737 +2063 2.19089 -0.000899213 0.003685065 +2064 2.19186 -0.000902776 0.003700457 +2065 2.19282 -0.000906355 0.003715914 +2066 2.19379 -0.000909949 0.003731435 +2067 2.19475 -0.000913557 0.003747020 +2068 2.19572 -0.000917181 0.003762671 +2069 2.19668 -0.000920819 0.003778387 +2070 2.19765 -0.000924473 0.003794169 +2071 2.19861 -0.000928143 0.003810016 +2072 2.19958 -0.000931827 0.003825929 +2073 2.20055 -0.000935527 0.003841909 +2074 2.20151 -0.000939242 0.003857955 +2075 2.20248 -0.000942973 0.003874068 +2076 2.20344 -0.000946720 0.003890249 +2077 2.20441 -0.000950482 0.003906496 +2078 2.20537 -0.000954260 0.003922811 +2079 2.20634 -0.000958053 0.003939194 +2080 2.20730 -0.000961863 0.003955645 +2081 2.20827 -0.000965688 0.003972164 +2082 2.20923 -0.000969529 0.003988752 +2083 2.21020 -0.000973387 0.004005408 +2084 2.21116 -0.000977260 0.004022134 +2085 2.21213 -0.000981150 0.004038929 +2086 2.21309 -0.000985056 0.004055794 +2087 2.21406 -0.000988978 0.004072728 +2088 2.21502 -0.000992917 0.004089733 +2089 2.21599 -0.000996872 0.004106808 +2090 2.21695 -0.001000843 0.004123954 +2091 2.21792 -0.001004831 0.004141171 +2092 2.21888 -0.001008836 0.004158459 +2093 2.21985 -0.001012857 0.004175818 +2094 2.22081 -0.001016896 0.004193249 +2095 2.22178 -0.001020951 0.004210752 +2096 2.22274 -0.001025023 0.004228327 +2097 2.22371 -0.001029112 0.004245975 +2098 2.22467 -0.001033218 0.004263696 +2099 2.22564 -0.001037341 0.004281490 +2100 2.22660 -0.001041482 0.004299356 +2101 2.22757 -0.001045640 0.004317297 +2102 2.22853 -0.001049815 0.004335311 +2103 2.22950 -0.001054007 0.004353400 +2104 2.23046 -0.001058217 0.004371563 +2105 2.23143 -0.001062445 0.004389800 +2106 2.23239 -0.001066690 0.004408113 +2107 2.23336 -0.001070953 0.004426500 +2108 2.23432 -0.001075234 0.004444964 +2109 2.23529 -0.001079532 0.004463502 +2110 2.23625 -0.001083849 0.004482117 +2111 2.23722 -0.001088183 0.004500808 +2112 2.23818 -0.001092536 0.004519576 +2113 2.23915 -0.001096906 0.004538421 +2114 2.24011 -0.001101295 0.004557342 +2115 2.24108 -0.001105702 0.004576341 +2116 2.24204 -0.001110128 0.004595418 +2117 2.24301 -0.001114572 0.004614572 +2118 2.24397 -0.001119035 0.004633805 +2119 2.24494 -0.001123516 0.004653117 +2120 2.24590 -0.001128016 0.004672506 +2121 2.24687 -0.001132534 0.004691976 +2122 2.24783 -0.001137072 0.004711524 +2123 2.24880 -0.001141628 0.004731152 +2124 2.24976 -0.001146203 0.004750859 +2125 2.25073 -0.001150797 0.004770647 +2126 2.25169 -0.001155411 0.004790516 +2127 2.25266 -0.001160044 0.004810465 +2128 2.25362 -0.001164696 0.004830495 +2129 2.25459 -0.001169367 0.004850606 +2130 2.25555 -0.001174058 0.004870799 +2131 2.25652 -0.001178768 0.004891073 +2132 2.25748 -0.001183498 0.004911430 +2133 2.25845 -0.001188248 0.004931869 +2134 2.25941 -0.001193017 0.004952391 +2135 2.26038 -0.001197806 0.004972995 +2136 2.26134 -0.001202615 0.004993683 +2137 2.26231 -0.001207445 0.005014454 +2138 2.26327 -0.001212294 0.005035309 +2139 2.26424 -0.001217163 0.005056248 +2140 2.26520 -0.001222053 0.005077272 +2141 2.26617 -0.001226963 0.005098380 +2142 2.26713 -0.001231893 0.005119573 +2143 2.26810 -0.001236844 0.005140851 +2144 2.26906 -0.001241816 0.005162215 +2145 2.27003 -0.001246808 0.005183664 +2146 2.27099 -0.001251821 0.005205200 +2147 2.27196 -0.001256854 0.005226821 +2148 2.27292 -0.001261909 0.005248530 +2149 2.27389 -0.001266985 0.005270325 +2150 2.27485 -0.001272081 0.005292207 +2151 2.27582 -0.001277199 0.005314177 +2152 2.27678 -0.001282338 0.005336235 +2153 2.27775 -0.001287499 0.005358380 +2154 2.27871 -0.001292680 0.005380614 +2155 2.27968 -0.001297884 0.005402937 +2156 2.28064 -0.001303109 0.005425348 +2157 2.28161 -0.001308355 0.005447849 +2158 2.28257 -0.001313624 0.005470439 +2159 2.28354 -0.001318914 0.005493118 +2160 2.28450 -0.001324226 0.005515888 +2161 2.28547 -0.001329560 0.005538748 +2162 2.28643 -0.001334916 0.005561698 +2163 2.28740 -0.001340295 0.005584740 +2164 2.28836 -0.001345695 0.005607872 +2165 2.28933 -0.001351118 0.005631096 +2166 2.29029 -0.001356564 0.005654412 +2167 2.29126 -0.001362032 0.005677819 +2168 2.29222 -0.001367523 0.005701319 +2169 2.29319 -0.001373036 0.005724912 +2170 2.29415 -0.001378572 0.005748597 +2171 2.29512 -0.001384131 0.005772375 +2172 2.29608 -0.001389714 0.005796247 +2173 2.29705 -0.001395319 0.005820213 +2174 2.29801 -0.001400947 0.005844272 +2175 2.29898 -0.001406599 0.005868426 +2176 2.29994 -0.001412274 0.005892674 +2177 2.30091 -0.001417972 0.005917017 +2178 2.30188 -0.001423694 0.005941455 +2179 2.30284 -0.001429440 0.005965988 +2180 2.30381 -0.001435209 0.005990617 +2181 2.30477 -0.001441002 0.006015343 +2182 2.30574 -0.001446819 0.006040164 +2183 2.30670 -0.001452661 0.006065081 +2184 2.30767 -0.001458526 0.006090096 +2185 2.30863 -0.001464415 0.006115207 +2186 2.30960 -0.001470329 0.006140416 +2187 2.31056 -0.001476267 0.006165723 +2188 2.31153 -0.001482229 0.006191127 +2189 2.31249 -0.001488216 0.006216629 +2190 2.31346 -0.001494228 0.006242230 +2191 2.31442 -0.001500264 0.006267930 +2192 2.31539 -0.001506326 0.006293728 +2193 2.31635 -0.001512412 0.006319626 +2194 2.31732 -0.001518523 0.006345623 +2195 2.31828 -0.001524660 0.006371720 +2196 2.31925 -0.001530821 0.006397917 +2197 2.32021 -0.001537008 0.006424215 +2198 2.32118 -0.001543221 0.006450613 +2199 2.32214 -0.001549458 0.006477112 +2200 2.32311 -0.001555722 0.006503712 +2201 2.32407 -0.001562011 0.006530413 +2202 2.32504 -0.001568326 0.006557216 +2203 2.32600 -0.001574667 0.006584121 +2204 2.32697 -0.001581034 0.006611128 +2205 2.32793 -0.001587428 0.006638238 +2206 2.32890 -0.001593847 0.006665451 +2207 2.32986 -0.001600293 0.006692766 +2208 2.33083 -0.001606765 0.006720185 +2209 2.33179 -0.001613263 0.006747707 +2210 2.33276 -0.001619788 0.006775333 +2211 2.33372 -0.001626340 0.006803063 +2212 2.33469 -0.001632919 0.006830897 +2213 2.33565 -0.001639525 0.006858836 +2214 2.33662 -0.001646157 0.006886880 +2215 2.33758 -0.001652817 0.006915028 +2216 2.33855 -0.001659504 0.006943282 +2217 2.33951 -0.001666218 0.006971642 +2218 2.34048 -0.001672960 0.007000107 +2219 2.34144 -0.001679729 0.007028679 +2220 2.34241 -0.001686526 0.007057356 +2221 2.34337 -0.001693351 0.007086141 +2222 2.34434 -0.001700203 0.007115032 +2223 2.34530 -0.001707083 0.007144030 +2224 2.34627 -0.001713992 0.007173135 +2225 2.34723 -0.001720928 0.007202348 +2226 2.34820 -0.001727893 0.007231669 +2227 2.34916 -0.001734886 0.007261098 +2228 2.35013 -0.001741908 0.007290635 +2229 2.35109 -0.001748958 0.007320280 +2230 2.35206 -0.001756036 0.007350035 +2231 2.35302 -0.001763144 0.007379898 +2232 2.35399 -0.001770280 0.007409870 +2233 2.35495 -0.001777446 0.007439952 +2234 2.35592 -0.001784640 0.007470143 +2235 2.35688 -0.001791864 0.007500444 +2236 2.35785 -0.001799117 0.007530855 +2237 2.35881 -0.001806399 0.007561377 +2238 2.35978 -0.001813711 0.007592009 +2239 2.36074 -0.001821053 0.007622752 +2240 2.36171 -0.001828424 0.007653606 +2241 2.36267 -0.001835825 0.007684570 +2242 2.36364 -0.001843256 0.007715647 +2243 2.36460 -0.001850717 0.007746835 +2244 2.36557 -0.001858208 0.007778134 +2245 2.36653 -0.001865729 0.007809546 +2246 2.36750 -0.001873281 0.007841070 +2247 2.36846 -0.001880863 0.007872706 +2248 2.36943 -0.001888476 0.007904455 +2249 2.37039 -0.001896120 0.007936317 +2250 2.37136 -0.001903794 0.007968291 +2251 2.37232 -0.001911499 0.008000379 +2252 2.37329 -0.001919236 0.008032580 +2253 2.37425 -0.001927003 0.008064895 +2254 2.37522 -0.001934802 0.008097324 +2255 2.37618 -0.001942632 0.008129866 +2256 2.37715 -0.001950493 0.008162523 +2257 2.37811 -0.001958386 0.008195294 +2258 2.37908 -0.001966311 0.008228179 +2259 2.38004 -0.001974267 0.008261179 +2260 2.38101 -0.001982256 0.008294294 +2261 2.38197 -0.001990276 0.008327523 +2262 2.38294 -0.001998329 0.008360868 +2263 2.38390 -0.002006414 0.008394328 +2264 2.38487 -0.002014531 0.008427904 +2265 2.38583 -0.002022680 0.008461595 +2266 2.38680 -0.002030862 0.008495402 +2267 2.38776 -0.002039077 0.008529325 +2268 2.38873 -0.002047325 0.008563363 +2269 2.38969 -0.002055605 0.008597519 +2270 2.39066 -0.002063919 0.008631790 +2271 2.39162 -0.002072266 0.008666178 +2272 2.39259 -0.002080646 0.008700682 +2273 2.39355 -0.002089059 0.008735303 +2274 2.39452 -0.002097506 0.008770041 +2275 2.39548 -0.002105986 0.008804896 +2276 2.39645 -0.002114500 0.008839868 +2277 2.39741 -0.002123048 0.008874957 +2278 2.39838 -0.002131630 0.008910164 +2279 2.39934 -0.002140245 0.008945488 +2280 2.40031 -0.002148895 0.008980929 +2281 2.40128 -0.002157579 0.009016488 +2282 2.40224 -0.002166298 0.009052165 +2283 2.40321 -0.002175051 0.009087960 +2284 2.40417 -0.002183839 0.009123873 +2285 2.40514 -0.002192661 0.009159903 +2286 2.40610 -0.002201518 0.009196052 +2287 2.40707 -0.002210410 0.009232318 +2288 2.40803 -0.002219337 0.009268703 +2289 2.40900 -0.002228300 0.009305207 +2290 2.40996 -0.002237297 0.009341828 +2291 2.41093 -0.002246330 0.009378568 +2292 2.41189 -0.002255399 0.009415427 +2293 2.41286 -0.002264503 0.009452404 +2294 2.41382 -0.002273643 0.009489499 +2295 2.41479 -0.002282819 0.009526713 +2296 2.41575 -0.002292031 0.009564046 +2297 2.41672 -0.002301278 0.009601498 +2298 2.41768 -0.002310563 0.009639068 +2299 2.41865 -0.002319883 0.009676757 +2300 2.41961 -0.002329240 0.009714564 +2301 2.42058 -0.002338633 0.009752491 +2302 2.42154 -0.002348063 0.009790536 +2303 2.42251 -0.002357530 0.009828699 +2304 2.42347 -0.002367033 0.009866982 +2305 2.42444 -0.002376574 0.009905384 +2306 2.42540 -0.002386152 0.009943904 +2307 2.42637 -0.002395767 0.009982542 +2308 2.42733 -0.002405419 0.010021300 +2309 2.42830 -0.002415109 0.010060176 +2310 2.42926 -0.002424836 0.010099171 +2311 2.43023 -0.002434601 0.010138285 +2312 2.43119 -0.002444404 0.010177517 +2313 2.43216 -0.002454245 0.010216867 +2314 2.43312 -0.002464124 0.010256336 +2315 2.43409 -0.002474041 0.010295923 +2316 2.43505 -0.002483996 0.010335629 +2317 2.43602 -0.002493989 0.010375453 +2318 2.43698 -0.002504021 0.010415395 +2319 2.43795 -0.002514092 0.010455455 +2320 2.43891 -0.002524202 0.010495633 +2321 2.43988 -0.002534350 0.010535929 +2322 2.44084 -0.002544537 0.010576342 +2323 2.44181 -0.002554763 0.010616874 +2324 2.44277 -0.002565029 0.010657522 +2325 2.44374 -0.002575333 0.010698289 +2326 2.44470 -0.002585677 0.010739172 +2327 2.44567 -0.002596061 0.010780173 +2328 2.44663 -0.002606484 0.010821291 +2329 2.44760 -0.002616947 0.010862525 +2330 2.44856 -0.002627450 0.010903876 +2331 2.44953 -0.002637993 0.010945344 +2332 2.45049 -0.002648575 0.010986928 +2333 2.45146 -0.002659199 0.011028628 +2334 2.45242 -0.002669862 0.011070444 +2335 2.45339 -0.002680566 0.011112376 +2336 2.45435 -0.002691310 0.011154424 +2337 2.45532 -0.002702095 0.011196586 +2338 2.45628 -0.002712920 0.011238864 +2339 2.45725 -0.002723787 0.011281257 +2340 2.45821 -0.002734694 0.011323765 +2341 2.45918 -0.002745643 0.011366387 +2342 2.46014 -0.002756633 0.011409123 +2343 2.46111 -0.002767664 0.011451973 +2344 2.46207 -0.002778736 0.011494937 +2345 2.46304 -0.002789850 0.011538014 +2346 2.46400 -0.002801006 0.011581204 +2347 2.46497 -0.002812203 0.011624507 +2348 2.46593 -0.002823442 0.011667923 +2349 2.46690 -0.002834723 0.011711450 +2350 2.46786 -0.002846046 0.011755090 +2351 2.46883 -0.002857412 0.011798841 +2352 2.46979 -0.002868819 0.011842704 +2353 2.47076 -0.002880269 0.011886677 +2354 2.47172 -0.002891762 0.011930761 +2355 2.47269 -0.002903297 0.011974955 +2356 2.47365 -0.002914875 0.012019259 +2357 2.47462 -0.002926495 0.012063672 +2358 2.47558 -0.002938159 0.012108194 +2359 2.47655 -0.002949865 0.012152825 +2360 2.47751 -0.002961615 0.012197564 +2361 2.47848 -0.002973408 0.012242411 +2362 2.47944 -0.002985244 0.012287365 +2363 2.48041 -0.002997123 0.012332427 +2364 2.48137 -0.003009047 0.012377594 +2365 2.48234 -0.003021013 0.012422868 +2366 2.48330 -0.003033024 0.012468247 +2367 2.48427 -0.003045078 0.012513732 +2368 2.48523 -0.003057177 0.012559321 +2369 2.48620 -0.003069319 0.012605014 +2370 2.48716 -0.003081506 0.012650811 +2371 2.48813 -0.003093736 0.012696711 +2372 2.48909 -0.003106012 0.012742713 +2373 2.49006 -0.003118331 0.012788817 +2374 2.49102 -0.003130695 0.012835023 +2375 2.49199 -0.003143104 0.012881330 +2376 2.49295 -0.003155558 0.012927737 +2377 2.49392 -0.003168056 0.012974244 +2378 2.49488 -0.003180599 0.013020850 +2379 2.49585 -0.003193187 0.013067554 +2380 2.49681 -0.003205821 0.013114357 +2381 2.49778 -0.003218499 0.013161256 +2382 2.49874 -0.003231223 0.013208253 +2383 2.49971 -0.003243993 0.013255345 +2384 2.50068 -0.003256807 0.013302533 +2385 2.50164 -0.003269668 0.013349815 +2386 2.50261 -0.003282574 0.013397191 +2387 2.50357 -0.003295526 0.013444661 +2388 2.50454 -0.003308523 0.013492223 +2389 2.50550 -0.003321567 0.013539877 +2390 2.50647 -0.003334657 0.013587622 +2391 2.50743 -0.003347792 0.013635457 +2392 2.50840 -0.003360974 0.013683382 +2393 2.50936 -0.003374203 0.013731396 +2394 2.51033 -0.003387477 0.013779497 +2395 2.51129 -0.003400799 0.013827686 +2396 2.51226 -0.003414166 0.013875962 +2397 2.51322 -0.003427581 0.013924322 +2398 2.51419 -0.003441042 0.013972768 +2399 2.51515 -0.003454549 0.014021297 +2400 2.51612 -0.003468104 0.014069910 +2401 2.51708 -0.003481706 0.014118604 +2402 2.51805 -0.003495354 0.014167380 +2403 2.51901 -0.003509050 0.014216236 +2404 2.51998 -0.003522793 0.014265171 +2405 2.52094 -0.003536583 0.014314185 +2406 2.52191 -0.003550421 0.014363277 +2407 2.52287 -0.003564306 0.014412445 +2408 2.52384 -0.003578238 0.014461688 +2409 2.52480 -0.003592218 0.014511006 +2410 2.52577 -0.003606246 0.014560397 +2411 2.52673 -0.003620321 0.014609862 +2412 2.52770 -0.003634444 0.014659397 +2413 2.52866 -0.003648615 0.014709003 +2414 2.52963 -0.003662834 0.014758678 +2415 2.53059 -0.003677101 0.014808422 +2416 2.53156 -0.003691416 0.014858233 +2417 2.53252 -0.003705779 0.014908110 +2418 2.53349 -0.003720190 0.014958052 +2419 2.53445 -0.003734649 0.015008057 +2420 2.53542 -0.003749157 0.015058126 +2421 2.53638 -0.003763713 0.015108256 +2422 2.53735 -0.003778317 0.015158446 +2423 2.53831 -0.003792970 0.015208695 +2424 2.53928 -0.003807672 0.015259002 +2425 2.54024 -0.003822422 0.015309366 +2426 2.54121 -0.003837220 0.015359785 +2427 2.54217 -0.003852068 0.015410258 +2428 2.54314 -0.003866964 0.015460784 +2429 2.54410 -0.003881908 0.015511361 +2430 2.54507 -0.003896902 0.015561989 +2431 2.54603 -0.003911945 0.015612665 +2432 2.54700 -0.003927036 0.015663389 +2433 2.54796 -0.003942176 0.015714160 +2434 2.54893 -0.003957366 0.015764974 +2435 2.54989 -0.003972604 0.015815833 +2436 2.55086 -0.003987892 0.015866733 +2437 2.55182 -0.004003229 0.015917674 +2438 2.55279 -0.004018615 0.015968654 +2439 2.55375 -0.004034050 0.016019671 +2440 2.55472 -0.004049534 0.016070724 +2441 2.55568 -0.004065068 0.016121812 +2442 2.55665 -0.004080651 0.016172934 +2443 2.55761 -0.004096283 0.016224086 +2444 2.55858 -0.004111965 0.016275269 +2445 2.55954 -0.004127696 0.016326480 +2446 2.56051 -0.004143477 0.016377718 +2447 2.56147 -0.004159307 0.016428981 +2448 2.56244 -0.004175186 0.016480267 +2449 2.56340 -0.004191115 0.016531576 +2450 2.56437 -0.004207094 0.016582905 +2451 2.56533 -0.004223122 0.016634252 +2452 2.56630 -0.004239199 0.016685616 +2453 2.56726 -0.004255327 0.016736996 +2454 2.56823 -0.004271503 0.016788388 +2455 2.56919 -0.004287730 0.016839793 +2456 2.57016 -0.004304006 0.016891207 +2457 2.57112 -0.004320331 0.016942629 +2458 2.57209 -0.004336707 0.016994058 +2459 2.57305 -0.004353132 0.017045491 +2460 2.57402 -0.004369606 0.017096927 +2461 2.57498 -0.004386130 0.017148363 +2462 2.57595 -0.004402704 0.017199799 +2463 2.57691 -0.004419328 0.017251231 +2464 2.57788 -0.004436001 0.017302658 +2465 2.57884 -0.004452723 0.017354078 +2466 2.57981 -0.004469496 0.017405489 +2467 2.58077 -0.004486318 0.017456890 +2468 2.58174 -0.004503189 0.017508277 +2469 2.58270 -0.004520110 0.017559650 +2470 2.58367 -0.004537081 0.017611006 +2471 2.58463 -0.004554101 0.017662342 +2472 2.58560 -0.004571171 0.017713658 +2473 2.58656 -0.004588290 0.017764950 +2474 2.58753 -0.004605459 0.017816217 +2475 2.58849 -0.004622677 0.017867456 +2476 2.58946 -0.004639945 0.017918666 +2477 2.59042 -0.004657262 0.017969844 +2478 2.59139 -0.004674629 0.018020988 +2479 2.59235 -0.004692044 0.018072096 +2480 2.59332 -0.004709509 0.018123166 +2481 2.59428 -0.004727024 0.018174194 +2482 2.59525 -0.004744587 0.018225180 +2483 2.59621 -0.004762200 0.018276120 +2484 2.59718 -0.004779862 0.018327013 +2485 2.59814 -0.004797573 0.018377856 +2486 2.59911 -0.004815333 0.018428646 +2487 2.60008 -0.004833142 0.018479382 +2488 2.60104 -0.004851000 0.018530061 +2489 2.60201 -0.004868907 0.018580680 +2490 2.60297 -0.004886862 0.018631237 +2491 2.60394 -0.004904867 0.018681730 +2492 2.60490 -0.004922920 0.018732156 +2493 2.60587 -0.004941022 0.018782512 +2494 2.60683 -0.004959172 0.018832797 +2495 2.60780 -0.004977371 0.018883007 +2496 2.60876 -0.004995618 0.018933140 +2497 2.60973 -0.005013914 0.018983194 +2498 2.61069 -0.005032257 0.019033165 +2499 2.61166 -0.005050649 0.019083052 +2500 2.61262 -0.005069089 0.019132851 +2501 2.61359 -0.005087578 0.019182560 +2502 2.61455 -0.005106114 0.019232176 +2503 2.61552 -0.005124698 0.019281697 +2504 2.61648 -0.005143329 0.019331120 +2505 2.61745 -0.005162008 0.019380441 +2506 2.61841 -0.005180735 0.019429659 +2507 2.61938 -0.005199510 0.019478771 +2508 2.62034 -0.005218331 0.019527773 +2509 2.62131 -0.005237200 0.019576663 +2510 2.62227 -0.005256116 0.019625439 +2511 2.62324 -0.005275079 0.019674096 +2512 2.62420 -0.005294089 0.019722633 +2513 2.62517 -0.005313145 0.019771046 +2514 2.62613 -0.005332249 0.019819333 +2515 2.62710 -0.005351399 0.019867491 +2516 2.62806 -0.005370595 0.019915516 +2517 2.62903 -0.005389837 0.019963405 +2518 2.62999 -0.005409126 0.020011157 +2519 2.63096 -0.005428461 0.020058767 +2520 2.63192 -0.005447841 0.020106233 +2521 2.63289 -0.005467268 0.020153551 +2522 2.63385 -0.005486740 0.020200719 +2523 2.63482 -0.005506257 0.020247734 +2524 2.63578 -0.005525820 0.020294592 +2525 2.63675 -0.005545428 0.020341290 +2526 2.63771 -0.005565080 0.020387825 +2527 2.63868 -0.005584778 0.020434194 +2528 2.63964 -0.005604520 0.020480393 +2529 2.64061 -0.005624307 0.020526421 +2530 2.64157 -0.005644138 0.020572272 +2531 2.64254 -0.005664013 0.020617945 +2532 2.64350 -0.005683933 0.020663436 +2533 2.64447 -0.005703896 0.020708741 +2534 2.64543 -0.005723903 0.020753857 +2535 2.64640 -0.005743953 0.020798782 +2536 2.64736 -0.005764046 0.020843511 +2537 2.64833 -0.005784183 0.020888042 +2538 2.64929 -0.005804362 0.020932370 +2539 2.65026 -0.005824584 0.020976494 +2540 2.65122 -0.005844849 0.021020408 +2541 2.65219 -0.005865155 0.021064110 +2542 2.65315 -0.005885504 0.021107597 +2543 2.65412 -0.005905895 0.021150865 +2544 2.65508 -0.005926327 0.021193910 +2545 2.65605 -0.005946801 0.021236730 +2546 2.65701 -0.005967316 0.021279320 +2547 2.65798 -0.005987872 0.021321677 +2548 2.65894 -0.006008469 0.021363798 +2549 2.65991 -0.006029106 0.021405679 +2550 2.66087 -0.006049784 0.021447317 +2551 2.66184 -0.006070502 0.021488707 +2552 2.66280 -0.006091259 0.021529847 +2553 2.66377 -0.006112056 0.021570733 +2554 2.66473 -0.006132893 0.021611361 +2555 2.66570 -0.006153768 0.021651728 +2556 2.66666 -0.006174683 0.021691830 +2557 2.66763 -0.006195636 0.021731663 +2558 2.66859 -0.006216627 0.021771224 +2559 2.66956 -0.006237656 0.021810509 +2560 2.67052 -0.006258723 0.021849515 +2561 2.67149 -0.006279828 0.021888237 +2562 2.67245 -0.006300969 0.021926672 +2563 2.67342 -0.006322148 0.021964817 +2564 2.67438 -0.006343363 0.022002668 +2565 2.67535 -0.006364615 0.022040220 +2566 2.67631 -0.006385903 0.022077471 +2567 2.67728 -0.006407227 0.022114416 +2568 2.67824 -0.006428586 0.022151051 +2569 2.67921 -0.006449980 0.022187374 +2570 2.68017 -0.006471410 0.022223380 +2571 2.68114 -0.006492874 0.022259065 +2572 2.68210 -0.006514372 0.022294426 +2573 2.68307 -0.006535904 0.022329458 +2574 2.68403 -0.006557470 0.022364159 +2575 2.68500 -0.006579069 0.022398524 +2576 2.68596 -0.006600701 0.022432549 +2577 2.68693 -0.006622366 0.022466230 +2578 2.68789 -0.006644063 0.022499565 +2579 2.68886 -0.006665792 0.022532548 +2580 2.68982 -0.006687553 0.022565176 +2581 2.69079 -0.006709345 0.022597445 +2582 2.69175 -0.006731168 0.022629351 +2583 2.69272 -0.006753021 0.022660891 +2584 2.69368 -0.006774905 0.022692061 +2585 2.69465 -0.006796819 0.022722856 +2586 2.69561 -0.006818762 0.022753273 +2587 2.69658 -0.006840735 0.022783308 +2588 2.69754 -0.006862736 0.022812957 +2589 2.69851 -0.006884766 0.022842216 +2590 2.69947 -0.006906824 0.022871081 +2591 2.70044 -0.006928909 0.022899549 +2592 2.70141 -0.006951022 0.022927616 +2593 2.70237 -0.006973162 0.022955277 +2594 2.70334 -0.006995328 0.022982528 +2595 2.70430 -0.007017520 0.023009367 +2596 2.70527 -0.007039738 0.023035789 +2597 2.70623 -0.007061981 0.023061789 +2598 2.70720 -0.007084249 0.023087365 +2599 2.70816 -0.007106542 0.023112512 +2600 2.70913 -0.007128859 0.023137227 +2601 2.71009 -0.007151199 0.023161505 +2602 2.71106 -0.007173562 0.023185343 +2603 2.71202 -0.007195949 0.023208737 +2604 2.71299 -0.007218357 0.023231683 +2605 2.71395 -0.007240788 0.023254177 +2606 2.71492 -0.007263240 0.023276215 +2607 2.71588 -0.007285713 0.023297794 +2608 2.71685 -0.007308207 0.023318909 +2609 2.71781 -0.007330721 0.023339557 +2610 2.71878 -0.007353254 0.023359734 +2611 2.71974 -0.007375807 0.023379437 +2612 2.72071 -0.007398379 0.023398660 +2613 2.72167 -0.007420969 0.023417401 +2614 2.72264 -0.007443576 0.023435656 +2615 2.72360 -0.007466202 0.023453420 +2616 2.72457 -0.007488844 0.023470691 +2617 2.72553 -0.007511502 0.023487464 +2618 2.72650 -0.007534177 0.023503736 +2619 2.72746 -0.007556866 0.023519503 +2620 2.72843 -0.007579571 0.023534761 +2621 2.72939 -0.007602291 0.023549507 +2622 2.73036 -0.007625024 0.023563736 +2623 2.73132 -0.007647771 0.023577446 +2624 2.73229 -0.007670531 0.023590632 +2625 2.73325 -0.007693303 0.023603290 +2626 2.73422 -0.007716087 0.023615419 +2627 2.73518 -0.007738883 0.023627012 +2628 2.73615 -0.007761689 0.023638068 +2629 2.73711 -0.007784506 0.023648582 +2630 2.73808 -0.007807333 0.023658550 +2631 2.73904 -0.007830169 0.023667971 +2632 2.74001 -0.007853014 0.023676838 +2633 2.74097 -0.007875868 0.023685150 +2634 2.74194 -0.007898729 0.023692903 +2635 2.74290 -0.007921597 0.023700094 +2636 2.74387 -0.007944472 0.023706718 +2637 2.74483 -0.007967353 0.023712772 +2638 2.74580 -0.007990240 0.023718254 +2639 2.74676 -0.008013132 0.023723159 +2640 2.74773 -0.008036028 0.023727485 +2641 2.74869 -0.008058928 0.023731227 +2642 2.74966 -0.008081831 0.023734383 +2643 2.75062 -0.008104737 0.023736950 +2644 2.75159 -0.008127645 0.023738924 +2645 2.75255 -0.008150555 0.023740302 +2646 2.75352 -0.008173466 0.023741081 +2647 2.75448 -0.008196378 0.023741257 +2648 2.75545 -0.008219289 0.023740828 +2649 2.75641 -0.008242200 0.023739791 +2650 2.75738 -0.008265109 0.023738141 +2651 2.75834 -0.008288016 0.023735877 +2652 2.75931 -0.008310921 0.023732995 +2653 2.76027 -0.008333823 0.023729493 +2654 2.76124 -0.008356721 0.023725367 +2655 2.76220 -0.008379615 0.023720614 +2656 2.76317 -0.008402504 0.023715232 +2657 2.76413 -0.008425388 0.023709217 +2658 2.76510 -0.008448265 0.023702568 +2659 2.76606 -0.008471136 0.023695280 +2660 2.76703 -0.008493999 0.023687352 +2661 2.76799 -0.008516854 0.023678781 +2662 2.76896 -0.008539701 0.023669564 +2663 2.76992 -0.008562539 0.023659698 +2664 2.77089 -0.008585366 0.023649181 +2665 2.77185 -0.008608184 0.023638010 +2666 2.77282 -0.008630990 0.023626183 +2667 2.77378 -0.008653784 0.023613698 +2668 2.77475 -0.008676566 0.023600551 +2669 2.77571 -0.008699335 0.023586741 +2670 2.77668 -0.008722091 0.023572264 +2671 2.77764 -0.008744832 0.023557120 +2672 2.77861 -0.008767558 0.023541305 +2673 2.77957 -0.008790269 0.023524818 +2674 2.78054 -0.008812963 0.023507656 +2675 2.78150 -0.008835641 0.023489817 +2676 2.78247 -0.008858300 0.023471299 +2677 2.78343 -0.008880942 0.023452100 +2678 2.78440 -0.008903565 0.023432218 +2679 2.78536 -0.008926168 0.023411651 +2680 2.78633 -0.008948752 0.023390397 +2681 2.78729 -0.008971314 0.023368454 +2682 2.78826 -0.008993855 0.023345821 +2683 2.78922 -0.009016373 0.023322496 +2684 2.79019 -0.009038869 0.023298477 +2685 2.79115 -0.009061341 0.023273762 +2686 2.79212 -0.009083790 0.023248350 +2687 2.79308 -0.009106213 0.023222240 +2688 2.79405 -0.009128611 0.023195429 +2689 2.79501 -0.009150982 0.023167917 +2690 2.79598 -0.009173327 0.023139701 +2691 2.79694 -0.009195644 0.023110781 +2692 2.79791 -0.009217932 0.023081156 +2693 2.79887 -0.009240192 0.023050824 +2694 2.79984 -0.009262422 0.023019784 +2695 2.80081 -0.009284622 0.022988034 +2696 2.80177 -0.009306791 0.022955575 +2697 2.80274 -0.009328929 0.022922404 +2698 2.80370 -0.009351034 0.022888522 +2699 2.80467 -0.009373106 0.022853926 +2700 2.80563 -0.009395144 0.022818616 +2701 2.80660 -0.009417147 0.022782592 +2702 2.80756 -0.009439116 0.022745852 +2703 2.80853 -0.009461049 0.022708396 +2704 2.80949 -0.009482945 0.022670224 +2705 2.81046 -0.009504804 0.022631335 +2706 2.81142 -0.009526626 0.022591728 +2707 2.81239 -0.009548408 0.022551403 +2708 2.81335 -0.009570152 0.022510359 +2709 2.81432 -0.009591855 0.022468597 +2710 2.81528 -0.009613518 0.022426116 +2711 2.81625 -0.009635140 0.022382916 +2712 2.81721 -0.009656719 0.022338996 +2713 2.81818 -0.009678256 0.022294358 +2714 2.81914 -0.009699749 0.022249000 +2715 2.82011 -0.009721198 0.022202923 +2716 2.82107 -0.009742603 0.022156126 +2717 2.82204 -0.009763962 0.022108611 +2718 2.82300 -0.009785274 0.022060378 +2719 2.82397 -0.009806540 0.022011426 +2720 2.82493 -0.009827758 0.021961756 +2721 2.82590 -0.009848928 0.021911369 +2722 2.82686 -0.009870049 0.021860265 +2723 2.82783 -0.009891120 0.021808444 +2724 2.82879 -0.009912141 0.021755908 +2725 2.82976 -0.009933111 0.021702657 +2726 2.83072 -0.009954029 0.021648692 +2727 2.83169 -0.009974895 0.021594014 +2728 2.83265 -0.009995707 0.021538623 +2729 2.83362 -0.010016466 0.021482520 +2730 2.83458 -0.010037171 0.021425708 +2731 2.83555 -0.010057820 0.021368185 +2732 2.83651 -0.010078413 0.021309955 +2733 2.83748 -0.010098950 0.021251018 +2734 2.83844 -0.010119429 0.021191375 +2735 2.83941 -0.010139851 0.021131028 +2736 2.84037 -0.010160214 0.021069979 +2737 2.84134 -0.010180518 0.021008228 +2738 2.84230 -0.010200762 0.020945777 +2739 2.84327 -0.010220945 0.020882628 +2740 2.84423 -0.010241067 0.020818783 +2741 2.84520 -0.010261127 0.020754244 +2742 2.84616 -0.010281124 0.020689011 +2743 2.84713 -0.010301059 0.020623088 +2744 2.84809 -0.010320929 0.020556476 +2745 2.84906 -0.010340734 0.020489177 +2746 2.85002 -0.010360475 0.020421193 +2747 2.85099 -0.010380149 0.020352526 +2748 2.85195 -0.010399757 0.020283180 +2749 2.85292 -0.010419297 0.020213155 +2750 2.85388 -0.010438770 0.020142454 +2751 2.85485 -0.010458174 0.020071081 +2752 2.85581 -0.010477509 0.019999036 +2753 2.85678 -0.010496774 0.019926323 +2754 2.85774 -0.010515968 0.019852945 +2755 2.85871 -0.010535092 0.019778904 +2756 2.85967 -0.010554143 0.019704203 +2757 2.86064 -0.010573122 0.019628844 +2758 2.86160 -0.010592029 0.019552831 +2759 2.86257 -0.010610861 0.019476166 +2760 2.86353 -0.010629619 0.019398853 +2761 2.86450 -0.010648302 0.019320895 +2762 2.86546 -0.010666910 0.019242294 +2763 2.86643 -0.010685442 0.019163055 +2764 2.86739 -0.010703897 0.019083179 +2765 2.86836 -0.010722274 0.019002672 +2766 2.86932 -0.010740573 0.018921535 +2767 2.87029 -0.010758794 0.018839772 +2768 2.87125 -0.010776936 0.018757388 +2769 2.87222 -0.010794998 0.018674384 +2770 2.87318 -0.010812979 0.018590767 +2771 2.87415 -0.010830879 0.018506537 +2772 2.87511 -0.010848698 0.018421701 +2773 2.87608 -0.010866435 0.018336261 +2774 2.87704 -0.010884089 0.018250221 +2775 2.87801 -0.010901659 0.018163585 +2776 2.87897 -0.010919146 0.018076358 +2777 2.87994 -0.010936548 0.017988543 +2778 2.88090 -0.010953865 0.017900144 +2779 2.88187 -0.010971097 0.017811166 +2780 2.88283 -0.010988242 0.017721613 +2781 2.88380 -0.011005301 0.017631488 +2782 2.88476 -0.011022273 0.017540797 +2783 2.88573 -0.011039156 0.017449544 +2784 2.88669 -0.011055952 0.017357733 +2785 2.88766 -0.011072658 0.017265369 +2786 2.88862 -0.011089276 0.017172456 +2787 2.88959 -0.011105803 0.017078999 +2788 2.89055 -0.011122239 0.016985003 +2789 2.89152 -0.011138585 0.016890472 +2790 2.89248 -0.011154840 0.016795410 +2791 2.89345 -0.011171002 0.016699824 +2792 2.89441 -0.011187072 0.016603717 +2793 2.89538 -0.011203048 0.016507095 +2794 2.89634 -0.011218932 0.016409963 +2795 2.89731 -0.011234721 0.016312325 +2796 2.89827 -0.011250416 0.016214186 +2797 2.89924 -0.011266016 0.016115553 +2798 2.90021 -0.011281520 0.016016429 +2799 2.90117 -0.011296929 0.015916821 +2800 2.90214 -0.011312241 0.015816732 +2801 2.90310 -0.011327457 0.015716169 +2802 2.90407 -0.011342575 0.015615138 +2803 2.90503 -0.011357595 0.015513642 +2804 2.90600 -0.011372517 0.015411688 +2805 2.90696 -0.011387341 0.015309282 +2806 2.90793 -0.011402066 0.015206428 +2807 2.90889 -0.011416691 0.015103132 +2808 2.90986 -0.011431216 0.014999400 +2809 2.91082 -0.011445641 0.014895237 +2810 2.91179 -0.011459965 0.014790649 +2811 2.91275 -0.011474188 0.014685642 +2812 2.91372 -0.011488310 0.014580221 +2813 2.91468 -0.011502329 0.014474393 +2814 2.91565 -0.011516247 0.014368162 +2815 2.91661 -0.011530061 0.014261535 +2816 2.91758 -0.011543773 0.014154518 +2817 2.91854 -0.011557381 0.014047117 +2818 2.91951 -0.011570885 0.013939336 +2819 2.92047 -0.011584285 0.013831183 +2820 2.92144 -0.011597580 0.013722664 +2821 2.92240 -0.011610771 0.013613784 +2822 2.92337 -0.011623856 0.013504549 +2823 2.92433 -0.011636836 0.013394965 +2824 2.92530 -0.011649709 0.013285040 +2825 2.92626 -0.011662477 0.013174778 +2826 2.92723 -0.011675138 0.013064185 +2827 2.92819 -0.011687692 0.012953269 +2828 2.92916 -0.011700139 0.012842034 +2829 2.93012 -0.011712478 0.012730488 +2830 2.93109 -0.011724710 0.012618637 +2831 2.93205 -0.011736833 0.012506487 +2832 2.93302 -0.011748849 0.012394043 +2833 2.93398 -0.011760755 0.012281313 +2834 2.93495 -0.011772553 0.012168302 +2835 2.93591 -0.011784241 0.012055018 +2836 2.93688 -0.011795820 0.011941466 +2837 2.93784 -0.011807289 0.011827652 +2838 2.93881 -0.011818648 0.011713583 +2839 2.93977 -0.011829897 0.011599266 +2840 2.94074 -0.011841036 0.011484707 +2841 2.94170 -0.011852064 0.011369912 +2842 2.94267 -0.011862981 0.011254887 +2843 2.94363 -0.011873787 0.011139640 +2844 2.94460 -0.011884481 0.011024176 +2845 2.94556 -0.011895064 0.010908501 +2846 2.94653 -0.011905536 0.010792623 +2847 2.94749 -0.011915895 0.010676548 +2848 2.94846 -0.011926142 0.010560282 +2849 2.94942 -0.011936277 0.010443832 +2850 2.95039 -0.011946300 0.010327204 +2851 2.95135 -0.011956210 0.010210405 +2852 2.95232 -0.011966007 0.010093440 +2853 2.95328 -0.011975691 0.009976318 +2854 2.95425 -0.011985262 0.009859043 +2855 2.95521 -0.011994720 0.009741622 +2856 2.95618 -0.012004064 0.009624063 +2857 2.95714 -0.012013295 0.009506371 +2858 2.95811 -0.012022413 0.009388553 +2859 2.95907 -0.012031416 0.009270616 +2860 2.96004 -0.012040306 0.009152565 +2861 2.96100 -0.012049081 0.009034407 +2862 2.96197 -0.012057743 0.008916149 +2863 2.96293 -0.012066290 0.008797798 +2864 2.96390 -0.012074724 0.008679358 +2865 2.96486 -0.012083042 0.008560838 +2866 2.96583 -0.012091247 0.008442243 +2867 2.96679 -0.012099337 0.008323580 +2868 2.96776 -0.012107312 0.008204855 +2869 2.96872 -0.012115173 0.008086074 +2870 2.96969 -0.012122919 0.007967245 +2871 2.97065 -0.012130550 0.007848372 +2872 2.97162 -0.012138067 0.007729463 +2873 2.97258 -0.012145469 0.007610524 +2874 2.97355 -0.012152756 0.007491561 +2875 2.97451 -0.012159928 0.007372581 +2876 2.97548 -0.012166986 0.007253589 +2877 2.97644 -0.012173929 0.007134592 +2878 2.97741 -0.012180756 0.007015596 +2879 2.97837 -0.012187469 0.006896608 +2880 2.97934 -0.012194067 0.006777633 +2881 2.98030 -0.012200551 0.006658678 +2882 2.98127 -0.012206919 0.006539749 +2883 2.98223 -0.012213173 0.006420851 +2884 2.98320 -0.012219312 0.006301992 +2885 2.98416 -0.012225337 0.006183177 +2886 2.98513 -0.012231246 0.006064412 +2887 2.98609 -0.012237042 0.005945703 +2888 2.98706 -0.012242722 0.005827056 +2889 2.98802 -0.012248288 0.005708478 +2890 2.98899 -0.012253740 0.005589974 +2891 2.98995 -0.012259078 0.005471549 +2892 2.99092 -0.012264301 0.005353211 +2893 2.99188 -0.012269410 0.005234964 +2894 2.99285 -0.012274405 0.005116815 +2895 2.99381 -0.012279286 0.004998769 +2896 2.99478 -0.012284053 0.004880832 +2897 2.99574 -0.012288706 0.004763010 +2898 2.99671 -0.012293246 0.004645308 +2899 2.99767 -0.012297672 0.004527733 +2900 2.99864 -0.012301985 0.004410289 +2901 2.99960 -0.012306185 0.004292983 +2902 3.00057 -0.012310271 0.004175820 +2903 3.00154 -0.012314244 0.004058805 +2904 3.00250 -0.012318105 0.003941944 +2905 3.00347 -0.012321853 0.003825243 +2906 3.00443 -0.012325488 0.003708706 +2907 3.00540 -0.012329011 0.003592340 +2908 3.00636 -0.012332422 0.003476149 +2909 3.00733 -0.012335720 0.003360139 +2910 3.00829 -0.012338907 0.003244315 +2911 3.00926 -0.012341982 0.003128683 +2912 3.01022 -0.012344946 0.003013246 +2913 3.01119 -0.012347798 0.002898012 +2914 3.01215 -0.012350539 0.002782984 +2915 3.01312 -0.012353170 0.002668168 +2916 3.01408 -0.012355689 0.002553569 +2917 3.01505 -0.012358098 0.002439192 +2918 3.01601 -0.012360397 0.002325041 +2919 3.01698 -0.012362586 0.002211122 +2920 3.01794 -0.012364665 0.002097440 +2921 3.01891 -0.012366634 0.001983998 +2922 3.01987 -0.012368494 0.001870803 +2923 3.02084 -0.012370245 0.001757859 +2924 3.02180 -0.012371887 0.001645169 +2925 3.02277 -0.012373421 0.001532740 +2926 3.02373 -0.012374846 0.001420576 +2927 3.02470 -0.012376163 0.001308681 +2928 3.02566 -0.012377372 0.001197059 +2929 3.02663 -0.012378473 0.001085716 +2930 3.02759 -0.012379467 0.000974655 +2931 3.02856 -0.012380354 0.000863881 +2932 3.02952 -0.012381135 0.000753398 +2933 3.03049 -0.012381809 0.000643211 +2934 3.03145 -0.012382376 0.000533323 +2935 3.03242 -0.012382838 0.000423739 +2936 3.03338 -0.012383194 0.000314463 +2937 3.03435 -0.012383445 0.000205499 +2938 3.03531 -0.012383591 0.000096851 +2939 3.03628 -0.012383632 -0.000011477 +2940 3.03724 -0.012383569 -0.000119481 +2941 3.03821 -0.012383402 -0.000227157 +2942 3.03917 -0.012383131 -0.000334501 +2943 3.04014 -0.012382756 -0.000441511 +2944 3.04110 -0.012382279 -0.000548181 +2945 3.04207 -0.012381698 -0.000654508 +2946 3.04303 -0.012381015 -0.000760488 +2947 3.04400 -0.012380230 -0.000866119 +2948 3.04496 -0.012379344 -0.000971396 +2949 3.04593 -0.012378356 -0.001076316 +2950 3.04689 -0.012377267 -0.001180875 +2951 3.04786 -0.012376077 -0.001285071 +2952 3.04882 -0.012374786 -0.001388899 +2953 3.04979 -0.012373396 -0.001492357 +2954 3.05075 -0.012371906 -0.001595441 +2955 3.05172 -0.012370317 -0.001698148 +2956 3.05268 -0.012368629 -0.001800474 +2957 3.05365 -0.012366842 -0.001902418 +2958 3.05461 -0.012364957 -0.002003975 +2959 3.05558 -0.012362974 -0.002105144 +2960 3.05654 -0.012360894 -0.002205920 +2961 3.05751 -0.012358717 -0.002306301 +2962 3.05847 -0.012356443 -0.002406284 +2963 3.05944 -0.012354072 -0.002505866 +2964 3.06040 -0.012351606 -0.002605045 +2965 3.06137 -0.012349044 -0.002703818 +2966 3.06233 -0.012346388 -0.002802182 +2967 3.06330 -0.012343636 -0.002900135 +2968 3.06426 -0.012340790 -0.002997674 +2969 3.06523 -0.012337850 -0.003094796 +2970 3.06619 -0.012334817 -0.003191500 +2971 3.06716 -0.012331691 -0.003287782 +2972 3.06812 -0.012328472 -0.003383641 +2973 3.06909 -0.012325160 -0.003479073 +2974 3.07005 -0.012321757 -0.003574078 +2975 3.07102 -0.012318262 -0.003668652 +2976 3.07198 -0.012314676 -0.003762794 +2977 3.07295 -0.012310999 -0.003856502 +2978 3.07391 -0.012307233 -0.003949772 +2979 3.07488 -0.012303376 -0.004042604 +2980 3.07584 -0.012299430 -0.004134995 +2981 3.07681 -0.012295395 -0.004226944 +2982 3.07777 -0.012291272 -0.004318448 +2983 3.07874 -0.012287061 -0.004409506 +2984 3.07970 -0.012282761 -0.004500117 +2985 3.08067 -0.012278375 -0.004590277 +2986 3.08163 -0.012273902 -0.004679986 +2987 3.08260 -0.012269342 -0.004769242 +2988 3.08356 -0.012264697 -0.004858043 +2989 3.08453 -0.012259966 -0.004946388 +2990 3.08549 -0.012255150 -0.005034276 +2991 3.08646 -0.012250249 -0.005121704 +2992 3.08742 -0.012245265 -0.005208672 +2993 3.08839 -0.012240196 -0.005295178 +2994 3.08935 -0.012235045 -0.005381221 +2995 3.09032 -0.012229810 -0.005466799 +2996 3.09128 -0.012224493 -0.005551912 +2997 3.09225 -0.012219095 -0.005636558 +2998 3.09321 -0.012213614 -0.005720736 +2999 3.09418 -0.012208053 -0.005804444 +3000 3.09514 -0.012202412 -0.005887683 +3001 3.09611 -0.012196690 -0.005970451 +3002 3.09707 -0.012190888 -0.006052746 +3003 3.09804 -0.012185007 -0.006134569 +3004 3.09900 -0.012179048 -0.006215917 +3005 3.09997 -0.012173010 -0.006296791 +3006 3.10094 -0.012166895 -0.006377190 +3007 3.10190 -0.012160702 -0.006457112 +3008 3.10287 -0.012154432 -0.006536557 +3009 3.10383 -0.012148086 -0.006615525 +3010 3.10480 -0.012141664 -0.006694014 +3011 3.10576 -0.012135166 -0.006772025 +3012 3.10673 -0.012128593 -0.006849556 +3013 3.10769 -0.012121946 -0.006926607 +3014 3.10866 -0.012115224 -0.007003178 +3015 3.10962 -0.012108429 -0.007079269 +3016 3.11059 -0.012101561 -0.007154877 +3017 3.11155 -0.012094620 -0.007230005 +3018 3.11252 -0.012087606 -0.007304650 +3019 3.11348 -0.012080521 -0.007378814 +3020 3.11445 -0.012073365 -0.007452495 +3021 3.11541 -0.012066137 -0.007525693 +3022 3.11638 -0.012058839 -0.007598409 +3023 3.11734 -0.012051472 -0.007670642 +3024 3.11831 -0.012044034 -0.007742392 +3025 3.11927 -0.012036528 -0.007813659 +3026 3.12024 -0.012028953 -0.007884443 +3027 3.12120 -0.012021311 -0.007954744 +3028 3.12217 -0.012013600 -0.008024562 +3029 3.12313 -0.012005823 -0.008093897 +3030 3.12410 -0.011997978 -0.008162749 +3031 3.12506 -0.011990068 -0.008231119 +3032 3.12603 -0.011982092 -0.008299006 +3033 3.12699 -0.011974050 -0.008366411 +3034 3.12796 -0.011965944 -0.008433334 +3035 3.12892 -0.011957773 -0.008499776 +3036 3.12989 -0.011949539 -0.008565736 +3037 3.13085 -0.011941241 -0.008631215 +3038 3.13182 -0.011932880 -0.008696213 +3039 3.13278 -0.011924456 -0.008760731 +3040 3.13375 -0.011915971 -0.008824769 +3041 3.13471 -0.011907424 -0.008888328 +3042 3.13568 -0.011898816 -0.008951408 +3043 3.13664 -0.011890147 -0.009014010 +3044 3.13761 -0.011881418 -0.009076134 +3045 3.13857 -0.011872629 -0.009137781 +3046 3.13954 -0.011863781 -0.009198951 +3047 3.14050 -0.011854874 -0.009259646 +3048 3.14147 -0.011845909 -0.009319865 +3049 3.14243 -0.011836886 -0.009379610 +3050 3.14340 -0.011827806 -0.009438880 +3051 3.14436 -0.011818669 -0.009497678 +3052 3.14533 -0.011809475 -0.009556004 +3053 3.14629 -0.011800225 -0.009613858 +3054 3.14726 -0.011790919 -0.009671241 +3055 3.14822 -0.011781558 -0.009728154 +3056 3.14919 -0.011772143 -0.009784599 +3057 3.15015 -0.011762673 -0.009840576 +3058 3.15112 -0.011753150 -0.009896085 +3059 3.15208 -0.011743573 -0.009951128 +3060 3.15305 -0.011733943 -0.010005706 +3061 3.15401 -0.011724261 -0.010059820 +3062 3.15498 -0.011714527 -0.010113471 +3063 3.15594 -0.011704741 -0.010166660 +3064 3.15691 -0.011694905 -0.010219387 +3065 3.15787 -0.011685017 -0.010271655 +3066 3.15884 -0.011675080 -0.010323464 +3067 3.15980 -0.011665092 -0.010374815 +3068 3.16077 -0.011655055 -0.010425710 +3069 3.16173 -0.011644970 -0.010476149 +3070 3.16270 -0.011634835 -0.010526133 +3071 3.16366 -0.011624653 -0.010575665 +3072 3.16463 -0.011614424 -0.010624745 +3073 3.16559 -0.011604147 -0.010673375 +3074 3.16656 -0.011593823 -0.010721555 +3075 3.16752 -0.011583453 -0.010769288 +3076 3.16849 -0.011573037 -0.010816573 +3077 3.16945 -0.011562576 -0.010863414 +3078 3.17042 -0.011552070 -0.010909810 +3079 3.17138 -0.011541519 -0.010955763 +3080 3.17235 -0.011530925 -0.011001276 +3081 3.17331 -0.011520286 -0.011046348 +3082 3.17428 -0.011509604 -0.011090982 +3083 3.17524 -0.011498880 -0.011135179 +3084 3.17621 -0.011488112 -0.011178940 +3085 3.17717 -0.011477303 -0.011222267 +3086 3.17814 -0.011466452 -0.011265161 +3087 3.17910 -0.011455561 -0.011307624 +3088 3.18007 -0.011444628 -0.011349658 +3089 3.18103 -0.011433655 -0.011391263 +3090 3.18200 -0.011422642 -0.011432441 +3091 3.18296 -0.011411589 -0.011473194 +3092 3.18393 -0.011400497 -0.011513524 +3093 3.18489 -0.011389367 -0.011553431 +3094 3.18586 -0.011378198 -0.011592918 +3095 3.18682 -0.011366992 -0.011631986 +3096 3.18779 -0.011355748 -0.011670637 +3097 3.18875 -0.011344466 -0.011708872 +3098 3.18972 -0.011333149 -0.011746693 +3099 3.19068 -0.011321794 -0.011784102 +3100 3.19165 -0.011310404 -0.011821099 +3101 3.19261 -0.011298979 -0.011857688 +3102 3.19358 -0.011287518 -0.011893869 +3103 3.19454 -0.011276022 -0.011929644 +3104 3.19551 -0.011264493 -0.011965015 +3105 3.19647 -0.011252929 -0.011999984 +3106 3.19744 -0.011241332 -0.012034551 +3107 3.19840 -0.011229701 -0.012068720 +3108 3.19937 -0.011218038 -0.012102492 +3109 3.20034 -0.011206342 -0.012135868 +3110 3.20130 -0.011194615 -0.012168850 +3111 3.20227 -0.011182855 -0.012201440 +3112 3.20323 -0.011171065 -0.012233639 +3113 3.20420 -0.011159243 -0.012265450 +3114 3.20516 -0.011147391 -0.012296874 +3115 3.20613 -0.011135509 -0.012327914 +3116 3.20709 -0.011123598 -0.012358569 +3117 3.20806 -0.011111656 -0.012388844 +3118 3.20902 -0.011099686 -0.012418738 +3119 3.20999 -0.011087687 -0.012448255 +3120 3.21095 -0.011075660 -0.012477395 +3121 3.21192 -0.011063605 -0.012506161 +3122 3.21288 -0.011051522 -0.012534555 +3123 3.21385 -0.011039412 -0.012562578 +3124 3.21481 -0.011027275 -0.012590232 +3125 3.21578 -0.011015112 -0.012617519 +3126 3.21674 -0.011002922 -0.012644441 +3127 3.21771 -0.010990707 -0.012670999 +3128 3.21867 -0.010978466 -0.012697196 +3129 3.21964 -0.010966200 -0.012723034 +3130 3.22060 -0.010953909 -0.012748513 +3131 3.22157 -0.010941594 -0.012773637 +3132 3.22253 -0.010929255 -0.012798406 +3133 3.22350 -0.010916892 -0.012822823 +3134 3.22446 -0.010904506 -0.012846889 +3135 3.22543 -0.010892097 -0.012870607 +3136 3.22639 -0.010879664 -0.012893978 +3137 3.22736 -0.010867210 -0.012917005 +3138 3.22832 -0.010854734 -0.012939688 +3139 3.22929 -0.010842235 -0.012962031 +3140 3.23025 -0.010829716 -0.012984034 +3141 3.23122 -0.010817175 -0.013005699 +3142 3.23218 -0.010804614 -0.013027029 +3143 3.23315 -0.010792032 -0.013048026 +3144 3.23411 -0.010779430 -0.013068691 +3145 3.23508 -0.010766808 -0.013089025 +3146 3.23604 -0.010754167 -0.013109032 +3147 3.23701 -0.010741506 -0.013128713 +3148 3.23797 -0.010728827 -0.013148069 +3149 3.23894 -0.010716129 -0.013167103 +3150 3.23990 -0.010703413 -0.013185817 +3151 3.24087 -0.010690680 -0.013204212 +3152 3.24183 -0.010677928 -0.013222290 +3153 3.24280 -0.010665159 -0.013240053 +3154 3.24376 -0.010652374 -0.013257503 +3155 3.24473 -0.010639571 -0.013274642 +3156 3.24569 -0.010626752 -0.013291471 +3157 3.24666 -0.010613918 -0.013307994 +3158 3.24762 -0.010601067 -0.013324210 +3159 3.24859 -0.010588201 -0.013340123 +3160 3.24955 -0.010575319 -0.013355734 +3161 3.25052 -0.010562423 -0.013371046 +3162 3.25148 -0.010549512 -0.013386059 +3163 3.25245 -0.010536587 -0.013400775 +3164 3.25341 -0.010523647 -0.013415198 +3165 3.25438 -0.010510694 -0.013429327 +3166 3.25534 -0.010497727 -0.013443167 +3167 3.25631 -0.010484748 -0.013456717 +3168 3.25727 -0.010471755 -0.013469980 +3169 3.25824 -0.010458749 -0.013482958 +3170 3.25920 -0.010445731 -0.013495653 +3171 3.26017 -0.010432701 -0.013508066 +3172 3.26113 -0.010419660 -0.013520200 +3173 3.26210 -0.010406606 -0.013532055 +3174 3.26306 -0.010393542 -0.013543635 +3175 3.26403 -0.010380466 -0.013554941 +3176 3.26499 -0.010367379 -0.013565974 +3177 3.26596 -0.010354282 -0.013576736 +3178 3.26692 -0.010341175 -0.013587230 +3179 3.26789 -0.010328058 -0.013597457 +3180 3.26885 -0.010314931 -0.013607419 +3181 3.26982 -0.010301794 -0.013617117 +3182 3.27078 -0.010288648 -0.013626554 +3183 3.27175 -0.010275494 -0.013635731 +3184 3.27271 -0.010262330 -0.013644651 +3185 3.27368 -0.010249158 -0.013653314 +3186 3.27464 -0.010235978 -0.013661722 +3187 3.27561 -0.010222790 -0.013669878 +3188 3.27657 -0.010209594 -0.013677783 +3189 3.27754 -0.010196391 -0.013685439 +3190 3.27850 -0.010183180 -0.013692847 +3191 3.27947 -0.010169962 -0.013700010 +3192 3.28043 -0.010156738 -0.013706929 +3193 3.28140 -0.010143506 -0.013713606 +3194 3.28236 -0.010130269 -0.013720043 +3195 3.28333 -0.010117026 -0.013726241 +3196 3.28429 -0.010103776 -0.013732202 +3197 3.28526 -0.010090521 -0.013737928 +3198 3.28622 -0.010077261 -0.013743420 +3199 3.28719 -0.010063995 -0.013748680 +3200 3.28815 -0.010050724 -0.013753711 +3201 3.28912 -0.010037449 -0.013758513 +3202 3.29008 -0.010024169 -0.013763088 +3203 3.29105 -0.010010885 -0.013767438 +3204 3.29201 -0.009997597 -0.013771565 +3205 3.29298 -0.009984305 -0.013775470 +3206 3.29394 -0.009971009 -0.013779156 +3207 3.29491 -0.009957710 -0.013782623 +3208 3.29587 -0.009944407 -0.013785873 +3209 3.29684 -0.009931102 -0.013788908 +3210 3.29780 -0.009917793 -0.013791730 +3211 3.29877 -0.009904482 -0.013794340 +3212 3.29973 -0.009891169 -0.013796740 +3213 3.30070 -0.009877853 -0.013798932 +3214 3.30167 -0.009864536 -0.013800917 +3215 3.30263 -0.009851216 -0.013802696 +3216 3.30360 -0.009837895 -0.013804272 +3217 3.30456 -0.009824573 -0.013805645 +3218 3.30553 -0.009811249 -0.013806819 +3219 3.30649 -0.009797924 -0.013807793 +3220 3.30746 -0.009784599 -0.013808570 +3221 3.30842 -0.009771273 -0.013809151 +3222 3.30939 -0.009757946 -0.013809539 +3223 3.31035 -0.009744619 -0.013809733 +3224 3.31132 -0.009731292 -0.013809737 +3225 3.31228 -0.009717965 -0.013809551 +3226 3.31325 -0.009704638 -0.013809177 +3227 3.31421 -0.009691312 -0.013808616 +3228 3.31518 -0.009677986 -0.013807871 +3229 3.31614 -0.009664661 -0.013806942 +3230 3.31711 -0.009651338 -0.013805832 +3231 3.31807 -0.009638015 -0.013804541 +3232 3.31904 -0.009624694 -0.013803071 +3233 3.32000 -0.009611374 -0.013801423 +3234 3.32097 -0.009598056 -0.013799600 +3235 3.32193 -0.009584739 -0.013797603 +3236 3.32290 -0.009571425 -0.013795432 +3237 3.32386 -0.009558113 -0.013793090 +3238 3.32483 -0.009544803 -0.013790578 +3239 3.32579 -0.009531496 -0.013787897 +3240 3.32676 -0.009518191 -0.013785049 +3241 3.32772 -0.009504889 -0.013782035 +3242 3.32869 -0.009491590 -0.013778857 +3243 3.32965 -0.009478295 -0.013775516 +3244 3.33062 -0.009465002 -0.013772014 +3245 3.33158 -0.009451714 -0.013768352 +3246 3.33255 -0.009438428 -0.013764531 +3247 3.33351 -0.009425147 -0.013760552 +3248 3.33448 -0.009411869 -0.013756418 +3249 3.33544 -0.009398596 -0.013752129 +3250 3.33641 -0.009385326 -0.013747687 +3251 3.33737 -0.009372061 -0.013743093 +3252 3.33834 -0.009358801 -0.013738349 +3253 3.33930 -0.009345545 -0.013733456 +3254 3.34027 -0.009332294 -0.013728415 +3255 3.34123 -0.009319048 -0.013723227 +3256 3.34220 -0.009305807 -0.013717895 +3257 3.34316 -0.009292571 -0.013712418 +3258 3.34413 -0.009279341 -0.013706800 +3259 3.34509 -0.009266116 -0.013701040 +3260 3.34606 -0.009252896 -0.013695140 +3261 3.34702 -0.009239683 -0.013689101 +3262 3.34799 -0.009226475 -0.013682926 +3263 3.34895 -0.009213273 -0.013676614 +3264 3.34992 -0.009200078 -0.013670167 +3265 3.35088 -0.009186889 -0.013663587 +3266 3.35185 -0.009173706 -0.013656875 +3267 3.35281 -0.009160530 -0.013650032 +3268 3.35378 -0.009147360 -0.013643059 +3269 3.35474 -0.009134197 -0.013635957 +3270 3.35571 -0.009121041 -0.013628728 +3271 3.35667 -0.009107893 -0.013621373 +3272 3.35764 -0.009094751 -0.013613893 +3273 3.35860 -0.009081616 -0.013606290 +3274 3.35957 -0.009068489 -0.013598564 +3275 3.36053 -0.009055370 -0.013590717 +3276 3.36150 -0.009042258 -0.013582749 +3277 3.36246 -0.009029154 -0.013574663 +3278 3.36343 -0.009016058 -0.013566459 +3279 3.36439 -0.009002969 -0.013558138 +3280 3.36536 -0.008989889 -0.013549702 +3281 3.36632 -0.008976817 -0.013541151 +3282 3.36729 -0.008963754 -0.013532488 +3283 3.36825 -0.008950698 -0.013523712 +3284 3.36922 -0.008937652 -0.013514825 +3285 3.37018 -0.008924613 -0.013505829 +3286 3.37115 -0.008911584 -0.013496724 +3287 3.37211 -0.008898563 -0.013487511 +3288 3.37308 -0.008885552 -0.013478192 +3289 3.37404 -0.008872549 -0.013468768 +3290 3.37501 -0.008859556 -0.013459239 +3291 3.37597 -0.008846572 -0.013449607 +3292 3.37694 -0.008833597 -0.013439872 +3293 3.37790 -0.008820631 -0.013430037 +3294 3.37887 -0.008807676 -0.013420102 +3295 3.37983 -0.008794729 -0.013410067 +3296 3.38080 -0.008781793 -0.013399935 +3297 3.38176 -0.008768866 -0.013389706 +3298 3.38273 -0.008755950 -0.013379380 +3299 3.38369 -0.008743043 -0.013368960 +3300 3.38466 -0.008730146 -0.013358446 +3301 3.38562 -0.008717260 -0.013347840 +3302 3.38659 -0.008704384 -0.013337141 +3303 3.38755 -0.008691518 -0.013326352 +3304 3.38852 -0.008678662 -0.013315472 +3305 3.38948 -0.008665818 -0.013304504 +3306 3.39045 -0.008652984 -0.013293448 +3307 3.39141 -0.008640160 -0.013282305 +3308 3.39238 -0.008627347 -0.013271076 +3309 3.39334 -0.008614546 -0.013259762 +3310 3.39431 -0.008601755 -0.013248364 +3311 3.39527 -0.008588975 -0.013236883 +3312 3.39624 -0.008576206 -0.013225319 +3313 3.39720 -0.008563449 -0.013213675 +3314 3.39817 -0.008550703 -0.013201950 +3315 3.39913 -0.008537968 -0.013190145 +3316 3.40010 -0.008525245 -0.013178262 +3317 3.40107 -0.008512533 -0.013166302 +3318 3.40203 -0.008499832 -0.013154265 +3319 3.40300 -0.008487144 -0.013142152 +3320 3.40396 -0.008474467 -0.013129964 +3321 3.40493 -0.008461802 -0.013117701 +3322 3.40589 -0.008449148 -0.013105366 +3323 3.40686 -0.008436507 -0.013092959 +3324 3.40782 -0.008423878 -0.013080480 +3325 3.40879 -0.008411260 -0.013067930 +3326 3.40975 -0.008398655 -0.013055311 +3327 3.41072 -0.008386062 -0.013042623 +3328 3.41168 -0.008373482 -0.013029866 +3329 3.41265 -0.008360914 -0.013017043 +3330 3.41361 -0.008348358 -0.013004153 +3331 3.41458 -0.008335814 -0.012991197 +3332 3.41554 -0.008323283 -0.012978177 +3333 3.41651 -0.008310765 -0.012965093 +3334 3.41747 -0.008298260 -0.012951945 +3335 3.41844 -0.008285767 -0.012938735 +3336 3.41940 -0.008273287 -0.012925464 +3337 3.42037 -0.008260819 -0.012912131 +3338 3.42133 -0.008248365 -0.012898739 +3339 3.42230 -0.008235924 -0.012885288 +3340 3.42326 -0.008223495 -0.012871778 +3341 3.42423 -0.008211080 -0.012858210 +3342 3.42519 -0.008198678 -0.012844585 +3343 3.42616 -0.008186289 -0.012830904 +3344 3.42712 -0.008173913 -0.012817167 +3345 3.42809 -0.008161550 -0.012803376 +3346 3.42905 -0.008149201 -0.012789531 +3347 3.43002 -0.008136865 -0.012775632 +3348 3.43098 -0.008124543 -0.012761681 +3349 3.43195 -0.008112234 -0.012747678 +3350 3.43291 -0.008099939 -0.012733624 +3351 3.43388 -0.008087657 -0.012719519 +3352 3.43484 -0.008075389 -0.012705365 +3353 3.43581 -0.008063134 -0.012691162 +3354 3.43677 -0.008050894 -0.012676910 +3355 3.43774 -0.008038667 -0.012662610 +3356 3.43870 -0.008026453 -0.012648264 +3357 3.43967 -0.008014254 -0.012633871 +3358 3.44063 -0.008002069 -0.012619433 +3359 3.44160 -0.007989898 -0.012604949 +3360 3.44256 -0.007977740 -0.012590422 +3361 3.44353 -0.007965597 -0.012575850 +3362 3.44449 -0.007953468 -0.012561236 +3363 3.44546 -0.007941352 -0.012546579 +3364 3.44642 -0.007929251 -0.012531880 +3365 3.44739 -0.007917165 -0.012517140 +3366 3.44835 -0.007905092 -0.012502360 +3367 3.44932 -0.007893034 -0.012487540 +3368 3.45028 -0.007880990 -0.012472681 +3369 3.45125 -0.007868960 -0.012457782 +3370 3.45221 -0.007856945 -0.012442846 +3371 3.45318 -0.007844945 -0.012427873 +3372 3.45414 -0.007832958 -0.012412862 +3373 3.45511 -0.007820987 -0.012397816 +3374 3.45607 -0.007809029 -0.012382733 +3375 3.45704 -0.007797087 -0.012367616 +3376 3.45800 -0.007785159 -0.012352464 +3377 3.45897 -0.007773245 -0.012337278 +3378 3.45993 -0.007761347 -0.012322059 +3379 3.46090 -0.007749463 -0.012306807 +3380 3.46186 -0.007737593 -0.012291523 +3381 3.46283 -0.007725739 -0.012276207 +3382 3.46379 -0.007713899 -0.012260859 +3383 3.46476 -0.007702074 -0.012245482 +3384 3.46572 -0.007690264 -0.012230074 +3385 3.46669 -0.007678469 -0.012214637 +3386 3.46765 -0.007666689 -0.012199171 +3387 3.46862 -0.007654923 -0.012183676 +3388 3.46958 -0.007643173 -0.012168153 +3389 3.47055 -0.007631438 -0.012152603 +3390 3.47151 -0.007619717 -0.012137026 +3391 3.47248 -0.007608012 -0.012121423 +3392 3.47344 -0.007596322 -0.012105794 +3393 3.47441 -0.007584647 -0.012090139 +3394 3.47537 -0.007572987 -0.012074460 +3395 3.47634 -0.007561342 -0.012058756 +3396 3.47730 -0.007549712 -0.012043029 +3397 3.47827 -0.007538098 -0.012027277 +3398 3.47923 -0.007526498 -0.012011503 +3399 3.48020 -0.007514914 -0.011995707 +3400 3.48116 -0.007503345 -0.011979889 +3401 3.48213 -0.007491792 -0.011964049 +3402 3.48309 -0.007480254 -0.011948188 +3403 3.48406 -0.007468731 -0.011932306 +3404 3.48502 -0.007457223 -0.011916404 +3405 3.48599 -0.007445731 -0.011900483 +3406 3.48695 -0.007434254 -0.011884542 +3407 3.48792 -0.007422793 -0.011868583 +3408 3.48888 -0.007411347 -0.011852605 +3409 3.48985 -0.007399916 -0.011836610 +3410 3.49081 -0.007388501 -0.011820597 +3411 3.49178 -0.007377101 -0.011804567 +3412 3.49274 -0.007365717 -0.011788520 +3413 3.49371 -0.007354348 -0.011772457 +3414 3.49467 -0.007342995 -0.011756378 +3415 3.49564 -0.007331657 -0.011740284 +3416 3.49660 -0.007320335 -0.011724176 +3417 3.49757 -0.007309028 -0.011708052 +3418 3.49853 -0.007297737 -0.011691914 +3419 3.49950 -0.007286462 -0.011675763 +3420 3.50047 -0.007275202 -0.011659598 +3421 3.50143 -0.007263958 -0.011643421 +3422 3.50240 -0.007252729 -0.011627231 +3423 3.50336 -0.007241516 -0.011611028 +3424 3.50433 -0.007230319 -0.011594814 +3425 3.50529 -0.007219137 -0.011578589 +3426 3.50626 -0.007207971 -0.011562352 +3427 3.50722 -0.007196820 -0.011546105 +3428 3.50819 -0.007185686 -0.011529848 +3429 3.50915 -0.007174567 -0.011513580 +3430 3.51012 -0.007163463 -0.011497303 +3431 3.51108 -0.007152376 -0.011481017 +3432 3.51205 -0.007141304 -0.011464722 +3433 3.51301 -0.007130248 -0.011448419 +3434 3.51398 -0.007119207 -0.011432107 +3435 3.51494 -0.007108183 -0.011415788 +3436 3.51591 -0.007097174 -0.011399461 +3437 3.51687 -0.007086181 -0.011383127 +3438 3.51784 -0.007075203 -0.011366787 +3439 3.51880 -0.007064242 -0.011350439 +3440 3.51977 -0.007053296 -0.011334086 +3441 3.52073 -0.007042366 -0.011317727 +3442 3.52170 -0.007031452 -0.011301363 +3443 3.52266 -0.007020553 -0.011284993 +3444 3.52363 -0.007009670 -0.011268619 +3445 3.52459 -0.006998804 -0.011252240 +3446 3.52556 -0.006987953 -0.011235857 +3447 3.52652 -0.006977117 -0.011219470 +3448 3.52749 -0.006966298 -0.011203079 +3449 3.52845 -0.006955494 -0.011186685 +3450 3.52942 -0.006944707 -0.011170288 +3451 3.53038 -0.006933935 -0.011153889 +3452 3.53135 -0.006923178 -0.011137487 +3453 3.53231 -0.006912438 -0.011121083 +3454 3.53328 -0.006901714 -0.011104677 +3455 3.53424 -0.006891005 -0.011088270 +3456 3.53521 -0.006880312 -0.011071862 +3457 3.53617 -0.006869635 -0.011055452 +3458 3.53714 -0.006858974 -0.011039042 +3459 3.53810 -0.006848329 -0.011022632 +3460 3.53907 -0.006837699 -0.011006221 +3461 3.54003 -0.006827086 -0.010989810 +3462 3.54100 -0.006816488 -0.010973400 +3463 3.54196 -0.006805906 -0.010956991 +3464 3.54293 -0.006795340 -0.010940583 +3465 3.54389 -0.006784790 -0.010924175 +3466 3.54486 -0.006774255 -0.010907770 +3467 3.54582 -0.006763737 -0.010891366 +3468 3.54679 -0.006753234 -0.010874963 +3469 3.54775 -0.006742747 -0.010858564 +3470 3.54872 -0.006732276 -0.010842166 +3471 3.54968 -0.006721820 -0.010825772 +3472 3.55065 -0.006711381 -0.010809380 +3473 3.55161 -0.006700957 -0.010792991 +3474 3.55258 -0.006690549 -0.010776606 +3475 3.55354 -0.006680157 -0.010760225 +3476 3.55451 -0.006669781 -0.010743847 +3477 3.55547 -0.006659421 -0.010727474 +3478 3.55644 -0.006649076 -0.010711105 +3479 3.55740 -0.006638747 -0.010694740 +3480 3.55837 -0.006628434 -0.010678380 +3481 3.55933 -0.006618137 -0.010662026 +3482 3.56030 -0.006607856 -0.010645676 +3483 3.56126 -0.006597590 -0.010629332 +3484 3.56223 -0.006587340 -0.010612994 +3485 3.56319 -0.006577106 -0.010596661 +3486 3.56416 -0.006566887 -0.010580335 +3487 3.56512 -0.006556685 -0.010564015 +3488 3.56609 -0.006546498 -0.010547701 +3489 3.56705 -0.006536327 -0.010531394 +3490 3.56802 -0.006526171 -0.010515094 +3491 3.56898 -0.006516031 -0.010498802 +3492 3.56995 -0.006505907 -0.010482516 +3493 3.57091 -0.006495799 -0.010466238 +3494 3.57188 -0.006485707 -0.010449968 +3495 3.57284 -0.006475630 -0.010433705 +3496 3.57381 -0.006465569 -0.010417451 +3497 3.57477 -0.006455523 -0.010401205 +3498 3.57574 -0.006445493 -0.010384967 +3499 3.57670 -0.006435479 -0.010368738 +3500 3.57767 -0.006425481 -0.010352517 +3501 3.57863 -0.006415498 -0.010336306 +3502 3.57960 -0.006405530 -0.010320104 +3503 3.58056 -0.006395579 -0.010303911 +3504 3.58153 -0.006385643 -0.010287728 +3505 3.58249 -0.006375723 -0.010271554 +3506 3.58346 -0.006365818 -0.010255390 +3507 3.58442 -0.006355929 -0.010239236 +3508 3.58539 -0.006346055 -0.010223093 +3509 3.58635 -0.006336197 -0.010206959 +3510 3.58732 -0.006326355 -0.010190837 +3511 3.58828 -0.006316528 -0.010174725 +3512 3.58925 -0.006306716 -0.010158623 +3513 3.59021 -0.006296921 -0.010142533 +3514 3.59118 -0.006287140 -0.010126454 +3515 3.59214 -0.006277376 -0.010110386 +3516 3.59311 -0.006267626 -0.010094329 +3517 3.59407 -0.006257893 -0.010078284 +3518 3.59504 -0.006248174 -0.010062251 +3519 3.59600 -0.006238471 -0.010046230 +3520 3.59697 -0.006228784 -0.010030220 +3521 3.59793 -0.006219112 -0.010014223 +3522 3.59890 -0.006209456 -0.009998239 +3523 3.59986 -0.006199815 -0.009982266 +3524 3.60083 -0.006190189 -0.009966306 +3525 3.60180 -0.006180579 -0.009950359 +3526 3.60276 -0.006170984 -0.009934425 +3527 3.60373 -0.006161404 -0.009918504 +3528 3.60469 -0.006151840 -0.009902595 +3529 3.60566 -0.006142291 -0.009886701 +3530 3.60662 -0.006132758 -0.009870819 +3531 3.60759 -0.006123240 -0.009854951 +3532 3.60855 -0.006113737 -0.009839097 +3533 3.60952 -0.006104249 -0.009823256 +3534 3.61048 -0.006094777 -0.009807429 +3535 3.61145 -0.006085320 -0.009791616 +3536 3.61241 -0.006075878 -0.009775817 +3537 3.61338 -0.006066452 -0.009760033 +3538 3.61434 -0.006057040 -0.009744263 +3539 3.61531 -0.006047644 -0.009728507 +3540 3.61627 -0.006038263 -0.009712766 +3541 3.61724 -0.006028898 -0.009697040 +3542 3.61820 -0.006019547 -0.009681328 +3543 3.61917 -0.006010212 -0.009665631 +3544 3.62013 -0.006000892 -0.009649949 +3545 3.62110 -0.005991586 -0.009634283 +3546 3.62206 -0.005982296 -0.009618631 +3547 3.62303 -0.005973022 -0.009602995 +3548 3.62399 -0.005963762 -0.009587374 +3549 3.62496 -0.005954517 -0.009571769 +3550 3.62592 -0.005945287 -0.009556180 +3551 3.62689 -0.005936073 -0.009540606 +3552 3.62785 -0.005926873 -0.009525048 +3553 3.62882 -0.005917688 -0.009509506 +3554 3.62978 -0.005908519 -0.009493980 +3555 3.63075 -0.005899364 -0.009478470 +3556 3.63171 -0.005890224 -0.009462976 +3557 3.63268 -0.005881100 -0.009447499 +3558 3.63364 -0.005871990 -0.009432038 +3559 3.63461 -0.005862895 -0.009416593 +3560 3.63557 -0.005853815 -0.009401165 +3561 3.63654 -0.005844750 -0.009385754 +3562 3.63750 -0.005835699 -0.009370359 +3563 3.63847 -0.005826664 -0.009354981 +3564 3.63943 -0.005817643 -0.009339620 +3565 3.64040 -0.005808638 -0.009324276 +3566 3.64136 -0.005799647 -0.009308949 +3567 3.64233 -0.005790670 -0.009293639 +3568 3.64329 -0.005781709 -0.009278346 +3569 3.64426 -0.005772762 -0.009263071 +3570 3.64522 -0.005763830 -0.009247813 +3571 3.64619 -0.005754913 -0.009232573 +3572 3.64715 -0.005746011 -0.009217350 +3573 3.64812 -0.005737123 -0.009202144 +3574 3.64908 -0.005728250 -0.009186956 +3575 3.65005 -0.005719391 -0.009171786 +3576 3.65101 -0.005710547 -0.009156634 +3577 3.65198 -0.005701718 -0.009141500 +3578 3.65294 -0.005692903 -0.009126383 +3579 3.65391 -0.005684103 -0.009111285 +3580 3.65487 -0.005675318 -0.009096204 +3581 3.65584 -0.005666547 -0.009081142 +3582 3.65680 -0.005657790 -0.009066098 +3583 3.65777 -0.005649048 -0.009051072 +3584 3.65873 -0.005640321 -0.009036064 +3585 3.65970 -0.005631608 -0.009021075 +3586 3.66066 -0.005622909 -0.009006105 +3587 3.66163 -0.005614225 -0.008991152 +3588 3.66259 -0.005605555 -0.008976219 +3589 3.66356 -0.005596900 -0.008961304 +3590 3.66452 -0.005588259 -0.008946407 +3591 3.66549 -0.005579633 -0.008931529 +3592 3.66645 -0.005571020 -0.008916671 +3593 3.66742 -0.005562423 -0.008901830 +3594 3.66838 -0.005553839 -0.008887009 +3595 3.66935 -0.005545270 -0.008872207 +3596 3.67031 -0.005536715 -0.008857424 +3597 3.67128 -0.005528174 -0.008842659 +3598 3.67224 -0.005519648 -0.008827914 +3599 3.67321 -0.005511135 -0.008813188 +3600 3.67417 -0.005502637 -0.008798481 +3601 3.67514 -0.005494153 -0.008783794 +3602 3.67610 -0.005485684 -0.008769125 +3603 3.67707 -0.005477228 -0.008754476 +3604 3.67803 -0.005468787 -0.008739846 +3605 3.67900 -0.005460359 -0.008725236 +3606 3.67996 -0.005451946 -0.008710645 +3607 3.68093 -0.005443547 -0.008696074 +3608 3.68189 -0.005435162 -0.008681522 +3609 3.68286 -0.005426791 -0.008666989 +3610 3.68382 -0.005418434 -0.008652477 +3611 3.68479 -0.005410091 -0.008637984 +3612 3.68575 -0.005401762 -0.008623510 +3613 3.68672 -0.005393446 -0.008609056 +3614 3.68768 -0.005385145 -0.008594622 +3615 3.68865 -0.005376858 -0.008580208 +3616 3.68961 -0.005368585 -0.008565814 +3617 3.69058 -0.005360325 -0.008551439 +3618 3.69154 -0.005352080 -0.008537085 +3619 3.69251 -0.005343848 -0.008522750 +3620 3.69347 -0.005335630 -0.008508435 +3621 3.69444 -0.005327426 -0.008494141 +3622 3.69540 -0.005319235 -0.008479866 +3623 3.69637 -0.005311059 -0.008465611 +3624 3.69733 -0.005302896 -0.008451376 +3625 3.69830 -0.005294747 -0.008437162 +3626 3.69926 -0.005286611 -0.008422967 +3627 3.70023 -0.005278490 -0.008408793 +3628 3.70120 -0.005270381 -0.008394639 +3629 3.70216 -0.005262287 -0.008380505 +3630 3.70313 -0.005254206 -0.008366392 +3631 3.70409 -0.005246139 -0.008352298 +3632 3.70506 -0.005238086 -0.008338225 +3633 3.70602 -0.005230046 -0.008324172 +3634 3.70699 -0.005222019 -0.008310140 +3635 3.70795 -0.005214006 -0.008296128 +3636 3.70892 -0.005206007 -0.008282136 +3637 3.70988 -0.005198021 -0.008268165 +3638 3.71085 -0.005190048 -0.008254214 +3639 3.71181 -0.005182089 -0.008240283 +3640 3.71278 -0.005174144 -0.008226373 +3641 3.71374 -0.005166212 -0.008212484 +3642 3.71471 -0.005158293 -0.008198614 +3643 3.71567 -0.005150388 -0.008184766 +3644 3.71664 -0.005142496 -0.008170938 +3645 3.71760 -0.005134617 -0.008157130 +3646 3.71857 -0.005126751 -0.008143343 +3647 3.71953 -0.005118899 -0.008129577 +3648 3.72050 -0.005111061 -0.008115831 +3649 3.72146 -0.005103235 -0.008102106 +3650 3.72243 -0.005095423 -0.008088401 +3651 3.72339 -0.005087624 -0.008074717 +3652 3.72436 -0.005079838 -0.008061054 +3653 3.72532 -0.005072065 -0.008047411 +3654 3.72629 -0.005064305 -0.008033789 +3655 3.72725 -0.005056559 -0.008020188 +3656 3.72822 -0.005048826 -0.008006607 +3657 3.72918 -0.005041106 -0.007993047 +3658 3.73015 -0.005033398 -0.007979508 +3659 3.73111 -0.005025704 -0.007965990 +3660 3.73208 -0.005018023 -0.007952492 +3661 3.73304 -0.005010355 -0.007939015 +3662 3.73401 -0.005002700 -0.007925558 +3663 3.73497 -0.004995058 -0.007912123 +3664 3.73594 -0.004987429 -0.007898708 +3665 3.73690 -0.004979813 -0.007885314 +3666 3.73787 -0.004972210 -0.007871941 +3667 3.73883 -0.004964619 -0.007858588 +3668 3.73980 -0.004957042 -0.007845257 +3669 3.74076 -0.004949477 -0.007831946 +3670 3.74173 -0.004941925 -0.007818655 +3671 3.74269 -0.004934386 -0.007805386 +3672 3.74366 -0.004926860 -0.007792137 +3673 3.74462 -0.004919347 -0.007778910 +3674 3.74559 -0.004911846 -0.007765703 +3675 3.74655 -0.004904358 -0.007752517 +3676 3.74752 -0.004896883 -0.007739351 +3677 3.74848 -0.004889420 -0.007726207 +3678 3.74945 -0.004881971 -0.007713083 +3679 3.75041 -0.004874533 -0.007699980 +3680 3.75138 -0.004867109 -0.007686898 +3681 3.75234 -0.004859697 -0.007673837 +3682 3.75331 -0.004852298 -0.007660796 +3683 3.75427 -0.004844911 -0.007647777 +3684 3.75524 -0.004837537 -0.007634778 +3685 3.75620 -0.004830175 -0.007621800 +3686 3.75717 -0.004822826 -0.007608842 +3687 3.75813 -0.004815489 -0.007595906 +3688 3.75910 -0.004808165 -0.007582990 +3689 3.76006 -0.004800853 -0.007570095 +3690 3.76103 -0.004793554 -0.007557221 +3691 3.76199 -0.004786267 -0.007544368 +3692 3.76296 -0.004778993 -0.007531536 +3693 3.76392 -0.004771731 -0.007518724 +3694 3.76489 -0.004764481 -0.007505933 +3695 3.76585 -0.004757243 -0.007493163 +3696 3.76682 -0.004750018 -0.007480414 +3697 3.76778 -0.004742805 -0.007467685 +3698 3.76875 -0.004735605 -0.007454977 +3699 3.76971 -0.004728417 -0.007442290 +3700 3.77068 -0.004721241 -0.007429624 +3701 3.77164 -0.004714077 -0.007416978 +3702 3.77261 -0.004706925 -0.007404353 +3703 3.77357 -0.004699786 -0.007391749 +3704 3.77454 -0.004692658 -0.007379166 +3705 3.77550 -0.004685543 -0.007366603 +3706 3.77647 -0.004678440 -0.007354061 +3707 3.77743 -0.004671349 -0.007341540 +3708 3.77840 -0.004664270 -0.007329039 +3709 3.77936 -0.004657203 -0.007316559 +3710 3.78033 -0.004650148 -0.007304100 +3711 3.78129 -0.004643106 -0.007291661 +3712 3.78226 -0.004636075 -0.007279243 +3713 3.78322 -0.004629056 -0.007266846 +3714 3.78419 -0.004622049 -0.007254469 +3715 3.78515 -0.004615054 -0.007242113 +3716 3.78612 -0.004608071 -0.007229777 +3717 3.78708 -0.004601100 -0.007217462 +3718 3.78805 -0.004594141 -0.007205168 +3719 3.78901 -0.004587193 -0.007192894 +3720 3.78998 -0.004580258 -0.007180641 +3721 3.79094 -0.004573334 -0.007168408 +3722 3.79191 -0.004566422 -0.007156196 +3723 3.79287 -0.004559522 -0.007144004 +3724 3.79384 -0.004552633 -0.007131833 +3725 3.79480 -0.004545757 -0.007119683 +3726 3.79577 -0.004538892 -0.007107553 +3727 3.79673 -0.004532038 -0.007095443 +3728 3.79770 -0.004525197 -0.007083354 +3729 3.79866 -0.004518367 -0.007071285 +3730 3.79963 -0.004511549 -0.007059236 +3731 3.80060 -0.004504742 -0.007047208 +3732 3.80156 -0.004497947 -0.007035201 +3733 3.80253 -0.004491163 -0.007023214 +3734 3.80349 -0.004484391 -0.007011247 +3735 3.80446 -0.004477631 -0.006999300 +3736 3.80542 -0.004470882 -0.006987374 +3737 3.80639 -0.004464145 -0.006975468 +3738 3.80735 -0.004457419 -0.006963583 +3739 3.80832 -0.004450704 -0.006951718 +3740 3.80928 -0.004444001 -0.006939873 +3741 3.81025 -0.004437310 -0.006928048 +3742 3.81121 -0.004430629 -0.006916243 +3743 3.81218 -0.004423961 -0.006904459 +3744 3.81314 -0.004417303 -0.006892695 +3745 3.81411 -0.004410657 -0.006880951 +3746 3.81507 -0.004404022 -0.006869227 +3747 3.81604 -0.004397399 -0.006857524 +3748 3.81700 -0.004390786 -0.006845840 +3749 3.81797 -0.004384186 -0.006834177 +3750 3.81893 -0.004377596 -0.006822534 +3751 3.81990 -0.004371017 -0.006810911 +3752 3.82086 -0.004364450 -0.006799308 +3753 3.82183 -0.004357894 -0.006787725 +3754 3.82279 -0.004351349 -0.006776162 +3755 3.82376 -0.004344815 -0.006764619 +3756 3.82472 -0.004338293 -0.006753095 +3757 3.82569 -0.004331781 -0.006741592 +3758 3.82665 -0.004325281 -0.006730109 +3759 3.82762 -0.004318791 -0.006718646 +3760 3.82858 -0.004312313 -0.006707203 +3761 3.82955 -0.004305846 -0.006695779 +3762 3.83051 -0.004299390 -0.006684376 +3763 3.83148 -0.004292944 -0.006672992 +3764 3.83244 -0.004286510 -0.006661628 +3765 3.83341 -0.004280087 -0.006650284 +3766 3.83437 -0.004273674 -0.006638960 +3767 3.83534 -0.004267273 -0.006627655 +3768 3.83630 -0.004260882 -0.006616370 +3769 3.83727 -0.004254503 -0.006605105 +3770 3.83823 -0.004248134 -0.006593860 +3771 3.83920 -0.004241776 -0.006582634 +3772 3.84016 -0.004235429 -0.006571428 +3773 3.84113 -0.004229092 -0.006560242 +3774 3.84209 -0.004222767 -0.006549075 +3775 3.84306 -0.004216452 -0.006537928 +3776 3.84402 -0.004210148 -0.006526800 +3777 3.84499 -0.004203855 -0.006515692 +3778 3.84595 -0.004197572 -0.006504603 +3779 3.84692 -0.004191300 -0.006493534 +3780 3.84788 -0.004185039 -0.006482485 +3781 3.84885 -0.004178788 -0.006471454 +3782 3.84981 -0.004172548 -0.006460444 +3783 3.85078 -0.004166319 -0.006449452 +3784 3.85174 -0.004160100 -0.006438481 +3785 3.85271 -0.004153892 -0.006427528 +3786 3.85367 -0.004147695 -0.006416595 +3787 3.85464 -0.004141508 -0.006405681 +3788 3.85560 -0.004135331 -0.006394786 +3789 3.85657 -0.004129165 -0.006383911 +3790 3.85753 -0.004123009 -0.006373055 +3791 3.85850 -0.004116864 -0.006362218 +3792 3.85946 -0.004110730 -0.006351401 +3793 3.86043 -0.004104606 -0.006340602 +3794 3.86139 -0.004098492 -0.006329823 +3795 3.86236 -0.004092388 -0.006319063 +3796 3.86332 -0.004086295 -0.006308322 +3797 3.86429 -0.004080213 -0.006297600 +3798 3.86525 -0.004074140 -0.006286897 +3799 3.86622 -0.004068078 -0.006276213 +3800 3.86718 -0.004062027 -0.006265548 +3801 3.86815 -0.004055985 -0.006254902 +3802 3.86911 -0.004049954 -0.006244275 +3803 3.87008 -0.004043933 -0.006233667 +3804 3.87104 -0.004037923 -0.006223078 +3805 3.87201 -0.004031922 -0.006212508 +3806 3.87297 -0.004025932 -0.006201956 +3807 3.87394 -0.004019952 -0.006191424 +3808 3.87490 -0.004013982 -0.006180910 +3809 3.87587 -0.004008022 -0.006170415 +3810 3.87683 -0.004002072 -0.006159939 +3811 3.87780 -0.003996133 -0.006149482 +3812 3.87876 -0.003990203 -0.006139043 +3813 3.87973 -0.003984284 -0.006128623 +3814 3.88069 -0.003978374 -0.006118222 +3815 3.88166 -0.003972475 -0.006107839 +3816 3.88262 -0.003966586 -0.006097475 +3817 3.88359 -0.003960706 -0.006087129 +3818 3.88455 -0.003954837 -0.006076802 +3819 3.88552 -0.003948977 -0.006066494 +3820 3.88648 -0.003943128 -0.006056204 +3821 3.88745 -0.003937288 -0.006045932 +3822 3.88841 -0.003931459 -0.006035679 +3823 3.88938 -0.003925639 -0.006025445 +3824 3.89034 -0.003919829 -0.006015229 +3825 3.89131 -0.003914029 -0.006005031 +3826 3.89227 -0.003908239 -0.005994851 +3827 3.89324 -0.003902458 -0.005984690 +3828 3.89420 -0.003896688 -0.005974547 +3829 3.89517 -0.003890927 -0.005964423 +3830 3.89613 -0.003885176 -0.005954317 +3831 3.89710 -0.003879434 -0.005944228 +3832 3.89806 -0.003873703 -0.005934159 +3833 3.89903 -0.003867981 -0.005924107 +3834 3.89999 -0.003862269 -0.005914073 +3835 3.90096 -0.003856566 -0.005904058 +3836 3.90193 -0.003850873 -0.005894060 +3837 3.90289 -0.003845190 -0.005884081 +3838 3.90386 -0.003839516 -0.005874120 +3839 3.90482 -0.003833852 -0.005864176 +3840 3.90579 -0.003828198 -0.005854251 +3841 3.90675 -0.003822553 -0.005844344 +3842 3.90772 -0.003816918 -0.005834454 +3843 3.90868 -0.003811292 -0.005824583 +3844 3.90965 -0.003805676 -0.005814729 +3845 3.91061 -0.003800069 -0.005804893 +3846 3.91158 -0.003794472 -0.005795075 +3847 3.91254 -0.003788884 -0.005785275 +3848 3.91351 -0.003783306 -0.005775492 +3849 3.91447 -0.003777737 -0.005765727 +3850 3.91544 -0.003772177 -0.005755980 +3851 3.91640 -0.003766627 -0.005746251 +3852 3.91737 -0.003761086 -0.005736539 +3853 3.91833 -0.003755555 -0.005726845 +3854 3.91930 -0.003750033 -0.005717169 +3855 3.92026 -0.003744520 -0.005707510 +3856 3.92123 -0.003739017 -0.005697868 +3857 3.92219 -0.003733523 -0.005688245 +3858 3.92316 -0.003728038 -0.005678638 +3859 3.92412 -0.003722563 -0.005669049 +3860 3.92509 -0.003717096 -0.005659478 +3861 3.92605 -0.003711639 -0.005649924 +3862 3.92702 -0.003706191 -0.005640387 +3863 3.92798 -0.003700753 -0.005630868 +3864 3.92895 -0.003695323 -0.005621366 +3865 3.92991 -0.003689903 -0.005611881 +3866 3.93088 -0.003684492 -0.005602414 +3867 3.93184 -0.003679090 -0.005592964 +3868 3.93281 -0.003673697 -0.005583531 +3869 3.93377 -0.003668313 -0.005574115 +3870 3.93474 -0.003662938 -0.005564716 +3871 3.93570 -0.003657573 -0.005555335 +3872 3.93667 -0.003652216 -0.005545970 +3873 3.93763 -0.003646868 -0.005536623 +3874 3.93860 -0.003641530 -0.005527293 +3875 3.93956 -0.003636200 -0.005517980 +3876 3.94053 -0.003630879 -0.005508683 +3877 3.94149 -0.003625568 -0.005499404 +3878 3.94246 -0.003620265 -0.005490142 +3879 3.94342 -0.003614971 -0.005480896 +3880 3.94439 -0.003609686 -0.005471668 +3881 3.94535 -0.003604410 -0.005462456 +3882 3.94632 -0.003599143 -0.005453261 +3883 3.94728 -0.003593885 -0.005444083 +3884 3.94825 -0.003588636 -0.005434922 +3885 3.94921 -0.003583395 -0.005425777 +3886 3.95018 -0.003578163 -0.005416649 +3887 3.95114 -0.003572941 -0.005407538 +3888 3.95211 -0.003567726 -0.005398444 +3889 3.95307 -0.003562521 -0.005389366 +3890 3.95404 -0.003557324 -0.005380305 +3891 3.95500 -0.003552137 -0.005371260 +3892 3.95597 -0.003546957 -0.005362232 +3893 3.95693 -0.003541787 -0.005353220 +3894 3.95790 -0.003536625 -0.005344225 +3895 3.95886 -0.003531472 -0.005335247 +3896 3.95983 -0.003526328 -0.005326285 +3897 3.96079 -0.003521192 -0.005317339 +3898 3.96176 -0.003516065 -0.005308410 +3899 3.96272 -0.003510946 -0.005299496 +3900 3.96369 -0.003505836 -0.005290600 +3901 3.96465 -0.003500735 -0.005281719 +3902 3.96562 -0.003495642 -0.005272855 +3903 3.96658 -0.003490558 -0.005264007 +3904 3.96755 -0.003485482 -0.005255176 +3905 3.96851 -0.003480415 -0.005246360 +3906 3.96948 -0.003475356 -0.005237561 +3907 3.97044 -0.003470306 -0.005228778 +3908 3.97141 -0.003465264 -0.005220011 +3909 3.97237 -0.003460230 -0.005211260 +3910 3.97334 -0.003455205 -0.005202525 +3911 3.97430 -0.003450189 -0.005193806 +3912 3.97527 -0.003445181 -0.005185103 +3913 3.97623 -0.003440181 -0.005176416 +3914 3.97720 -0.003435190 -0.005167745 +3915 3.97816 -0.003430207 -0.005159090 +3916 3.97913 -0.003425232 -0.005150451 +3917 3.98009 -0.003420266 -0.005141828 +3918 3.98106 -0.003415308 -0.005133220 +3919 3.98202 -0.003410358 -0.005124628 +3920 3.98299 -0.003405417 -0.005116053 +3921 3.98395 -0.003400484 -0.005107492 +3922 3.98492 -0.003395559 -0.005098948 +3923 3.98588 -0.003390643 -0.005090419 +3924 3.98685 -0.003385734 -0.005081906 +3925 3.98781 -0.003380834 -0.005073409 +3926 3.98878 -0.003375942 -0.005064927 +3927 3.98974 -0.003371058 -0.005056461 +3928 3.99071 -0.003366183 -0.005048010 +3929 3.99167 -0.003361315 -0.005039575 +3930 3.99264 -0.003356456 -0.005031155 +3931 3.99360 -0.003351604 -0.005022751 +3932 3.99457 -0.003346761 -0.005014362 +3933 3.99553 -0.003341926 -0.005005989 +3934 3.99650 -0.003337099 -0.004997631 +3935 3.99746 -0.003332280 -0.004989288 +3936 3.99843 -0.003327469 -0.004980961 +3937 3.99939 -0.003322667 -0.004972649 +3938 4.00036 -0.003317872 -0.004964352 +3939 4.00133 -0.003313085 -0.004956071 +3940 4.00229 -0.003308306 -0.004947805 +3941 4.00326 -0.003303535 -0.004939554 +3942 4.00422 -0.003298772 -0.004931318 +3943 4.00519 -0.003294017 -0.004923098 +3944 4.00615 -0.003289270 -0.004914892 +3945 4.00712 -0.003284531 -0.004906702 +3946 4.00808 -0.003279800 -0.004898526 +3947 4.00905 -0.003275076 -0.004890366 +3948 4.01001 -0.003270361 -0.004882221 +3949 4.01098 -0.003265653 -0.004874090 +3950 4.01194 -0.003260953 -0.004865975 +3951 4.01291 -0.003256261 -0.004857874 +3952 4.01387 -0.003251577 -0.004849789 +3953 4.01484 -0.003246901 -0.004841718 +3954 4.01580 -0.003242232 -0.004833662 +3955 4.01677 -0.003237571 -0.004825621 +3956 4.01773 -0.003232918 -0.004817595 +3957 4.01870 -0.003228273 -0.004809584 +3958 4.01966 -0.003223635 -0.004801587 +3959 4.02063 -0.003219005 -0.004793605 +3960 4.02159 -0.003214383 -0.004785638 +3961 4.02256 -0.003209769 -0.004777685 +3962 4.02352 -0.003205162 -0.004769747 +3963 4.02449 -0.003200563 -0.004761824 +3964 4.02545 -0.003195971 -0.004753915 +3965 4.02642 -0.003191387 -0.004746021 +3966 4.02738 -0.003186811 -0.004738141 +3967 4.02835 -0.003182242 -0.004730276 +3968 4.02931 -0.003177681 -0.004722425 +3969 4.03028 -0.003173127 -0.004714589 +3970 4.03124 -0.003168581 -0.004706767 +3971 4.03221 -0.003164043 -0.004698959 +3972 4.03317 -0.003159512 -0.004691166 +3973 4.03414 -0.003154988 -0.004683387 +3974 4.03510 -0.003150472 -0.004675623 +3975 4.03607 -0.003145964 -0.004667873 +3976 4.03703 -0.003141463 -0.004660137 +3977 4.03800 -0.003136969 -0.004652415 +3978 4.03896 -0.003132483 -0.004644708 +3979 4.03993 -0.003128005 -0.004637014 +3980 4.04089 -0.003123533 -0.004629335 +3981 4.04186 -0.003119070 -0.004621670 +3982 4.04282 -0.003114613 -0.004614019 +3983 4.04379 -0.003110164 -0.004606383 +3984 4.04475 -0.003105722 -0.004598760 +3985 4.04572 -0.003101288 -0.004591151 +3986 4.04668 -0.003096861 -0.004583556 +3987 4.04765 -0.003092441 -0.004575976 +3988 4.04861 -0.003088029 -0.004568409 +3989 4.04958 -0.003083624 -0.004560856 +3990 4.05054 -0.003079226 -0.004553317 +3991 4.05151 -0.003074835 -0.004545792 +3992 4.05247 -0.003070452 -0.004538280 +3993 4.05344 -0.003066076 -0.004530783 +3994 4.05440 -0.003061707 -0.004523299 +3995 4.05537 -0.003057346 -0.004515829 +3996 4.05633 -0.003052991 -0.004508373 +3997 4.05730 -0.003048644 -0.004500931 +3998 4.05826 -0.003044304 -0.004493502 +3999 4.05923 -0.003039971 -0.004486087 +4000 4.06019 -0.003035646 -0.004478685 +4001 4.06116 -0.003031327 -0.004471298 +4002 4.06212 -0.003027016 -0.004463923 +4003 4.06309 -0.003022711 -0.004456563 +4004 4.06405 -0.003018414 -0.004449216 +4005 4.06502 -0.003014124 -0.004441882 +4006 4.06598 -0.003009841 -0.004434562 +4007 4.06695 -0.003005565 -0.004427255 +4008 4.06791 -0.003001296 -0.004419962 +4009 4.06888 -0.002997034 -0.004412682 +4010 4.06984 -0.002992779 -0.004405415 +4011 4.07081 -0.002988531 -0.004398162 +4012 4.07177 -0.002984290 -0.004390923 +4013 4.07274 -0.002980056 -0.004383696 +4014 4.07370 -0.002975829 -0.004376483 +4015 4.07467 -0.002971609 -0.004369283 +4016 4.07563 -0.002967396 -0.004362096 +4017 4.07660 -0.002963190 -0.004354923 +4018 4.07756 -0.002958990 -0.004347762 +4019 4.07853 -0.002954798 -0.004340615 +4020 4.07949 -0.002950612 -0.004333481 +4021 4.08046 -0.002946434 -0.004326360 +4022 4.08142 -0.002942262 -0.004319252 +4023 4.08239 -0.002938097 -0.004312158 +4024 4.08335 -0.002933939 -0.004305076 +4025 4.08432 -0.002929788 -0.004298007 +4026 4.08528 -0.002925644 -0.004290952 +4027 4.08625 -0.002921506 -0.004283909 +4028 4.08721 -0.002917375 -0.004276879 +4029 4.08818 -0.002913251 -0.004269862 +4030 4.08914 -0.002909134 -0.004262858 +4031 4.09011 -0.002905024 -0.004255867 +4032 4.09107 -0.002900920 -0.004248888 +4033 4.09204 -0.002896823 -0.004241923 +4034 4.09300 -0.002892733 -0.004234970 +4035 4.09397 -0.002888649 -0.004228030 +4036 4.09493 -0.002884572 -0.004221103 +4037 4.09590 -0.002880502 -0.004214188 +4038 4.09686 -0.002876438 -0.004207286 +4039 4.09783 -0.002872381 -0.004200397 +4040 4.09879 -0.002868331 -0.004193521 +4041 4.09976 -0.002864287 -0.004186657 +4042 4.10073 -0.002860250 -0.004179805 +4043 4.10169 -0.002856220 -0.004172967 +4044 4.10266 -0.002852196 -0.004166140 +4045 4.10362 -0.002848179 -0.004159327 +4046 4.10459 -0.002844168 -0.004152525 +4047 4.10555 -0.002840164 -0.004145737 +4048 4.10652 -0.002836167 -0.004138960 +4049 4.10748 -0.002832176 -0.004132197 +4050 4.10845 -0.002828191 -0.004125445 +4051 4.10941 -0.002824213 -0.004118706 +4052 4.11038 -0.002820242 -0.004111979 +4053 4.11134 -0.002816277 -0.004105265 +4054 4.11231 -0.002812318 -0.004098563 +4055 4.11327 -0.002808366 -0.004091873 +4056 4.11424 -0.002804420 -0.004085195 +4057 4.11520 -0.002800481 -0.004078530 +4058 4.11617 -0.002796548 -0.004071877 +4059 4.11713 -0.002792622 -0.004065236 +4060 4.11810 -0.002788702 -0.004058607 +4061 4.11906 -0.002784788 -0.004051991 +4062 4.12003 -0.002780881 -0.004045386 +4063 4.12099 -0.002776980 -0.004038794 +4064 4.12196 -0.002773086 -0.004032213 +4065 4.12292 -0.002769198 -0.004025645 +4066 4.12389 -0.002765316 -0.004019089 +4067 4.12485 -0.002761441 -0.004012545 +4068 4.12582 -0.002757572 -0.004006012 +4069 4.12678 -0.002753709 -0.003999492 +4070 4.12775 -0.002749852 -0.003992984 +4071 4.12871 -0.002746002 -0.003986487 +4072 4.12968 -0.002742158 -0.003980002 +4073 4.13064 -0.002738320 -0.003973530 +4074 4.13161 -0.002734489 -0.003967069 +4075 4.13257 -0.002730663 -0.003960620 +4076 4.13354 -0.002726844 -0.003954182 +4077 4.13450 -0.002723031 -0.003947757 +4078 4.13547 -0.002719225 -0.003941343 +4079 4.13643 -0.002715424 -0.003934941 +4080 4.13740 -0.002711630 -0.003928551 +4081 4.13836 -0.002707842 -0.003922172 +4082 4.13933 -0.002704060 -0.003915805 +4083 4.14029 -0.002700284 -0.003909450 +4084 4.14126 -0.002696514 -0.003903106 +4085 4.14222 -0.002692750 -0.003896774 +4086 4.14319 -0.002688993 -0.003890453 +4087 4.14415 -0.002685241 -0.003884144 +4088 4.14512 -0.002681496 -0.003877846 +4089 4.14608 -0.002677757 -0.003871560 +4090 4.14705 -0.002674024 -0.003865286 +4091 4.14801 -0.002670296 -0.003859023 +4092 4.14898 -0.002666575 -0.003852771 +4093 4.14994 -0.002662860 -0.003846531 +4094 4.15091 -0.002659151 -0.003840302 +4095 4.15187 -0.002655448 -0.003834084 +4096 4.15284 -0.002651751 -0.003827878 +4097 4.15380 -0.002648060 -0.003821683 +4098 4.15477 -0.002644375 -0.003815500 +4099 4.15573 -0.002640696 -0.003809327 +4100 4.15670 -0.002637022 -0.003803166 +4101 4.15766 -0.002633355 -0.003797017 +4102 4.15863 -0.002629694 -0.003790878 +4103 4.15959 -0.002626038 -0.003784751 +4104 4.16056 -0.002622389 -0.003778634 +4105 4.16152 -0.002618745 -0.003772529 +4106 4.16249 -0.002615107 -0.003766435 +4107 4.16345 -0.002611476 -0.003760353 +4108 4.16442 -0.002607850 -0.003754281 +4109 4.16538 -0.002604229 -0.003748220 +4110 4.16635 -0.002600615 -0.003742170 +4111 4.16731 -0.002597007 -0.003736132 +4112 4.16828 -0.002593404 -0.003730104 +4113 4.16924 -0.002589807 -0.003724087 +4114 4.17021 -0.002586216 -0.003718082 +4115 4.17117 -0.002582631 -0.003712087 +4116 4.17214 -0.002579052 -0.003706103 +4117 4.17310 -0.002575478 -0.003700130 +4118 4.17407 -0.002571910 -0.003694168 +4119 4.17503 -0.002568348 -0.003688217 +4120 4.17600 -0.002564791 -0.003682276 +4121 4.17696 -0.002561241 -0.003676346 +4122 4.17793 -0.002557696 -0.003670428 +4123 4.17889 -0.002554156 -0.003664519 +4124 4.17986 -0.002550623 -0.003658622 +4125 4.18082 -0.002547095 -0.003652735 +4126 4.18179 -0.002543573 -0.003646859 +4127 4.18275 -0.002540056 -0.003640994 +4128 4.18372 -0.002536545 -0.003635140 +4129 4.18468 -0.002533040 -0.003629296 +4130 4.18565 -0.002529540 -0.003623462 +4131 4.18661 -0.002526046 -0.003617639 +4132 4.18758 -0.002522558 -0.003611827 +4133 4.18854 -0.002519075 -0.003606026 +4134 4.18951 -0.002515598 -0.003600234 +4135 4.19047 -0.002512126 -0.003594454 +4136 4.19144 -0.002508660 -0.003588684 +4137 4.19240 -0.002505200 -0.003582924 +4138 4.19337 -0.002501745 -0.003577175 +4139 4.19433 -0.002498295 -0.003571436 +4140 4.19530 -0.002494852 -0.003565708 +4141 4.19626 -0.002491413 -0.003559990 +4142 4.19723 -0.002487980 -0.003554282 +4143 4.19819 -0.002484553 -0.003548585 +4144 4.19916 -0.002481131 -0.003542898 +4145 4.20013 -0.002477715 -0.003537221 +4146 4.20109 -0.002474304 -0.003531555 +4147 4.20206 -0.002470899 -0.003525899 +4148 4.20302 -0.002467499 -0.003520253 +4149 4.20399 -0.002464104 -0.003514617 +4150 4.20495 -0.002460715 -0.003508992 +4151 4.20592 -0.002457332 -0.003503377 +4152 4.20688 -0.002453953 -0.003497772 +4153 4.20785 -0.002450581 -0.003492177 +4154 4.20881 -0.002447213 -0.003486592 +4155 4.20978 -0.002443851 -0.003481017 +4156 4.21074 -0.002440494 -0.003475453 +4157 4.21171 -0.002437143 -0.003469898 +4158 4.21267 -0.002433797 -0.003464354 +4159 4.21364 -0.002430457 -0.003458819 +4160 4.21460 -0.002427121 -0.003453295 +4161 4.21557 -0.002423791 -0.003447781 +4162 4.21653 -0.002420467 -0.003442276 +4163 4.21750 -0.002417147 -0.003436782 +4164 4.21846 -0.002413833 -0.003431297 +4165 4.21943 -0.002410525 -0.003425822 +4166 4.22039 -0.002407221 -0.003420357 +4167 4.22136 -0.002403923 -0.003414903 +4168 4.22232 -0.002400630 -0.003409457 +4169 4.22329 -0.002397343 -0.003404022 +4170 4.22425 -0.002394060 -0.003398597 +4171 4.22522 -0.002390783 -0.003393181 +4172 4.22618 -0.002387511 -0.003387775 +4173 4.22715 -0.002384244 -0.003382379 +4174 4.22811 -0.002380983 -0.003376993 +4175 4.22908 -0.002377726 -0.003371616 +4176 4.23004 -0.002374475 -0.003366249 +4177 4.23101 -0.002371229 -0.003360892 +4178 4.23197 -0.002367988 -0.003355544 +4179 4.23294 -0.002364753 -0.003350206 +4180 4.23390 -0.002361522 -0.003344878 +4181 4.23487 -0.002358297 -0.003339559 +4182 4.23583 -0.002355076 -0.003334250 +4183 4.23680 -0.002351861 -0.003328951 +4184 4.23776 -0.002348651 -0.003323660 +4185 4.23873 -0.002345446 -0.003318380 +4186 4.23969 -0.002342246 -0.003313109 +4187 4.24066 -0.002339052 -0.003307847 +4188 4.24162 -0.002335862 -0.003302595 +4189 4.24259 -0.002332677 -0.003297353 +4190 4.24355 -0.002329498 -0.003292120 +4191 4.24452 -0.002326323 -0.003286896 +4192 4.24548 -0.002323154 -0.003281682 +4193 4.24645 -0.002319989 -0.003276477 +4194 4.24741 -0.002316830 -0.003271281 +4195 4.24838 -0.002313675 -0.003266095 +4196 4.24934 -0.002310526 -0.003260918 +4197 4.25031 -0.002307381 -0.003255750 +4198 4.25127 -0.002304242 -0.003250592 +4199 4.25224 -0.002301107 -0.003245443 +4200 4.25320 -0.002297978 -0.003240303 +4201 4.25417 -0.002294853 -0.003235172 +4202 4.25513 -0.002291734 -0.003230051 +4203 4.25610 -0.002288619 -0.003224939 +4204 4.25706 -0.002285509 -0.003219836 +4205 4.25803 -0.002282404 -0.003214742 +4206 4.25899 -0.002279304 -0.003209657 +4207 4.25996 -0.002276209 -0.003204582 +4208 4.26092 -0.002273119 -0.003199515 +4209 4.26189 -0.002270034 -0.003194458 +4210 4.26285 -0.002266954 -0.003189409 +4211 4.26382 -0.002263878 -0.003184370 +4212 4.26478 -0.002260808 -0.003179340 +4213 4.26575 -0.002257742 -0.003174319 +4214 4.26671 -0.002254681 -0.003169306 +4215 4.26768 -0.002251625 -0.003164303 +4216 4.26864 -0.002248573 -0.003159309 +4217 4.26961 -0.002245527 -0.003154324 +4218 4.27057 -0.002242485 -0.003149347 +4219 4.27154 -0.002239448 -0.003144380 +4220 4.27250 -0.002236416 -0.003139421 +4221 4.27347 -0.002233389 -0.003134472 +4222 4.27443 -0.002230366 -0.003129531 +4223 4.27540 -0.002227349 -0.003124599 +4224 4.27636 -0.002224336 -0.003119675 +4225 4.27733 -0.002221327 -0.003114761 +4226 4.27829 -0.002218324 -0.003109856 +4227 4.27926 -0.002215325 -0.003104959 +4228 4.28022 -0.002212331 -0.003100071 +4229 4.28119 -0.002209342 -0.003095191 +4230 4.28215 -0.002206357 -0.003090321 +4231 4.28312 -0.002203377 -0.003085459 +4232 4.28408 -0.002200402 -0.003080606 +4233 4.28505 -0.002197431 -0.003075761 +4234 4.28601 -0.002194465 -0.003070925 +4235 4.28698 -0.002191504 -0.003066098 +4236 4.28794 -0.002188547 -0.003061279 +4237 4.28891 -0.002185595 -0.003056469 +4238 4.28987 -0.002182648 -0.003051668 +4239 4.29084 -0.002179705 -0.003046875 +4240 4.29180 -0.002176767 -0.003042091 +4241 4.29277 -0.002173834 -0.003037315 +4242 4.29373 -0.002170905 -0.003032548 +4243 4.29470 -0.002167981 -0.003027789 +4244 4.29566 -0.002165061 -0.003023039 +4245 4.29663 -0.002162146 -0.003018297 +4246 4.29759 -0.002159235 -0.003013563 +4247 4.29856 -0.002156330 -0.003008839 +4248 4.29952 -0.002153428 -0.003004122 +4249 4.30049 -0.002150531 -0.002999414 +4250 4.30146 -0.002147639 -0.002994714 +4251 4.30242 -0.002144751 -0.002990023 +4252 4.30339 -0.002141868 -0.002985340 +4253 4.30435 -0.002138989 -0.002980665 +4254 4.30532 -0.002136115 -0.002975999 +4255 4.30628 -0.002133245 -0.002971341 +4256 4.30725 -0.002130380 -0.002966691 +4257 4.30821 -0.002127519 -0.002962050 +4258 4.30918 -0.002124663 -0.002957416 +4259 4.31014 -0.002121811 -0.002952791 +4260 4.31111 -0.002118964 -0.002948175 +4261 4.31207 -0.002116121 -0.002943566 +4262 4.31304 -0.002113282 -0.002938966 +4263 4.31400 -0.002110448 -0.002934373 +4264 4.31497 -0.002107619 -0.002929789 +4265 4.31593 -0.002104794 -0.002925213 +4266 4.31690 -0.002101973 -0.002920646 +4267 4.31786 -0.002099156 -0.002916086 +4268 4.31883 -0.002096344 -0.002911534 +4269 4.31979 -0.002093537 -0.002906991 +4270 4.32076 -0.002090734 -0.002902455 +4271 4.32172 -0.002087935 -0.002897928 +4272 4.32269 -0.002085140 -0.002893409 +4273 4.32365 -0.002082350 -0.002888897 +4274 4.32462 -0.002079565 -0.002884394 +4275 4.32558 -0.002076783 -0.002879899 +4276 4.32655 -0.002074006 -0.002875411 +4277 4.32751 -0.002071233 -0.002870932 +4278 4.32848 -0.002068465 -0.002866460 +4279 4.32944 -0.002065701 -0.002861997 +4280 4.33041 -0.002062941 -0.002857541 +4281 4.33137 -0.002060185 -0.002853093 +4282 4.33234 -0.002057434 -0.002848653 +4283 4.33330 -0.002054687 -0.002844221 +4284 4.33427 -0.002051945 -0.002839797 +4285 4.33523 -0.002049206 -0.002835381 +4286 4.33620 -0.002046472 -0.002830972 +4287 4.33716 -0.002043742 -0.002826571 +4288 4.33813 -0.002041016 -0.002822178 +4289 4.33909 -0.002038295 -0.002817793 +4290 4.34006 -0.002035578 -0.002813416 +4291 4.34102 -0.002032865 -0.002809046 +4292 4.34199 -0.002030156 -0.002804684 +4293 4.34295 -0.002027452 -0.002800330 +4294 4.34392 -0.002024751 -0.002795983 +4295 4.34488 -0.002022055 -0.002791644 +4296 4.34585 -0.002019363 -0.002787313 +4297 4.34681 -0.002016675 -0.002782989 +4298 4.34778 -0.002013992 -0.002778673 +4299 4.34874 -0.002011312 -0.002774365 +4300 4.34971 -0.002008637 -0.002770064 +4301 4.35067 -0.002005966 -0.002765771 +4302 4.35164 -0.002003299 -0.002761485 +4303 4.35260 -0.002000636 -0.002757207 +4304 4.35357 -0.001997977 -0.002752937 +4305 4.35453 -0.001995322 -0.002748674 +4306 4.35550 -0.001992672 -0.002744418 +4307 4.35646 -0.001990025 -0.002740170 +4308 4.35743 -0.001987383 -0.002735930 +4309 4.35839 -0.001984745 -0.002731697 +4310 4.35936 -0.001982111 -0.002727471 +4311 4.36032 -0.001979480 -0.002723253 +4312 4.36129 -0.001976854 -0.002719042 +4313 4.36225 -0.001974232 -0.002714839 +4314 4.36322 -0.001971614 -0.002710643 +4315 4.36418 -0.001969001 -0.002706455 +4316 4.36515 -0.001966391 -0.002702274 +4317 4.36611 -0.001963785 -0.002698100 +4318 4.36708 -0.001961183 -0.002693933 +4319 4.36804 -0.001958585 -0.002689774 +4320 4.36901 -0.001955992 -0.002685622 +4321 4.36997 -0.001953402 -0.002681478 +4322 4.37094 -0.001950816 -0.002677341 +4323 4.37190 -0.001948234 -0.002673211 +4324 4.37287 -0.001945657 -0.002669088 +4325 4.37383 -0.001943083 -0.002664972 +4326 4.37480 -0.001940513 -0.002660864 +4327 4.37576 -0.001937947 -0.002656763 +4328 4.37673 -0.001935385 -0.002652669 +4329 4.37769 -0.001932827 -0.002648583 +4330 4.37866 -0.001930273 -0.002644503 +4331 4.37962 -0.001927723 -0.002640431 +4332 4.38059 -0.001925177 -0.002636366 +4333 4.38155 -0.001922635 -0.002632308 +4334 4.38252 -0.001920096 -0.002628257 +4335 4.38348 -0.001917562 -0.002624213 +4336 4.38445 -0.001915031 -0.002620176 +4337 4.38541 -0.001912505 -0.002616146 +4338 4.38638 -0.001909982 -0.002612124 +4339 4.38734 -0.001907463 -0.002608108 +4340 4.38831 -0.001904948 -0.002604100 +4341 4.38927 -0.001902437 -0.002600098 +4342 4.39024 -0.001899929 -0.002596104 +4343 4.39120 -0.001897426 -0.002592116 +4344 4.39217 -0.001894926 -0.002588135 +4345 4.39313 -0.001892431 -0.002584162 +4346 4.39410 -0.001889939 -0.002580195 +4347 4.39506 -0.001887451 -0.002576235 +4348 4.39603 -0.001884966 -0.002572283 +4349 4.39699 -0.001882486 -0.002568337 +4350 4.39796 -0.001880009 -0.002564398 +4351 4.39892 -0.001877536 -0.002560466 +4352 4.39989 -0.001875067 -0.002556540 +4353 4.40086 -0.001872602 -0.002552622 +4354 4.40182 -0.001870140 -0.002548711 +4355 4.40279 -0.001867683 -0.002544806 +4356 4.40375 -0.001865229 -0.002540908 +4357 4.40472 -0.001862778 -0.002537017 +4358 4.40568 -0.001860332 -0.002533133 +4359 4.40665 -0.001857889 -0.002529255 +4360 4.40761 -0.001855450 -0.002525384 +4361 4.40858 -0.001853015 -0.002521520 +4362 4.40954 -0.001850584 -0.002517663 +4363 4.41051 -0.001848156 -0.002513812 +4364 4.41147 -0.001845732 -0.002509969 +4365 4.41244 -0.001843311 -0.002506131 +4366 4.41340 -0.001840895 -0.002502301 +4367 4.41437 -0.001838482 -0.002498477 +4368 4.41533 -0.001836072 -0.002494660 +4369 4.41630 -0.001833667 -0.002490849 +4370 4.41726 -0.001831265 -0.002487046 +4371 4.41823 -0.001828866 -0.002483248 +4372 4.41919 -0.001826472 -0.002479458 +4373 4.42016 -0.001824081 -0.002475674 +4374 4.42112 -0.001821693 -0.002471896 +4375 4.42209 -0.001819310 -0.002468125 +4376 4.42305 -0.001816930 -0.002464361 +4377 4.42402 -0.001814553 -0.002460603 +4378 4.42498 -0.001812181 -0.002456852 +4379 4.42595 -0.001809811 -0.002453107 +4380 4.42691 -0.001807446 -0.002449369 +4381 4.42788 -0.001805084 -0.002445637 +4382 4.42884 -0.001802725 -0.002441912 +4383 4.42981 -0.001800371 -0.002438193 +4384 4.43077 -0.001798020 -0.002434481 +4385 4.43174 -0.001795672 -0.002430775 +4386 4.43270 -0.001793328 -0.002427076 +4387 4.43367 -0.001790987 -0.002423383 +4388 4.43463 -0.001788651 -0.002419696 +4389 4.43560 -0.001786317 -0.002416016 +4390 4.43656 -0.001783987 -0.002412342 +4391 4.43753 -0.001781661 -0.002408674 +4392 4.43849 -0.001779338 -0.002405013 +4393 4.43946 -0.001777019 -0.002401358 +4394 4.44042 -0.001774704 -0.002397710 +4395 4.44139 -0.001772391 -0.002394068 +4396 4.44235 -0.001770083 -0.002390432 +4397 4.44332 -0.001767778 -0.002386802 +4398 4.44428 -0.001765476 -0.002383179 +4399 4.44525 -0.001763178 -0.002379562 +4400 4.44621 -0.001760883 -0.002375951 +4401 4.44718 -0.001758592 -0.002372347 +4402 4.44814 -0.001756304 -0.002368749 +4403 4.44911 -0.001754020 -0.002365157 +4404 4.45007 -0.001751739 -0.002361571 +4405 4.45104 -0.001749462 -0.002357991 +4406 4.45200 -0.001747188 -0.002354418 +4407 4.45297 -0.001744918 -0.002350850 +4408 4.45393 -0.001742651 -0.002347289 +4409 4.45490 -0.001740387 -0.002343734 +4410 4.45586 -0.001738127 -0.002340186 +4411 4.45683 -0.001735871 -0.002336643 +4412 4.45779 -0.001733617 -0.002333106 +4413 4.45876 -0.001731367 -0.002329576 +4414 4.45972 -0.001729121 -0.002326052 +4415 4.46069 -0.001726878 -0.002322533 +4416 4.46165 -0.001724638 -0.002319021 +4417 4.46262 -0.001722402 -0.002315515 +4418 4.46358 -0.001720169 -0.002312015 +4419 4.46455 -0.001717940 -0.002308521 +4420 4.46551 -0.001715713 -0.002305033 +4421 4.46648 -0.001713491 -0.002301551 +4422 4.46744 -0.001711271 -0.002298074 +4423 4.46841 -0.001709055 -0.002294604 +4424 4.46937 -0.001706842 -0.002291140 +4425 4.47034 -0.001704633 -0.002287682 +4426 4.47130 -0.001702427 -0.002284230 +4427 4.47227 -0.001700224 -0.002280784 +4428 4.47323 -0.001698025 -0.002277343 +4429 4.47420 -0.001695829 -0.002273909 +4430 4.47516 -0.001693636 -0.002270481 +4431 4.47613 -0.001691446 -0.002267058 +4432 4.47709 -0.001689260 -0.002263641 +4433 4.47806 -0.001687077 -0.002260230 +4434 4.47902 -0.001684898 -0.002256826 +4435 4.47999 -0.001682722 -0.002253426 +4436 4.48095 -0.001680548 -0.002250033 +4437 4.48192 -0.001678379 -0.002246646 +4438 4.48288 -0.001676212 -0.002243264 +4439 4.48385 -0.001674049 -0.002239888 +4440 4.48481 -0.001671889 -0.002236518 +4441 4.48578 -0.001669732 -0.002233154 +4442 4.48674 -0.001667579 -0.002229795 +4443 4.48771 -0.001665429 -0.002226443 +4444 4.48867 -0.001663282 -0.002223096 +4445 4.48964 -0.001661138 -0.002219755 +4446 4.49060 -0.001658997 -0.002216419 +4447 4.49157 -0.001656860 -0.002213089 +4448 4.49253 -0.001654726 -0.002209765 +4449 4.49350 -0.001652595 -0.002206447 +4450 4.49446 -0.001650467 -0.002203134 +4451 4.49543 -0.001648343 -0.002199827 +4452 4.49639 -0.001646221 -0.002196526 +4453 4.49736 -0.001644103 -0.002193230 +4454 4.49832 -0.001641988 -0.002189940 +4455 4.49929 -0.001639876 -0.002186656 +4456 4.50026 -0.001637768 -0.002183377 +4457 4.50122 -0.001635662 -0.002180104 +4458 4.50219 -0.001633560 -0.002176836 +4459 4.50315 -0.001631461 -0.002173574 +4460 4.50412 -0.001629365 -0.002170318 +4461 4.50508 -0.001627272 -0.002167067 +4462 4.50605 -0.001625182 -0.002163821 +4463 4.50701 -0.001623095 -0.002160582 +4464 4.50798 -0.001621012 -0.002157347 +4465 4.50894 -0.001618931 -0.002154119 +4466 4.50991 -0.001616854 -0.002150895 +4467 4.51087 -0.001614780 -0.002147678 +4468 4.51184 -0.001612709 -0.002144465 +4469 4.51280 -0.001610641 -0.002141259 +4470 4.51377 -0.001608576 -0.002138058 +4471 4.51473 -0.001606514 -0.002134862 +4472 4.51570 -0.001604456 -0.002131671 +4473 4.51666 -0.001602400 -0.002128486 +4474 4.51763 -0.001600347 -0.002125307 +4475 4.51859 -0.001598298 -0.002122133 +4476 4.51956 -0.001596251 -0.002118964 +4477 4.52052 -0.001594208 -0.002115801 +4478 4.52149 -0.001592168 -0.002112643 +4479 4.52245 -0.001590131 -0.002109491 +4480 4.52342 -0.001588096 -0.002106344 +4481 4.52438 -0.001586065 -0.002103202 +4482 4.52535 -0.001584037 -0.002100065 +4483 4.52631 -0.001582012 -0.002096934 +4484 4.52728 -0.001579990 -0.002093809 +4485 4.52824 -0.001577970 -0.002090688 +4486 4.52921 -0.001575954 -0.002087573 +4487 4.53017 -0.001573941 -0.002084463 +4488 4.53114 -0.001571931 -0.002081359 +4489 4.53210 -0.001569924 -0.002078259 +4490 4.53307 -0.001567920 -0.002075165 +4491 4.53403 -0.001565919 -0.002072077 +4492 4.53500 -0.001563921 -0.002068993 +4493 4.53596 -0.001561925 -0.002065915 +4494 4.53693 -0.001559933 -0.002062842 +4495 4.53789 -0.001557944 -0.002059774 +4496 4.53886 -0.001555958 -0.002056711 +4497 4.53982 -0.001553974 -0.002053654 +4498 4.54079 -0.001551994 -0.002050602 +4499 4.54175 -0.001550016 -0.002047555 +4500 4.54272 -0.001548042 -0.002044513 +4501 4.54368 -0.001546070 -0.002041476 +4502 4.54465 -0.001544102 -0.002038445 +4503 4.54561 -0.001542136 -0.002035418 +4504 4.54658 -0.001540173 -0.002032397 +4505 4.54754 -0.001538213 -0.002029381 +4506 4.54851 -0.001536256 -0.002026370 +4507 4.54947 -0.001534302 -0.002023364 +4508 4.55044 -0.001532351 -0.002020363 +4509 4.55140 -0.001530403 -0.002017367 +4510 4.55237 -0.001528457 -0.002014376 +4511 4.55333 -0.001526515 -0.002011390 +4512 4.55430 -0.001524575 -0.002008410 +4513 4.55526 -0.001522638 -0.002005434 +4514 4.55623 -0.001520704 -0.002002464 +4515 4.55719 -0.001518773 -0.001999498 +4516 4.55816 -0.001516845 -0.001996538 +4517 4.55912 -0.001514920 -0.001993582 +4518 4.56009 -0.001512997 -0.001990631 +4519 4.56105 -0.001511078 -0.001987686 +4520 4.56202 -0.001509161 -0.001984745 +4521 4.56298 -0.001507247 -0.001981810 +4522 4.56395 -0.001505336 -0.001978879 +4523 4.56491 -0.001503427 -0.001975953 +4524 4.56588 -0.001501522 -0.001973033 +4525 4.56684 -0.001499619 -0.001970117 +4526 4.56781 -0.001497719 -0.001967206 +4527 4.56877 -0.001495822 -0.001964300 +4528 4.56974 -0.001493928 -0.001961399 +4529 4.57070 -0.001492037 -0.001958503 +4530 4.57167 -0.001490148 -0.001955611 +4531 4.57263 -0.001488262 -0.001952725 +4532 4.57360 -0.001486379 -0.001949843 +4533 4.57456 -0.001484499 -0.001946966 +4534 4.57553 -0.001482621 -0.001944094 +4535 4.57649 -0.001480747 -0.001941227 +4536 4.57746 -0.001478875 -0.001938365 +4537 4.57842 -0.001477005 -0.001935508 +4538 4.57939 -0.001475139 -0.001932655 +4539 4.58035 -0.001473275 -0.001929807 +4540 4.58132 -0.001471414 -0.001926964 +4541 4.58228 -0.001469556 -0.001924126 +4542 4.58325 -0.001467700 -0.001921293 +4543 4.58421 -0.001465848 -0.001918464 +4544 4.58518 -0.001463998 -0.001915640 +4545 4.58614 -0.001462150 -0.001912821 +4546 4.58711 -0.001460306 -0.001910006 +4547 4.58807 -0.001458464 -0.001907196 +4548 4.58904 -0.001456625 -0.001904391 +4549 4.59000 -0.001454788 -0.001901591 +4550 4.59097 -0.001452954 -0.001898795 +4551 4.59193 -0.001451123 -0.001896005 +4552 4.59290 -0.001449295 -0.001893218 +4553 4.59386 -0.001447469 -0.001890437 +4554 4.59483 -0.001445646 -0.001887660 +4555 4.59579 -0.001443826 -0.001884888 +4556 4.59676 -0.001442008 -0.001882120 +4557 4.59772 -0.001440193 -0.001879357 +4558 4.59869 -0.001438381 -0.001876599 +4559 4.59965 -0.001436571 -0.001873845 +4560 4.60062 -0.001434764 -0.001871096 +4561 4.60159 -0.001432960 -0.001868351 +4562 4.60255 -0.001431158 -0.001865611 +4563 4.60352 -0.001429359 -0.001862876 +4564 4.60448 -0.001427562 -0.001860145 +4565 4.60545 -0.001425769 -0.001857419 +4566 4.60641 -0.001423977 -0.001854698 +4567 4.60738 -0.001422189 -0.001851981 +4568 4.60834 -0.001420403 -0.001849268 +4569 4.60931 -0.001418620 -0.001846560 +4570 4.61027 -0.001416839 -0.001843857 +4571 4.61124 -0.001415061 -0.001841158 +4572 4.61220 -0.001413285 -0.001838463 +4573 4.61317 -0.001411512 -0.001835774 +4574 4.61413 -0.001409742 -0.001833088 +4575 4.61510 -0.001407974 -0.001830407 +4576 4.61606 -0.001406209 -0.001827731 +4577 4.61703 -0.001404447 -0.001825059 +4578 4.61799 -0.001402687 -0.001822391 +4579 4.61896 -0.001400929 -0.001819728 +4580 4.61992 -0.001399174 -0.001817070 +4581 4.62089 -0.001397422 -0.001814415 +4582 4.62185 -0.001395672 -0.001811766 +4583 4.62282 -0.001393925 -0.001809120 +4584 4.62378 -0.001392181 -0.001806479 +4585 4.62475 -0.001390438 -0.001803843 +4586 4.62571 -0.001388699 -0.001801211 +4587 4.62668 -0.001386962 -0.001798583 +4588 4.62764 -0.001385228 -0.001795959 +4589 4.62861 -0.001383496 -0.001793340 +4590 4.62957 -0.001381766 -0.001790726 +4591 4.63054 -0.001380039 -0.001788116 +4592 4.63150 -0.001378315 -0.001785510 +4593 4.63247 -0.001376593 -0.001782908 +4594 4.63343 -0.001374874 -0.001780311 +4595 4.63440 -0.001373157 -0.001777718 +4596 4.63536 -0.001371443 -0.001775129 +4597 4.63633 -0.001369731 -0.001772545 +4598 4.63729 -0.001368021 -0.001769965 +4599 4.63826 -0.001366315 -0.001767389 +4600 4.63922 -0.001364610 -0.001764817 +4601 4.64019 -0.001362908 -0.001762250 +4602 4.64115 -0.001361209 -0.001759687 +4603 4.64212 -0.001359512 -0.001757129 +4604 4.64308 -0.001357817 -0.001754574 +4605 4.64405 -0.001356125 -0.001752024 +4606 4.64501 -0.001354436 -0.001749478 +4607 4.64598 -0.001352749 -0.001746936 +4608 4.64694 -0.001351064 -0.001744399 +4609 4.64791 -0.001349382 -0.001741865 +4610 4.64887 -0.001347702 -0.001739336 +4611 4.64984 -0.001346025 -0.001736811 +4612 4.65080 -0.001344350 -0.001734291 +4613 4.65177 -0.001342677 -0.001731774 +4614 4.65273 -0.001341007 -0.001729262 +4615 4.65370 -0.001339340 -0.001726753 +4616 4.65466 -0.001337675 -0.001724249 +4617 4.65563 -0.001336012 -0.001721749 +4618 4.65659 -0.001334351 -0.001719254 +4619 4.65756 -0.001332694 -0.001716762 +4620 4.65852 -0.001331038 -0.001714274 +4621 4.65949 -0.001329385 -0.001711791 +4622 4.66045 -0.001327734 -0.001709312 +4623 4.66142 -0.001326086 -0.001706836 +4624 4.66238 -0.001324440 -0.001704365 +4625 4.66335 -0.001322796 -0.001701898 +4626 4.66431 -0.001321155 -0.001699435 +4627 4.66528 -0.001319516 -0.001696976 +4628 4.66624 -0.001317880 -0.001694522 +4629 4.66721 -0.001316245 -0.001692071 +4630 4.66817 -0.001314614 -0.001689624 +4631 4.66914 -0.001312984 -0.001687181 +4632 4.67010 -0.001311357 -0.001684743 +4633 4.67107 -0.001309733 -0.001682308 +4634 4.67203 -0.001308110 -0.001679878 +4635 4.67300 -0.001306490 -0.001677451 +4636 4.67396 -0.001304873 -0.001675028 +4637 4.67493 -0.001303257 -0.001672610 +4638 4.67589 -0.001301644 -0.001670195 +4639 4.67686 -0.001300034 -0.001667785 +4640 4.67782 -0.001298425 -0.001665378 +4641 4.67879 -0.001296819 -0.001662975 +4642 4.67975 -0.001295216 -0.001660577 +4643 4.68072 -0.001293614 -0.001658182 +4644 4.68168 -0.001292015 -0.001655791 +4645 4.68265 -0.001290418 -0.001653404 +4646 4.68361 -0.001288824 -0.001651021 +4647 4.68458 -0.001287232 -0.001648642 +4648 4.68554 -0.001285642 -0.001646267 +4649 4.68651 -0.001284054 -0.001643896 +4650 4.68747 -0.001282469 -0.001641528 +4651 4.68844 -0.001280886 -0.001639165 +4652 4.68940 -0.001279305 -0.001636805 +4653 4.69037 -0.001277727 -0.001634450 +4654 4.69133 -0.001276151 -0.001632098 +4655 4.69230 -0.001274577 -0.001629750 +4656 4.69326 -0.001273005 -0.001627406 +4657 4.69423 -0.001271436 -0.001625066 +4658 4.69519 -0.001269868 -0.001622729 +4659 4.69616 -0.001268304 -0.001620397 +4660 4.69712 -0.001266741 -0.001618068 +4661 4.69809 -0.001265181 -0.001615743 +4662 4.69905 -0.001263622 -0.001613422 +4663 4.70002 -0.001262067 -0.001611105 +4664 4.70099 -0.001260513 -0.001608792 +4665 4.70195 -0.001258961 -0.001606482 +4666 4.70292 -0.001257412 -0.001604176 +4667 4.70388 -0.001255865 -0.001601874 +4668 4.70485 -0.001254320 -0.001599576 +4669 4.70581 -0.001252778 -0.001597281 +4670 4.70678 -0.001251237 -0.001594990 +4671 4.70774 -0.001249699 -0.001592703 +4672 4.70871 -0.001248163 -0.001590420 +4673 4.70967 -0.001246630 -0.001588140 +4674 4.71064 -0.001245098 -0.001585864 +4675 4.71160 -0.001243569 -0.001583592 +4676 4.71257 -0.001242042 -0.001581324 +4677 4.71353 -0.001240517 -0.001579059 +4678 4.71450 -0.001238994 -0.001576798 +4679 4.71546 -0.001237473 -0.001574541 +4680 4.71643 -0.001235955 -0.001572287 +4681 4.71739 -0.001234439 -0.001570038 +4682 4.71836 -0.001232925 -0.001567791 +4683 4.71932 -0.001231413 -0.001565549 +4684 4.72029 -0.001229903 -0.001563310 +4685 4.72125 -0.001228395 -0.001561075 +4686 4.72222 -0.001226890 -0.001558843 +4687 4.72318 -0.001225387 -0.001556615 +4688 4.72415 -0.001223885 -0.001554391 +4689 4.72511 -0.001222386 -0.001552170 +4690 4.72608 -0.001220890 -0.001549953 +4691 4.72704 -0.001219395 -0.001547740 +4692 4.72801 -0.001217902 -0.001545530 +4693 4.72897 -0.001216412 -0.001543324 +4694 4.72994 -0.001214924 -0.001541121 +4695 4.73090 -0.001213437 -0.001538922 +4696 4.73187 -0.001211953 -0.001536727 +4697 4.73283 -0.001210471 -0.001534535 +4698 4.73380 -0.001208992 -0.001532346 +4699 4.73476 -0.001207514 -0.001530162 +4700 4.73573 -0.001206038 -0.001527981 +4701 4.73669 -0.001204565 -0.001525803 +4702 4.73766 -0.001203093 -0.001523629 +4703 4.73862 -0.001201624 -0.001521458 +4704 4.73959 -0.001200157 -0.001519291 +4705 4.74055 -0.001198692 -0.001517128 +4706 4.74152 -0.001197228 -0.001514968 +4707 4.74248 -0.001195767 -0.001512811 +4708 4.74345 -0.001194309 -0.001510658 +4709 4.74441 -0.001192852 -0.001508509 +4710 4.74538 -0.001191397 -0.001506363 +4711 4.74634 -0.001189944 -0.001504221 +4712 4.74731 -0.001188494 -0.001502082 +4713 4.74827 -0.001187045 -0.001499946 +4714 4.74924 -0.001185599 -0.001497814 +4715 4.75020 -0.001184154 -0.001495685 +4716 4.75117 -0.001182712 -0.001493560 +4717 4.75213 -0.001181272 -0.001491439 +4718 4.75310 -0.001179833 -0.001489320 +4719 4.75406 -0.001178397 -0.001487206 +4720 4.75503 -0.001176963 -0.001485094 +4721 4.75599 -0.001175531 -0.001482986 +4722 4.75696 -0.001174100 -0.001480882 +4723 4.75792 -0.001172672 -0.001478781 +4724 4.75889 -0.001171246 -0.001476683 +4725 4.75985 -0.001169822 -0.001474589 +4726 4.76082 -0.001168400 -0.001472498 +4727 4.76178 -0.001166980 -0.001470410 +4728 4.76275 -0.001165562 -0.001468326 +4729 4.76371 -0.001164146 -0.001466245 +4730 4.76468 -0.001162732 -0.001464168 +4731 4.76564 -0.001161320 -0.001462094 +4732 4.76661 -0.001159910 -0.001460023 +4733 4.76757 -0.001158502 -0.001457956 +4734 4.76854 -0.001157096 -0.001455892 +4735 4.76950 -0.001155692 -0.001453831 +4736 4.77047 -0.001154290 -0.001451774 +4737 4.77143 -0.001152890 -0.001449720 +4738 4.77240 -0.001151492 -0.001447669 +4739 4.77336 -0.001150096 -0.001445622 +4740 4.77433 -0.001148702 -0.001443578 +4741 4.77529 -0.001147310 -0.001441537 +4742 4.77626 -0.001145920 -0.001439500 +4743 4.77722 -0.001144531 -0.001437465 +4744 4.77819 -0.001143145 -0.001435435 +4745 4.77915 -0.001141761 -0.001433407 +4746 4.78012 -0.001140378 -0.001431383 +4747 4.78108 -0.001138998 -0.001429361 +4748 4.78205 -0.001137620 -0.001427344 +4749 4.78301 -0.001136243 -0.001425329 +4750 4.78398 -0.001134869 -0.001423318 +4751 4.78494 -0.001133496 -0.001421310 +4752 4.78591 -0.001132125 -0.001419305 +4753 4.78687 -0.001130757 -0.001417303 +4754 4.78784 -0.001129390 -0.001415305 +4755 4.78880 -0.001128025 -0.001413310 +4756 4.78977 -0.001126662 -0.001411318 +4757 4.79073 -0.001125301 -0.001409329 +4758 4.79170 -0.001123942 -0.001407344 +4759 4.79266 -0.001122585 -0.001405361 +4760 4.79363 -0.001121229 -0.001403382 +4761 4.79459 -0.001119876 -0.001401406 +4762 4.79556 -0.001118525 -0.001399433 +4763 4.79652 -0.001117175 -0.001397464 +4764 4.79749 -0.001115827 -0.001395497 +4765 4.79845 -0.001114482 -0.001393534 +4766 4.79942 -0.001113138 -0.001391574 +4767 4.80039 -0.001111796 -0.001389617 +4768 4.80135 -0.001110456 -0.001387663 +4769 4.80232 -0.001109117 -0.001385712 +4770 4.80328 -0.001107781 -0.001383765 +4771 4.80425 -0.001106447 -0.001381820 +4772 4.80521 -0.001105114 -0.001379879 +4773 4.80618 -0.001103783 -0.001377941 +4774 4.80714 -0.001102454 -0.001376006 +4775 4.80811 -0.001101127 -0.001374074 +4776 4.80907 -0.001099802 -0.001372145 +4777 4.81004 -0.001098479 -0.001370219 +4778 4.81100 -0.001097158 -0.001368296 +4779 4.81197 -0.001095838 -0.001366377 +4780 4.81293 -0.001094520 -0.001364460 +4781 4.81390 -0.001093205 -0.001362547 +4782 4.81486 -0.001091891 -0.001360636 +4783 4.81583 -0.001090578 -0.001358729 +4784 4.81679 -0.001089268 -0.001356824 +4785 4.81776 -0.001087960 -0.001354923 +4786 4.81872 -0.001086653 -0.001353025 +4787 4.81969 -0.001085348 -0.001351130 +4788 4.82065 -0.001084045 -0.001349238 +4789 4.82162 -0.001082744 -0.001347349 +4790 4.82258 -0.001081445 -0.001345462 +4791 4.82355 -0.001080147 -0.001343579 +4792 4.82451 -0.001078851 -0.001341699 +4793 4.82548 -0.001077557 -0.001339822 +4794 4.82644 -0.001076265 -0.001337948 +4795 4.82741 -0.001074975 -0.001336077 +4796 4.82837 -0.001073687 -0.001334209 +4797 4.82934 -0.001072400 -0.001332344 +4798 4.83030 -0.001071115 -0.001330482 +4799 4.83127 -0.001069832 -0.001328623 +4800 4.83223 -0.001068551 -0.001326767 +4801 4.83320 -0.001067271 -0.001324913 +4802 4.83416 -0.001065993 -0.001323063 +4803 4.83513 -0.001064718 -0.001321216 +4804 4.83609 -0.001063443 -0.001319371 +4805 4.83706 -0.001062171 -0.001317530 +4806 4.83802 -0.001060900 -0.001315692 +4807 4.83899 -0.001059632 -0.001313856 +4808 4.83995 -0.001058365 -0.001312023 +4809 4.84092 -0.001057099 -0.001310194 +4810 4.84188 -0.001055836 -0.001308367 +4811 4.84285 -0.001054574 -0.001306543 +4812 4.84381 -0.001053314 -0.001304722 +4813 4.84478 -0.001052056 -0.001302904 +4814 4.84574 -0.001050799 -0.001301089 +4815 4.84671 -0.001049545 -0.001299277 +4816 4.84767 -0.001048292 -0.001297467 +4817 4.84864 -0.001047040 -0.001295661 +4818 4.84960 -0.001045791 -0.001293857 +4819 4.85057 -0.001044543 -0.001292056 +4820 4.85153 -0.001043297 -0.001290258 +4821 4.85250 -0.001042053 -0.001288463 +4822 4.85346 -0.001040810 -0.001286671 +4823 4.85443 -0.001039569 -0.001284882 +4824 4.85539 -0.001038330 -0.001283095 +4825 4.85636 -0.001037093 -0.001281311 +4826 4.85732 -0.001035857 -0.001279531 +4827 4.85829 -0.001034623 -0.001277753 +4828 4.85925 -0.001033391 -0.001275977 +4829 4.86022 -0.001032160 -0.001274205 +4830 4.86118 -0.001030932 -0.001272435 +4831 4.86215 -0.001029705 -0.001270669 +4832 4.86311 -0.001028479 -0.001268905 +4833 4.86408 -0.001027255 -0.001267144 +4834 4.86504 -0.001026033 -0.001265385 +4835 4.86601 -0.001024813 -0.001263630 +4836 4.86697 -0.001023594 -0.001261877 +4837 4.86794 -0.001022378 -0.001260127 +4838 4.86890 -0.001021162 -0.001258380 +4839 4.86987 -0.001019949 -0.001256635 +4840 4.87083 -0.001018737 -0.001254893 +4841 4.87180 -0.001017527 -0.001253154 +4842 4.87276 -0.001016318 -0.001251418 +4843 4.87373 -0.001015111 -0.001249685 +4844 4.87469 -0.001013906 -0.001247954 +4845 4.87566 -0.001012703 -0.001246226 +4846 4.87662 -0.001011501 -0.001244501 +4847 4.87759 -0.001010301 -0.001242778 +4848 4.87855 -0.001009102 -0.001241058 +4849 4.87952 -0.001007905 -0.001239341 +4850 4.88048 -0.001006710 -0.001237627 +4851 4.88145 -0.001005517 -0.001235915 +4852 4.88241 -0.001004325 -0.001234206 +4853 4.88338 -0.001003134 -0.001232500 +4854 4.88434 -0.001001946 -0.001230797 +4855 4.88531 -0.001000759 -0.001229096 +4856 4.88627 -0.000999574 -0.001227398 +4857 4.88724 -0.000998390 -0.001225702 +4858 4.88820 -0.000997208 -0.001224009 +4859 4.88917 -0.000996027 -0.001222319 +4860 4.89013 -0.000994849 -0.001220632 +4861 4.89110 -0.000993671 -0.001218947 +4862 4.89206 -0.000992496 -0.001217265 +4863 4.89303 -0.000991322 -0.001215585 +4864 4.89399 -0.000990150 -0.001213908 +4865 4.89496 -0.000988979 -0.001212234 +4866 4.89592 -0.000987810 -0.001210563 +4867 4.89689 -0.000986643 -0.001208894 +4868 4.89785 -0.000985477 -0.001207227 +4869 4.89882 -0.000984312 -0.001205564 +4870 4.89978 -0.000983150 -0.001203903 +4871 4.90075 -0.000981989 -0.001202244 +4872 4.90172 -0.000980829 -0.001200588 +4873 4.90268 -0.000979672 -0.001198935 +4874 4.90365 -0.000978515 -0.001197284 +4875 4.90461 -0.000977361 -0.001195636 +4876 4.90558 -0.000976208 -0.001193991 +4877 4.90654 -0.000975056 -0.001192348 +4878 4.90751 -0.000973906 -0.001190708 +4879 4.90847 -0.000972758 -0.001189070 +4880 4.90944 -0.000971611 -0.001187435 +4881 4.91040 -0.000970466 -0.001185802 +4882 4.91137 -0.000969323 -0.001184172 +4883 4.91233 -0.000968181 -0.001182545 +4884 4.91330 -0.000967040 -0.001180920 +4885 4.91426 -0.000965901 -0.001179298 +4886 4.91523 -0.000964764 -0.001177678 +4887 4.91619 -0.000963628 -0.001176061 +4888 4.91716 -0.000962494 -0.001174446 +4889 4.91812 -0.000961361 -0.001172834 +4890 4.91909 -0.000960230 -0.001171224 +4891 4.92005 -0.000959101 -0.001169617 +4892 4.92102 -0.000957973 -0.001168012 +4893 4.92198 -0.000956847 -0.001166410 +4894 4.92295 -0.000955722 -0.001164811 +4895 4.92391 -0.000954598 -0.001163214 +4896 4.92488 -0.000953477 -0.001161619 +4897 4.92584 -0.000952356 -0.001160027 +4898 4.92681 -0.000951238 -0.001158437 +4899 4.92777 -0.000950120 -0.001156850 +4900 4.92874 -0.000949005 -0.001155266 +4901 4.92970 -0.000947891 -0.001153684 +4902 4.93067 -0.000946778 -0.001152104 +4903 4.93163 -0.000945667 -0.001150527 +4904 4.93260 -0.000944557 -0.001148952 +4905 4.93356 -0.000943449 -0.001147380 +4906 4.93453 -0.000942343 -0.001145810 +4907 4.93549 -0.000941238 -0.001144243 +4908 4.93646 -0.000940134 -0.001142678 +4909 4.93742 -0.000939032 -0.001141115 +4910 4.93839 -0.000937932 -0.001139555 +4911 4.93935 -0.000936833 -0.001137998 +4912 4.94032 -0.000935735 -0.001136443 +4913 4.94128 -0.000934639 -0.001134890 +4914 4.94225 -0.000933545 -0.001133340 +4915 4.94321 -0.000932452 -0.001131792 +4916 4.94418 -0.000931360 -0.001130246 +4917 4.94514 -0.000930270 -0.001128703 +4918 4.94611 -0.000929182 -0.001127163 +4919 4.94707 -0.000928095 -0.001125624 +4920 4.94804 -0.000927009 -0.001124089 +4921 4.94900 -0.000925925 -0.001122555 +4922 4.94997 -0.000924843 -0.001121024 +4923 4.95093 -0.000923762 -0.001119495 +4924 4.95190 -0.000922682 -0.001117969 +4925 4.95286 -0.000921604 -0.001116445 +4926 4.95383 -0.000920527 -0.001114924 +4927 4.95479 -0.000919452 -0.001113405 +4928 4.95576 -0.000918378 -0.001111888 +4929 4.95672 -0.000917306 -0.001110373 +4930 4.95769 -0.000916235 -0.001108861 +4931 4.95865 -0.000915166 -0.001107352 +4932 4.95962 -0.000914098 -0.001105844 +4933 4.96058 -0.000913031 -0.001104339 +4934 4.96155 -0.000911966 -0.001102836 +4935 4.96251 -0.000910903 -0.001101336 +4936 4.96348 -0.000909841 -0.001099838 +4937 4.96444 -0.000908780 -0.001098342 +4938 4.96541 -0.000907721 -0.001096849 +4939 4.96637 -0.000906663 -0.001095358 +4940 4.96734 -0.000905607 -0.001093869 +4941 4.96830 -0.000904552 -0.001092383 +4942 4.96927 -0.000903498 -0.001090899 +4943 4.97023 -0.000902446 -0.001089417 +4944 4.97120 -0.000901395 -0.001087937 +4945 4.97216 -0.000900346 -0.001086460 +4946 4.97313 -0.000899299 -0.001084985 +4947 4.97409 -0.000898252 -0.001083513 +4948 4.97506 -0.000897207 -0.001082042 +4949 4.97602 -0.000896164 -0.001080574 +4950 4.97699 -0.000895122 -0.001079108 +4951 4.97795 -0.000894081 -0.001077645 +4952 4.97892 -0.000893042 -0.001076184 +4953 4.97988 -0.000892004 -0.001074725 +4954 4.98085 -0.000890967 -0.001073268 +4955 4.98181 -0.000889932 -0.001071813 +4956 4.98278 -0.000888899 -0.001070361 +4957 4.98374 -0.000887866 -0.001068911 +4958 4.98471 -0.000886836 -0.001067463 +4959 4.98567 -0.000885806 -0.001066018 +4960 4.98664 -0.000884778 -0.001064575 +4961 4.98760 -0.000883751 -0.001063134 +4962 4.98857 -0.000882726 -0.001061695 +4963 4.98953 -0.000881702 -0.001060258 +4964 4.99050 -0.000880680 -0.001058824 +4965 4.99146 -0.000879659 -0.001057392 +4966 4.99243 -0.000878639 -0.001055962 +4967 4.99339 -0.000877620 -0.001054534 +4968 4.99436 -0.000876603 -0.001053109 +4969 4.99532 -0.000875588 -0.001051685 +4970 4.99629 -0.000874574 -0.001050264 +4971 4.99725 -0.000873561 -0.001048845 +4972 4.99822 -0.000872549 -0.001047429 +4973 4.99918 -0.000871539 -0.001046014 +4974 5.00015 -0.000870530 -0.001044602 +4975 5.00112 -0.000869523 -0.001043192 +4976 5.00208 -0.000868517 -0.001041784 +4977 5.00305 -0.000867512 -0.001040378 +4978 5.00401 -0.000866509 -0.001038974 +4979 5.00498 -0.000865507 -0.001037573 +4980 5.00594 -0.000864506 -0.001036173 +4981 5.00691 -0.000863507 -0.001034776 +4982 5.00787 -0.000862509 -0.001033381 +4983 5.00884 -0.000861512 -0.001031988 +4984 5.00980 -0.000860517 -0.001030598 +4985 5.01077 -0.000859523 -0.001029209 +4986 5.01173 -0.000858531 -0.001027822 +4987 5.01270 -0.000857539 -0.001026438 +4988 5.01366 -0.000856550 -0.001025056 +4989 5.01463 -0.000855561 -0.001023676 +4990 5.01559 -0.000854574 -0.001022298 +4991 5.01656 -0.000853588 -0.001020922 +4992 5.01752 -0.000852603 -0.001019548 +4993 5.01849 -0.000851620 -0.001018177 +4994 5.01945 -0.000850638 -0.001016807 +4995 5.02042 -0.000849657 -0.001015440 +4996 5.02138 -0.000848678 -0.001014075 +4997 5.02235 -0.000847700 -0.001012712 +4998 5.02331 -0.000846724 -0.001011350 +4999 5.02428 -0.000845748 -0.001009991 +5000 5.02524 -0.000844774 -0.001008635 +5001 5.02621 -0.000843801 -0.001007280 +5002 5.02717 -0.000842830 -0.001005927 +5003 5.02814 -0.000841860 -0.001004576 +5004 5.02910 -0.000840891 -0.001003228 +5005 5.03007 -0.000839924 -0.001001881 +5006 5.03103 -0.000838957 -0.001000537 +5007 5.03200 -0.000837992 -0.000999194 +5008 5.03296 -0.000837029 -0.000997854 +5009 5.03393 -0.000836066 -0.000996516 +5010 5.03489 -0.000835105 -0.000995179 +5011 5.03586 -0.000834146 -0.000993845 +5012 5.03682 -0.000833187 -0.000992513 +5013 5.03779 -0.000832230 -0.000991183 +5014 5.03875 -0.000831274 -0.000989855 +5015 5.03972 -0.000830320 -0.000988529 +5016 5.04068 -0.000829366 -0.000987205 +5017 5.04165 -0.000828414 -0.000985883 +5018 5.04261 -0.000827463 -0.000984563 +5019 5.04358 -0.000826514 -0.000983245 +5020 5.04454 -0.000825566 -0.000981929 +5021 5.04551 -0.000824619 -0.000980615 +5022 5.04647 -0.000823673 -0.000979303 +5023 5.04744 -0.000822728 -0.000977993 +5024 5.04840 -0.000821785 -0.000976685 +5025 5.04937 -0.000820843 -0.000975379 +5026 5.05033 -0.000819903 -0.000974076 +5027 5.05130 -0.000818963 -0.000972774 +5028 5.05226 -0.000818025 -0.000971474 +5029 5.05323 -0.000817088 -0.000970176 +5030 5.05419 -0.000816153 -0.000968880 +5031 5.05516 -0.000815218 -0.000967586 +5032 5.05612 -0.000814285 -0.000966294 +5033 5.05709 -0.000813353 -0.000965004 +5034 5.05805 -0.000812423 -0.000963716 +5035 5.05902 -0.000811493 -0.000962430 +5036 5.05998 -0.000810565 -0.000961145 +5037 5.06095 -0.000809638 -0.000959863 +5038 5.06191 -0.000808712 -0.000958583 +5039 5.06288 -0.000807788 -0.000957305 +5040 5.06384 -0.000806865 -0.000956028 +5041 5.06481 -0.000805943 -0.000954754 +5042 5.06577 -0.000805022 -0.000953482 +5043 5.06674 -0.000804102 -0.000952211 +5044 5.06770 -0.000803184 -0.000950943 +5045 5.06867 -0.000802267 -0.000949676 +5046 5.06963 -0.000801351 -0.000948411 +5047 5.07060 -0.000800436 -0.000947148 +5048 5.07156 -0.000799523 -0.000945887 +5049 5.07253 -0.000798611 -0.000944629 +5050 5.07349 -0.000797700 -0.000943371 +5051 5.07446 -0.000796790 -0.000942116 +5052 5.07542 -0.000795881 -0.000940863 +5053 5.07639 -0.000794974 -0.000939612 +5054 5.07735 -0.000794068 -0.000938362 +5055 5.07832 -0.000793163 -0.000937115 +5056 5.07928 -0.000792259 -0.000935869 +5057 5.08025 -0.000791357 -0.000934626 +5058 5.08121 -0.000790455 -0.000933384 +5059 5.08218 -0.000789555 -0.000932144 +5060 5.08314 -0.000788656 -0.000930906 +5061 5.08411 -0.000787758 -0.000929670 +5062 5.08507 -0.000786862 -0.000928435 +5063 5.08604 -0.000785966 -0.000927203 +5064 5.08700 -0.000785072 -0.000925972 +5065 5.08797 -0.000784179 -0.000924743 +5066 5.08893 -0.000783287 -0.000923517 +5067 5.08990 -0.000782397 -0.000922292 +5068 5.09086 -0.000781507 -0.000921068 +5069 5.09183 -0.000780619 -0.000919847 +5070 5.09279 -0.000779732 -0.000918628 +5071 5.09376 -0.000778846 -0.000917410 +5072 5.09472 -0.000777961 -0.000916194 +5073 5.09569 -0.000777077 -0.000914981 +5074 5.09665 -0.000776195 -0.000913769 +5075 5.09762 -0.000775314 -0.000912558 +5076 5.09858 -0.000774434 -0.000911350 +5077 5.09955 -0.000773555 -0.000910143 +5078 5.10052 -0.000772677 -0.000908939 +5079 5.10148 -0.000771800 -0.000907736 +5080 5.10245 -0.000770925 -0.000906535 +5081 5.10341 -0.000770051 -0.000905335 +5082 5.10438 -0.000769178 -0.000904138 +5083 5.10534 -0.000768306 -0.000902942 +5084 5.10631 -0.000767435 -0.000901748 +5085 5.10727 -0.000766565 -0.000900556 +5086 5.10824 -0.000765697 -0.000899366 +5087 5.10920 -0.000764829 -0.000898178 +5088 5.11017 -0.000763963 -0.000896991 +5089 5.11113 -0.000763098 -0.000895806 +5090 5.11210 -0.000762234 -0.000894623 +5091 5.11306 -0.000761371 -0.000893442 +5092 5.11403 -0.000760510 -0.000892262 +5093 5.11499 -0.000759649 -0.000891084 +5094 5.11596 -0.000758790 -0.000889909 +5095 5.11692 -0.000757932 -0.000888734 +5096 5.11789 -0.000757075 -0.000887562 +5097 5.11885 -0.000756219 -0.000886391 +5098 5.11982 -0.000755364 -0.000885222 +5099 5.12078 -0.000754510 -0.000884055 +5100 5.12175 -0.000753657 -0.000882890 +5101 5.12271 -0.000752806 -0.000881726 +5102 5.12368 -0.000751956 -0.000880564 +5103 5.12464 -0.000751106 -0.000879404 +5104 5.12561 -0.000750258 -0.000878246 +5105 5.12657 -0.000749411 -0.000877089 +5106 5.12754 -0.000748565 -0.000875935 +5107 5.12850 -0.000747721 -0.000874781 +5108 5.12947 -0.000746877 -0.000873630 +5109 5.13043 -0.000746034 -0.000872480 +5110 5.13140 -0.000745193 -0.000871332 +5111 5.13236 -0.000744353 -0.000870186 +5112 5.13333 -0.000743513 -0.000869042 +5113 5.13429 -0.000742675 -0.000867899 +5114 5.13526 -0.000741838 -0.000866758 +5115 5.13622 -0.000741002 -0.000865619 +5116 5.13719 -0.000740168 -0.000864481 +5117 5.13815 -0.000739334 -0.000863345 +5118 5.13912 -0.000738501 -0.000862211 +5119 5.14008 -0.000737670 -0.000861078 +5120 5.14105 -0.000736839 -0.000859947 +5121 5.14201 -0.000736010 -0.000858818 +5122 5.14298 -0.000735182 -0.000857691 +5123 5.14394 -0.000734354 -0.000856565 +5124 5.14491 -0.000733528 -0.000855441 +5125 5.14587 -0.000732703 -0.000854319 +5126 5.14684 -0.000731879 -0.000853198 +5127 5.14780 -0.000731057 -0.000852079 +5128 5.14877 -0.000730235 -0.000850962 +5129 5.14973 -0.000729414 -0.000849846 +5130 5.15070 -0.000728595 -0.000848732 +5131 5.15166 -0.000727776 -0.000847620 +5132 5.15263 -0.000726959 -0.000846509 +5133 5.15359 -0.000726142 -0.000845400 +5134 5.15456 -0.000725327 -0.000844293 +5135 5.15552 -0.000724513 -0.000843187 +5136 5.15649 -0.000723699 -0.000842083 +5137 5.15745 -0.000722887 -0.000840981 +5138 5.15842 -0.000722076 -0.000839880 +5139 5.15938 -0.000721266 -0.000838781 +5140 5.16035 -0.000720457 -0.000837684 +5141 5.16131 -0.000719650 -0.000836588 +5142 5.16228 -0.000718843 -0.000835494 +5143 5.16324 -0.000718037 -0.000834401 +5144 5.16421 -0.000717232 -0.000833310 +5145 5.16517 -0.000716429 -0.000832221 +5146 5.16614 -0.000715626 -0.000831134 +5147 5.16710 -0.000714824 -0.000830048 +5148 5.16807 -0.000714024 -0.000828963 +5149 5.16903 -0.000713224 -0.000827881 +5150 5.17000 -0.000712426 -0.000826799 +5151 5.17096 -0.000711629 -0.000825720 +5152 5.17193 -0.000710832 -0.000824642 +5153 5.17289 -0.000710037 -0.000823566 +5154 5.17386 -0.000709243 -0.000822491 +5155 5.17482 -0.000708449 -0.000821418 +5156 5.17579 -0.000707657 -0.000820346 +5157 5.17675 -0.000706866 -0.000819276 +5158 5.17772 -0.000706076 -0.000818208 +5159 5.17868 -0.000705287 -0.000817141 +5160 5.17965 -0.000704499 -0.000816076 +5161 5.18061 -0.000703712 -0.000815013 +5162 5.18158 -0.000702926 -0.000813951 +5163 5.18254 -0.000702141 -0.000812891 +5164 5.18351 -0.000701357 -0.000811832 +5165 5.18447 -0.000700574 -0.000810775 +5166 5.18544 -0.000699792 -0.000809719 +5167 5.18640 -0.000699011 -0.000808665 +5168 5.18737 -0.000698231 -0.000807612 +5169 5.18833 -0.000697452 -0.000806561 +5170 5.18930 -0.000696674 -0.000805512 +5171 5.19026 -0.000695898 -0.000804464 +5172 5.19123 -0.000695122 -0.000803418 +5173 5.19219 -0.000694347 -0.000802373 +5174 5.19316 -0.000693573 -0.000801330 +5175 5.19412 -0.000692800 -0.000800288 +5176 5.19509 -0.000692028 -0.000799248 +5177 5.19605 -0.000691258 -0.000798210 +5178 5.19702 -0.000690488 -0.000797173 +5179 5.19798 -0.000689719 -0.000796137 +5180 5.19895 -0.000688951 -0.000795104 +5181 5.19991 -0.000688184 -0.000794071 +5182 5.20088 -0.000687419 -0.000793040 +5183 5.20185 -0.000686654 -0.000792011 +5184 5.20281 -0.000685890 -0.000790983 +5185 5.20378 -0.000685127 -0.000789957 +5186 5.20474 -0.000684365 -0.000788932 +5187 5.20571 -0.000683604 -0.000787909 +5188 5.20667 -0.000682844 -0.000786887 +5189 5.20764 -0.000682086 -0.000785867 +5190 5.20860 -0.000681328 -0.000784849 +5191 5.20957 -0.000680571 -0.000783831 +5192 5.21053 -0.000679815 -0.000782816 +5193 5.21150 -0.000679060 -0.000781802 +5194 5.21246 -0.000678306 -0.000780789 +5195 5.21343 -0.000677553 -0.000779778 +5196 5.21439 -0.000676801 -0.000778768 +5197 5.21536 -0.000676050 -0.000777760 +5198 5.21632 -0.000675300 -0.000776753 +5199 5.21729 -0.000674551 -0.000775748 +5200 5.21825 -0.000673802 -0.000774745 +5201 5.21922 -0.000673055 -0.000773742 +5202 5.22018 -0.000672309 -0.000772742 +5203 5.22115 -0.000671564 -0.000771742 +5204 5.22211 -0.000670819 -0.000770745 +5205 5.22308 -0.000670076 -0.000769748 +5206 5.22404 -0.000669334 -0.000768753 +5207 5.22501 -0.000668592 -0.000767760 +5208 5.22597 -0.000667852 -0.000766768 +5209 5.22694 -0.000667112 -0.000765778 +5210 5.22790 -0.000666374 -0.000764789 +5211 5.22887 -0.000665636 -0.000763801 +5212 5.22983 -0.000664900 -0.000762815 +5213 5.23080 -0.000664164 -0.000761831 +5214 5.23176 -0.000663429 -0.000760848 +5215 5.23273 -0.000662695 -0.000759866 +5216 5.23369 -0.000661963 -0.000758886 +5217 5.23466 -0.000661231 -0.000757907 +5218 5.23562 -0.000660500 -0.000756929 +5219 5.23659 -0.000659770 -0.000755954 +5220 5.23755 -0.000659041 -0.000754979 +5221 5.23852 -0.000658313 -0.000754006 +5222 5.23948 -0.000657585 -0.000753034 +5223 5.24045 -0.000656859 -0.000752064 +5224 5.24141 -0.000656134 -0.000751095 +5225 5.24238 -0.000655409 -0.000750128 +5226 5.24334 -0.000654686 -0.000749162 +5227 5.24431 -0.000653964 -0.000748198 +5228 5.24527 -0.000653242 -0.000747235 +5229 5.24624 -0.000652521 -0.000746273 +5230 5.24720 -0.000651802 -0.000745313 +5231 5.24817 -0.000651083 -0.000744354 +5232 5.24913 -0.000650365 -0.000743397 +5233 5.25010 -0.000649648 -0.000742441 +5234 5.25106 -0.000648932 -0.000741486 +5235 5.25203 -0.000648217 -0.000740533 +5236 5.25299 -0.000647503 -0.000739581 +5237 5.25396 -0.000646789 -0.000738631 +5238 5.25492 -0.000646077 -0.000737682 +5239 5.25589 -0.000645366 -0.000736734 +5240 5.25685 -0.000644655 -0.000735788 +5241 5.25782 -0.000643945 -0.000734843 +5242 5.25878 -0.000643237 -0.000733899 +5243 5.25975 -0.000642529 -0.000732957 +5244 5.26071 -0.000641822 -0.000732017 +5245 5.26168 -0.000641116 -0.000731077 +5246 5.26264 -0.000640411 -0.000730140 +5247 5.26361 -0.000639707 -0.000729203 +5248 5.26457 -0.000639004 -0.000728268 +5249 5.26554 -0.000638301 -0.000727334 +5250 5.26650 -0.000637600 -0.000726402 +5251 5.26747 -0.000636899 -0.000725471 +5252 5.26843 -0.000636199 -0.000724541 +5253 5.26940 -0.000635501 -0.000723612 +5254 5.27036 -0.000634803 -0.000722685 +5255 5.27133 -0.000634106 -0.000721760 +5256 5.27229 -0.000633410 -0.000720836 +5257 5.27326 -0.000632715 -0.000719913 +5258 5.27422 -0.000632020 -0.000718991 +5259 5.27519 -0.000631327 -0.000718071 +5260 5.27615 -0.000630634 -0.000717152 +5261 5.27712 -0.000629943 -0.000716234 +5262 5.27808 -0.000629252 -0.000715318 +5263 5.27905 -0.000628562 -0.000714403 +5264 5.28001 -0.000627873 -0.000713490 +5265 5.28098 -0.000627185 -0.000712578 +5266 5.28194 -0.000626498 -0.000711667 +5267 5.28291 -0.000625811 -0.000710757 +5268 5.28387 -0.000625126 -0.000709849 +5269 5.28484 -0.000624441 -0.000708942 +5270 5.28580 -0.000623758 -0.000708037 +5271 5.28677 -0.000623075 -0.000707133 +5272 5.28773 -0.000622393 -0.000706230 +5273 5.28870 -0.000621712 -0.000705328 +5274 5.28966 -0.000621031 -0.000704428 +5275 5.29063 -0.000620352 -0.000703529 +5276 5.29159 -0.000619673 -0.000702631 +5277 5.29256 -0.000618996 -0.000701735 +5278 5.29352 -0.000618319 -0.000700840 +5279 5.29449 -0.000617643 -0.000699946 +5280 5.29545 -0.000616968 -0.000699054 +5281 5.29642 -0.000616294 -0.000698163 +5282 5.29738 -0.000615621 -0.000697273 +5283 5.29835 -0.000614948 -0.000696384 +5284 5.29931 -0.000614276 -0.000695497 +5285 5.30028 -0.000613606 -0.000694611 +5286 5.30125 -0.000612936 -0.000693726 +5287 5.30221 -0.000612267 -0.000692843 +5288 5.30318 -0.000611599 -0.000691961 +5289 5.30414 -0.000610931 -0.000691080 +5290 5.30511 -0.000610265 -0.000690201 +5291 5.30607 -0.000609599 -0.000689322 +5292 5.30704 -0.000608934 -0.000688445 +5293 5.30800 -0.000608270 -0.000687570 +5294 5.30897 -0.000607607 -0.000686695 +5295 5.30993 -0.000606945 -0.000685822 +5296 5.31090 -0.000606283 -0.000684950 +5297 5.31186 -0.000605623 -0.000684080 +5298 5.31283 -0.000604963 -0.000683210 +5299 5.31379 -0.000604304 -0.000682342 +5300 5.31476 -0.000603646 -0.000681475 +5301 5.31572 -0.000602989 -0.000680610 +5302 5.31669 -0.000602333 -0.000679746 +5303 5.31765 -0.000601677 -0.000678883 +5304 5.31862 -0.000601022 -0.000678021 +5305 5.31958 -0.000600368 -0.000677160 +5306 5.32055 -0.000599715 -0.000676301 +5307 5.32151 -0.000599063 -0.000675443 +5308 5.32248 -0.000598412 -0.000674586 +5309 5.32344 -0.000597761 -0.000673730 +5310 5.32441 -0.000597111 -0.000672876 +5311 5.32537 -0.000596462 -0.000672023 +5312 5.32634 -0.000595814 -0.000671171 +5313 5.32730 -0.000595167 -0.000670321 +5314 5.32827 -0.000594520 -0.000669471 +5315 5.32923 -0.000593875 -0.000668623 +5316 5.33020 -0.000593230 -0.000667776 +5317 5.33116 -0.000592586 -0.000666930 +5318 5.33213 -0.000591943 -0.000666086 +5319 5.33309 -0.000591300 -0.000665243 +5320 5.33406 -0.000590659 -0.000664401 +5321 5.33502 -0.000590018 -0.000663560 +5322 5.33599 -0.000589378 -0.000662720 +5323 5.33695 -0.000588739 -0.000661882 +5324 5.33792 -0.000588100 -0.000661045 +5325 5.33888 -0.000587463 -0.000660209 +5326 5.33985 -0.000586826 -0.000659374 +5327 5.34081 -0.000586190 -0.000658540 +5328 5.34178 -0.000585555 -0.000657708 +5329 5.34274 -0.000584921 -0.000656877 +5330 5.34371 -0.000584287 -0.000656047 +5331 5.34467 -0.000583655 -0.000655218 +5332 5.34564 -0.000583023 -0.000654391 +5333 5.34660 -0.000582391 -0.000653564 +5334 5.34757 -0.000581761 -0.000652739 +5335 5.34853 -0.000581132 -0.000651915 +5336 5.34950 -0.000580503 -0.000651092 +5337 5.35046 -0.000579875 -0.000650271 +5338 5.35143 -0.000579248 -0.000649450 +5339 5.35239 -0.000578621 -0.000648631 +5340 5.35336 -0.000577996 -0.000647813 +5341 5.35432 -0.000577371 -0.000646996 +5342 5.35529 -0.000576747 -0.000646180 +5343 5.35625 -0.000576124 -0.000645366 +5344 5.35722 -0.000575501 -0.000644552 +5345 5.35818 -0.000574880 -0.000643740 +5346 5.35915 -0.000574259 -0.000642929 +5347 5.36011 -0.000573639 -0.000642119 +5348 5.36108 -0.000573020 -0.000641311 +5349 5.36204 -0.000572401 -0.000640503 +5350 5.36301 -0.000571783 -0.000639697 +5351 5.36397 -0.000571166 -0.000638892 +5352 5.36494 -0.000570550 -0.000638088 +5353 5.36590 -0.000569935 -0.000637285 +5354 5.36687 -0.000569320 -0.000636483 +5355 5.36783 -0.000568706 -0.000635682 +5356 5.36880 -0.000568093 -0.000634883 +5357 5.36976 -0.000567481 -0.000634085 +5358 5.37073 -0.000566870 -0.000633288 +5359 5.37169 -0.000566259 -0.000632492 +5360 5.37266 -0.000565649 -0.000631697 +5361 5.37362 -0.000565040 -0.000630903 +5362 5.37459 -0.000564431 -0.000630111 +5363 5.37555 -0.000563823 -0.000629319 +5364 5.37652 -0.000563216 -0.000628529 +5365 5.37748 -0.000562610 -0.000627740 +5366 5.37845 -0.000562005 -0.000626952 +5367 5.37941 -0.000561400 -0.000626165 +5368 5.38038 -0.000560796 -0.000625379 +5369 5.38134 -0.000560193 -0.000624594 +5370 5.38231 -0.000559591 -0.000623811 +5371 5.38327 -0.000558989 -0.000623029 +5372 5.38424 -0.000558388 -0.000622247 +5373 5.38520 -0.000557788 -0.000621467 +5374 5.38617 -0.000557189 -0.000620688 +5375 5.38713 -0.000556590 -0.000619910 +5376 5.38810 -0.000555992 -0.000619133 +5377 5.38906 -0.000555395 -0.000618358 +5378 5.39003 -0.000554799 -0.000617583 +5379 5.39099 -0.000554203 -0.000616810 +5380 5.39196 -0.000553608 -0.000616037 +5381 5.39292 -0.000553014 -0.000615266 +5382 5.39389 -0.000552421 -0.000614496 +5383 5.39485 -0.000551828 -0.000613727 +5384 5.39582 -0.000551236 -0.000612959 +5385 5.39678 -0.000550645 -0.000612192 +5386 5.39775 -0.000550055 -0.000611426 +5387 5.39871 -0.000549465 -0.000610662 +5388 5.39968 -0.000548876 -0.000609898 +5389 5.40065 -0.000548288 -0.000609136 +5390 5.40161 -0.000547700 -0.000608374 +5391 5.40258 -0.000547113 -0.000607614 +5392 5.40354 -0.000546527 -0.000606855 +5393 5.40451 -0.000545942 -0.000606097 +5394 5.40547 -0.000545358 -0.000605340 +5395 5.40644 -0.000544774 -0.000604584 +5396 5.40740 -0.000544191 -0.000603829 +5397 5.40837 -0.000543608 -0.000603075 +5398 5.40933 -0.000543027 -0.000602322 +5399 5.41030 -0.000542446 -0.000601571 +5400 5.41126 -0.000541866 -0.000600820 +5401 5.41223 -0.000541286 -0.000600070 +5402 5.41319 -0.000540707 -0.000599322 +5403 5.41416 -0.000540129 -0.000598575 +5404 5.41512 -0.000539552 -0.000597828 +5405 5.41609 -0.000538976 -0.000597083 +5406 5.41705 -0.000538400 -0.000596339 +5407 5.41802 -0.000537825 -0.000595596 +5408 5.41898 -0.000537250 -0.000594854 +5409 5.41995 -0.000536676 -0.000594113 +5410 5.42091 -0.000536103 -0.000593373 +5411 5.42188 -0.000535531 -0.000592634 +5412 5.42284 -0.000534960 -0.000591896 +5413 5.42381 -0.000534389 -0.000591159 +5414 5.42477 -0.000533819 -0.000590423 +5415 5.42574 -0.000533249 -0.000589689 +5416 5.42670 -0.000532680 -0.000588955 +5417 5.42767 -0.000532112 -0.000588222 +5418 5.42863 -0.000531545 -0.000587491 +5419 5.42960 -0.000530979 -0.000586760 +5420 5.43056 -0.000530413 -0.000586031 +5421 5.43153 -0.000529847 -0.000585302 +5422 5.43249 -0.000529283 -0.000584575 +5423 5.43346 -0.000528719 -0.000583848 +5424 5.43442 -0.000528156 -0.000583123 +5425 5.43539 -0.000527594 -0.000582399 +5426 5.43635 -0.000527032 -0.000581675 +5427 5.43732 -0.000526471 -0.000580953 +5428 5.43828 -0.000525911 -0.000580232 +5429 5.43925 -0.000525351 -0.000579511 +5430 5.44021 -0.000524792 -0.000578792 +5431 5.44118 -0.000524234 -0.000578074 +5432 5.44214 -0.000523676 -0.000577357 +5433 5.44311 -0.000523120 -0.000576641 +5434 5.44407 -0.000522563 -0.000575926 +5435 5.44504 -0.000522008 -0.000575211 +5436 5.44600 -0.000521453 -0.000574498 +5437 5.44697 -0.000520899 -0.000573786 +5438 5.44793 -0.000520346 -0.000573075 +5439 5.44890 -0.000519793 -0.000572365 +5440 5.44986 -0.000519241 -0.000571656 +5441 5.45083 -0.000518690 -0.000570948 +5442 5.45179 -0.000518139 -0.000570241 +5443 5.45276 -0.000517589 -0.000569535 +5444 5.45372 -0.000517040 -0.000568830 +5445 5.45469 -0.000516491 -0.000568125 +5446 5.45565 -0.000515943 -0.000567422 +5447 5.45662 -0.000515396 -0.000566720 +5448 5.45758 -0.000514849 -0.000566019 +5449 5.45855 -0.000514304 -0.000565319 +5450 5.45951 -0.000513758 -0.000564620 +5451 5.46048 -0.000513214 -0.000563922 +5452 5.46144 -0.000512670 -0.000563225 +5453 5.46241 -0.000512127 -0.000562528 +5454 5.46337 -0.000511584 -0.000561833 +5455 5.46434 -0.000511042 -0.000561139 +5456 5.46530 -0.000510501 -0.000560446 +5457 5.46627 -0.000509961 -0.000559754 +5458 5.46723 -0.000509421 -0.000559062 +5459 5.46820 -0.000508882 -0.000558372 +5460 5.46916 -0.000508343 -0.000557683 +5461 5.47013 -0.000507805 -0.000556994 +5462 5.47109 -0.000507268 -0.000556307 +5463 5.47206 -0.000506731 -0.000555621 +5464 5.47302 -0.000506196 -0.000554935 +5465 5.47399 -0.000505660 -0.000554251 +5466 5.47495 -0.000505126 -0.000553567 +5467 5.47592 -0.000504592 -0.000552885 +5468 5.47688 -0.000504059 -0.000552203 +5469 5.47785 -0.000503526 -0.000551522 +5470 5.47881 -0.000502994 -0.000550843 +5471 5.47978 -0.000502463 -0.000550164 +5472 5.48074 -0.000501932 -0.000549486 +5473 5.48171 -0.000501402 -0.000548810 +5474 5.48267 -0.000500873 -0.000548134 +5475 5.48364 -0.000500344 -0.000547459 +5476 5.48460 -0.000499816 -0.000546785 +5477 5.48557 -0.000499289 -0.000546112 +5478 5.48653 -0.000498762 -0.000545440 +5479 5.48750 -0.000498236 -0.000544769 +5480 5.48846 -0.000497711 -0.000544099 +5481 5.48943 -0.000497186 -0.000543429 +5482 5.49039 -0.000496662 -0.000542761 +5483 5.49136 -0.000496139 -0.000542094 +5484 5.49232 -0.000495616 -0.000541427 +5485 5.49329 -0.000495094 -0.000540762 +5486 5.49425 -0.000494572 -0.000540097 +5487 5.49522 -0.000494051 -0.000539434 +5488 5.49618 -0.000493531 -0.000538771 +5489 5.49715 -0.000493011 -0.000538109 +5490 5.49811 -0.000492492 -0.000537448 +5491 5.49908 -0.000491974 -0.000536789 +5492 5.50005 -0.000491456 -0.000536130 +5493 5.50101 -0.000490939 -0.000535472 +5494 5.50198 -0.000490423 -0.000534815 +5495 5.50294 -0.000489907 -0.000534158 +5496 5.50391 -0.000489392 -0.000533503 +5497 5.50487 -0.000488877 -0.000532849 +5498 5.50584 -0.000488363 -0.000532195 +5499 5.50680 -0.000487850 -0.000531543 +5500 5.50777 -0.000487337 -0.000530891 +5501 5.50873 -0.000486825 -0.000530240 +5502 5.50970 -0.000486314 -0.000529591 +5503 5.51066 -0.000485803 -0.000528942 +5504 5.51163 -0.000485293 -0.000528294 +5505 5.51259 -0.000484784 -0.000527647 +5506 5.51356 -0.000484275 -0.000527001 +5507 5.51452 -0.000483766 -0.000526355 +5508 5.51549 -0.000483259 -0.000525711 +5509 5.51645 -0.000482752 -0.000525068 +5510 5.51742 -0.000482245 -0.000524425 +5511 5.51838 -0.000481740 -0.000523783 +5512 5.51935 -0.000481234 -0.000523143 +5513 5.52031 -0.000480730 -0.000522503 +5514 5.52128 -0.000480226 -0.000521864 +5515 5.52224 -0.000479723 -0.000521226 +5516 5.52321 -0.000479220 -0.000520589 +5517 5.52417 -0.000478718 -0.000519952 +5518 5.52514 -0.000478216 -0.000519317 +5519 5.52610 -0.000477715 -0.000518683 +5520 5.52707 -0.000477215 -0.000518049 +5521 5.52803 -0.000476716 -0.000517416 +5522 5.52900 -0.000476217 -0.000516784 +5523 5.52996 -0.000475718 -0.000516153 +5524 5.53093 -0.000475220 -0.000515523 +5525 5.53189 -0.000474723 -0.000514894 +5526 5.53286 -0.000474226 -0.000514266 +5527 5.53382 -0.000473731 -0.000513638 +5528 5.53479 -0.000473235 -0.000513012 +5529 5.53575 -0.000472740 -0.000512386 +5530 5.53672 -0.000472246 -0.000511761 +5531 5.53768 -0.000471753 -0.000511137 +5532 5.53865 -0.000471260 -0.000510514 +5533 5.53961 -0.000470767 -0.000509892 +5534 5.54058 -0.000470275 -0.000509271 +5535 5.54154 -0.000469784 -0.000508650 +5536 5.54251 -0.000469294 -0.000508030 +5537 5.54347 -0.000468804 -0.000507412 +5538 5.54444 -0.000468314 -0.000506794 +5539 5.54540 -0.000467826 -0.000506177 +5540 5.54637 -0.000467337 -0.000505561 +5541 5.54733 -0.000466850 -0.000504945 +5542 5.54830 -0.000466363 -0.000504331 +5543 5.54926 -0.000465876 -0.000503717 +5544 5.55023 -0.000465391 -0.000503104 +5545 5.55119 -0.000464905 -0.000502492 +5546 5.55216 -0.000464421 -0.000501881 +5547 5.55312 -0.000463937 -0.000501271 +5548 5.55409 -0.000463453 -0.000500662 +5549 5.55505 -0.000462970 -0.000500053 +5550 5.55602 -0.000462488 -0.000499446 +5551 5.55698 -0.000462006 -0.000498839 +5552 5.55795 -0.000461525 -0.000498233 +5553 5.55891 -0.000461045 -0.000497628 +5554 5.55988 -0.000460565 -0.000497023 +5555 5.56084 -0.000460085 -0.000496420 +5556 5.56181 -0.000459607 -0.000495817 +5557 5.56277 -0.000459128 -0.000495215 +5558 5.56374 -0.000458651 -0.000494614 +5559 5.56470 -0.000458174 -0.000494014 +5560 5.56567 -0.000457697 -0.000493415 +5561 5.56663 -0.000457221 -0.000492816 +5562 5.56760 -0.000456746 -0.000492219 +5563 5.56856 -0.000456271 -0.000491622 +5564 5.56953 -0.000455797 -0.000491026 +5565 5.57049 -0.000455324 -0.000490431 +5566 5.57146 -0.000454851 -0.000489837 +5567 5.57242 -0.000454378 -0.000489243 +5568 5.57339 -0.000453906 -0.000488650 +5569 5.57435 -0.000453435 -0.000488058 +5570 5.57532 -0.000452964 -0.000487467 +5571 5.57628 -0.000452494 -0.000486877 +5572 5.57725 -0.000452025 -0.000486288 +5573 5.57821 -0.000451556 -0.000485699 +5574 5.57918 -0.000451087 -0.000485111 +5575 5.58014 -0.000450619 -0.000484524 +5576 5.58111 -0.000450152 -0.000483938 +5577 5.58207 -0.000449685 -0.000483353 +5578 5.58304 -0.000449219 -0.000482768 +5579 5.58400 -0.000448754 -0.000482185 +5580 5.58497 -0.000448289 -0.000481602 +5581 5.58593 -0.000447824 -0.000481020 +5582 5.58690 -0.000447360 -0.000480438 +5583 5.58786 -0.000446897 -0.000479858 +5584 5.58883 -0.000446434 -0.000479278 +5585 5.58979 -0.000445972 -0.000478699 +5586 5.59076 -0.000445510 -0.000478121 +5587 5.59172 -0.000445049 -0.000477544 +5588 5.59269 -0.000444588 -0.000476967 +5589 5.59365 -0.000444128 -0.000476391 +5590 5.59462 -0.000443669 -0.000475817 +5591 5.59558 -0.000443210 -0.000475242 +5592 5.59655 -0.000442752 -0.000474669 +5593 5.59751 -0.000442294 -0.000474097 +5594 5.59848 -0.000441836 -0.000473525 +5595 5.59944 -0.000441380 -0.000472954 +5596 5.60041 -0.000440924 -0.000472384 +5597 5.60138 -0.000440468 -0.000471814 +5598 5.60234 -0.000440013 -0.000471246 +5599 5.60331 -0.000439558 -0.000470678 +5600 5.60427 -0.000439105 -0.000470111 +5601 5.60524 -0.000438651 -0.000469544 +5602 5.60620 -0.000438198 -0.000468979 +5603 5.60717 -0.000437746 -0.000468414 +5604 5.60813 -0.000437294 -0.000467850 +5605 5.60910 -0.000436843 -0.000467287 +5606 5.61006 -0.000436392 -0.000466725 +5607 5.61103 -0.000435942 -0.000466163 +5608 5.61199 -0.000435493 -0.000465602 +5609 5.61296 -0.000435043 -0.000465042 +5610 5.61392 -0.000434595 -0.000464483 +5611 5.61489 -0.000434147 -0.000463924 +5612 5.61585 -0.000433700 -0.000463366 +5613 5.61682 -0.000433253 -0.000462809 +5614 5.61778 -0.000432806 -0.000462253 +5615 5.61875 -0.000432360 -0.000461698 +5616 5.61971 -0.000431915 -0.000461143 +5617 5.62068 -0.000431470 -0.000460589 +5618 5.62164 -0.000431026 -0.000460036 +5619 5.62261 -0.000430582 -0.000459483 +5620 5.62357 -0.000430139 -0.000458932 +5621 5.62454 -0.000429697 -0.000458381 +5622 5.62550 -0.000429255 -0.000457831 +5623 5.62647 -0.000428813 -0.000457281 +5624 5.62743 -0.000428372 -0.000456733 +5625 5.62840 -0.000427931 -0.000456185 +5626 5.62936 -0.000427492 -0.000455638 +5627 5.63033 -0.000427052 -0.000455091 +5628 5.63129 -0.000426613 -0.000454546 +5629 5.63226 -0.000426175 -0.000454001 +5630 5.63322 -0.000425737 -0.000453457 +5631 5.63419 -0.000425300 -0.000452913 +5632 5.63515 -0.000424863 -0.000452370 +5633 5.63612 -0.000424426 -0.000451829 +5634 5.63708 -0.000423991 -0.000451287 +5635 5.63805 -0.000423555 -0.000450747 +5636 5.63901 -0.000423121 -0.000450207 +5637 5.63998 -0.000422686 -0.000449668 +5638 5.64094 -0.000422253 -0.000449130 +5639 5.64191 -0.000421820 -0.000448593 +5640 5.64287 -0.000421387 -0.000448056 +5641 5.64384 -0.000420955 -0.000447520 +5642 5.64480 -0.000420523 -0.000446984 +5643 5.64577 -0.000420092 -0.000446450 +5644 5.64673 -0.000419661 -0.000445916 +5645 5.64770 -0.000419231 -0.000445383 +5646 5.64866 -0.000418802 -0.000444851 +5647 5.64963 -0.000418373 -0.000444319 +5648 5.65059 -0.000417944 -0.000443788 +5649 5.65156 -0.000417516 -0.000443258 +5650 5.65252 -0.000417089 -0.000442728 +5651 5.65349 -0.000416662 -0.000442200 +5652 5.65445 -0.000416235 -0.000441672 +5653 5.65542 -0.000415809 -0.000441144 +5654 5.65638 -0.000415384 -0.000440618 +5655 5.65735 -0.000414959 -0.000440092 +5656 5.65831 -0.000414534 -0.000439567 +5657 5.65928 -0.000414110 -0.000439042 +5658 5.66024 -0.000413687 -0.000438519 +5659 5.66121 -0.000413264 -0.000437996 +5660 5.66217 -0.000412842 -0.000437473 +5661 5.66314 -0.000412420 -0.000436952 +5662 5.66410 -0.000411998 -0.000436431 +5663 5.66507 -0.000411577 -0.000435911 +5664 5.66603 -0.000411157 -0.000435391 +5665 5.66700 -0.000410737 -0.000434873 +5666 5.66796 -0.000410318 -0.000434354 +5667 5.66893 -0.000409899 -0.000433837 +5668 5.66989 -0.000409480 -0.000433321 +5669 5.67086 -0.000409062 -0.000432805 +5670 5.67182 -0.000408645 -0.000432289 +5671 5.67279 -0.000408228 -0.000431775 +5672 5.67375 -0.000407811 -0.000431261 +5673 5.67472 -0.000407396 -0.000430748 +5674 5.67568 -0.000406980 -0.000430235 +5675 5.67665 -0.000406565 -0.000429724 +5676 5.67761 -0.000406151 -0.000429213 +5677 5.67858 -0.000405737 -0.000428702 +5678 5.67954 -0.000405323 -0.000428193 +5679 5.68051 -0.000404910 -0.000427684 +5680 5.68147 -0.000404498 -0.000427176 +5681 5.68244 -0.000404086 -0.000426668 +5682 5.68340 -0.000403674 -0.000426161 +5683 5.68437 -0.000403263 -0.000425655 +5684 5.68533 -0.000402853 -0.000425149 +5685 5.68630 -0.000402443 -0.000424645 +5686 5.68726 -0.000402033 -0.000424140 +5687 5.68823 -0.000401624 -0.000423637 +5688 5.68919 -0.000401215 -0.000423134 +5689 5.69016 -0.000400807 -0.000422632 +5690 5.69112 -0.000400400 -0.000422131 +5691 5.69209 -0.000399993 -0.000421630 +5692 5.69305 -0.000399586 -0.000421130 +5693 5.69402 -0.000399180 -0.000420631 +5694 5.69498 -0.000398774 -0.000420132 +5695 5.69595 -0.000398369 -0.000419634 +5696 5.69691 -0.000397964 -0.000419136 +5697 5.69788 -0.000397560 -0.000418640 +5698 5.69884 -0.000397156 -0.000418144 +5699 5.69981 -0.000396753 -0.000417648 +5700 5.70078 -0.000396350 -0.000417154 +5701 5.70174 -0.000395948 -0.000416660 +5702 5.70271 -0.000395546 -0.000416167 +5703 5.70367 -0.000395144 -0.000415674 +5704 5.70464 -0.000394744 -0.000415182 +5705 5.70560 -0.000394343 -0.000414691 +5706 5.70657 -0.000393943 -0.000414200 +5707 5.70753 -0.000393544 -0.000413710 +5708 5.70850 -0.000393145 -0.000413221 +5709 5.70946 -0.000392746 -0.000412732 +5710 5.71043 -0.000392348 -0.000412244 +5711 5.71139 -0.000391950 -0.000411757 +5712 5.71236 -0.000391553 -0.000411270 +5713 5.71332 -0.000391157 -0.000410784 +5714 5.71429 -0.000390760 -0.000410298 +5715 5.71525 -0.000390365 -0.000409814 +5716 5.71622 -0.000389969 -0.000409330 +5717 5.71718 -0.000389575 -0.000408846 +5718 5.71815 -0.000389180 -0.000408364 +5719 5.71911 -0.000388787 -0.000407881 +5720 5.72008 -0.000388393 -0.000407400 +5721 5.72104 -0.000388000 -0.000406919 +5722 5.72201 -0.000387608 -0.000406439 +5723 5.72297 -0.000387216 -0.000405959 +5724 5.72394 -0.000386824 -0.000405481 +5725 5.72490 -0.000386433 -0.000405002 +5726 5.72587 -0.000386042 -0.000404525 +5727 5.72683 -0.000385652 -0.000404048 +5728 5.72780 -0.000385263 -0.000403572 +5729 5.72876 -0.000384873 -0.000403096 +5730 5.72973 -0.000384485 -0.000402621 +5731 5.73069 -0.000384096 -0.000402147 +5732 5.73166 -0.000383708 -0.000401673 +5733 5.73262 -0.000383321 -0.000401200 +5734 5.73359 -0.000382934 -0.000400727 +5735 5.73455 -0.000382548 -0.000400255 +5736 5.73552 -0.000382162 -0.000399784 +5737 5.73648 -0.000381776 -0.000399314 +5738 5.73745 -0.000381391 -0.000398844 +5739 5.73841 -0.000381006 -0.000398374 +5740 5.73938 -0.000380622 -0.000397906 +5741 5.74034 -0.000380238 -0.000397438 +5742 5.74131 -0.000379855 -0.000396970 +5743 5.74227 -0.000379472 -0.000396504 +5744 5.74324 -0.000379090 -0.000396038 +5745 5.74420 -0.000378708 -0.000395572 +5746 5.74517 -0.000378326 -0.000395107 +5747 5.74613 -0.000377945 -0.000394643 +5748 5.74710 -0.000377564 -0.000394179 +5749 5.74806 -0.000377184 -0.000393716 +5750 5.74903 -0.000376804 -0.000393254 +5751 5.74999 -0.000376425 -0.000392792 +5752 5.75096 -0.000376046 -0.000392331 +5753 5.75192 -0.000375668 -0.000391870 +5754 5.75289 -0.000375290 -0.000391410 +5755 5.75385 -0.000374912 -0.000390951 +5756 5.75482 -0.000374535 -0.000390492 +5757 5.75578 -0.000374159 -0.000390034 +5758 5.75675 -0.000373783 -0.000389577 +5759 5.75771 -0.000373407 -0.000389120 +5760 5.75868 -0.000373032 -0.000388664 +5761 5.75964 -0.000372657 -0.000388208 +5762 5.76061 -0.000372282 -0.000387753 +5763 5.76157 -0.000371908 -0.000387299 +5764 5.76254 -0.000371535 -0.000386845 +5765 5.76350 -0.000371162 -0.000386392 +5766 5.76447 -0.000370789 -0.000385939 +5767 5.76543 -0.000370417 -0.000385487 +5768 5.76640 -0.000370045 -0.000385036 +5769 5.76736 -0.000369674 -0.000384585 +5770 5.76833 -0.000369303 -0.000384135 +5771 5.76929 -0.000368932 -0.000383685 +5772 5.77026 -0.000368562 -0.000383236 +5773 5.77122 -0.000368192 -0.000382788 +5774 5.77219 -0.000367823 -0.000382340 +5775 5.77315 -0.000367455 -0.000381893 +5776 5.77412 -0.000367086 -0.000381446 +5777 5.77508 -0.000366718 -0.000381001 +5778 5.77605 -0.000366351 -0.000380555 +5779 5.77701 -0.000365984 -0.000380110 +5780 5.77798 -0.000365617 -0.000379666 +5781 5.77894 -0.000365251 -0.000379223 +5782 5.77991 -0.000364885 -0.000378780 +5783 5.78087 -0.000364520 -0.000378337 +5784 5.78184 -0.000364155 -0.000377895 +5785 5.78280 -0.000363791 -0.000377454 +5786 5.78377 -0.000363427 -0.000377013 +5787 5.78473 -0.000363063 -0.000376573 +5788 5.78570 -0.000362700 -0.000376134 +5789 5.78666 -0.000362337 -0.000375695 +5790 5.78763 -0.000361975 -0.000375257 +5791 5.78859 -0.000361613 -0.000374819 +5792 5.78956 -0.000361251 -0.000374382 +5793 5.79052 -0.000360890 -0.000373945 +5794 5.79149 -0.000360529 -0.000373509 +5795 5.79245 -0.000360169 -0.000373074 +5796 5.79342 -0.000359809 -0.000372639 +5797 5.79438 -0.000359450 -0.000372205 +5798 5.79535 -0.000359091 -0.000371771 +5799 5.79631 -0.000358732 -0.000371338 +5800 5.79728 -0.000358374 -0.000370906 +5801 5.79824 -0.000358016 -0.000370474 +5802 5.79921 -0.000357659 -0.000370043 +5803 5.80018 -0.000357302 -0.000369612 +5804 5.80114 -0.000356946 -0.000369182 +5805 5.80211 -0.000356590 -0.000368752 +5806 5.80307 -0.000356234 -0.000368323 +5807 5.80404 -0.000355879 -0.000367895 +5808 5.80500 -0.000355524 -0.000367467 +5809 5.80597 -0.000355170 -0.000367039 +5810 5.80693 -0.000354816 -0.000366612 +5811 5.80790 -0.000354462 -0.000366186 +5812 5.80886 -0.000354109 -0.000365761 +5813 5.80983 -0.000353756 -0.000365336 +5814 5.81079 -0.000353404 -0.000364911 +5815 5.81176 -0.000353052 -0.000364487 +5816 5.81272 -0.000352700 -0.000364064 +5817 5.81369 -0.000352349 -0.000363641 +5818 5.81465 -0.000351998 -0.000363219 +5819 5.81562 -0.000351648 -0.000362797 +5820 5.81658 -0.000351298 -0.000362376 +5821 5.81755 -0.000350949 -0.000361955 +5822 5.81851 -0.000350599 -0.000361535 +5823 5.81948 -0.000350251 -0.000361116 +5824 5.82044 -0.000349902 -0.000360697 +5825 5.82141 -0.000349555 -0.000360278 +5826 5.82237 -0.000349207 -0.000359861 +5827 5.82334 -0.000348860 -0.000359443 +5828 5.82430 -0.000348513 -0.000359027 +5829 5.82527 -0.000348167 -0.000358611 +5830 5.82623 -0.000347821 -0.000358195 +5831 5.82720 -0.000347476 -0.000357780 +5832 5.82816 -0.000347131 -0.000357365 +5833 5.82913 -0.000346786 -0.000356952 +5834 5.83009 -0.000346442 -0.000356538 +5835 5.83106 -0.000346098 -0.000356125 +5836 5.83202 -0.000345754 -0.000355713 +5837 5.83299 -0.000345411 -0.000355301 +5838 5.83395 -0.000345069 -0.000354890 +5839 5.83492 -0.000344726 -0.000354479 +5840 5.83588 -0.000344384 -0.000354069 +5841 5.83685 -0.000344043 -0.000353660 +5842 5.83781 -0.000343702 -0.000353251 +5843 5.83878 -0.000343361 -0.000352842 +5844 5.83974 -0.000343021 -0.000352434 +5845 5.84071 -0.000342681 -0.000352027 +5846 5.84167 -0.000342341 -0.000351620 +5847 5.84264 -0.000342002 -0.000351213 +5848 5.84360 -0.000341663 -0.000350808 +5849 5.84457 -0.000341325 -0.000350402 +5850 5.84553 -0.000340987 -0.000349998 +5851 5.84650 -0.000340650 -0.000349593 +5852 5.84746 -0.000340312 -0.000349190 +5853 5.84843 -0.000339976 -0.000348787 +5854 5.84939 -0.000339639 -0.000348384 +5855 5.85036 -0.000339303 -0.000347982 +5856 5.85132 -0.000338968 -0.000347580 +5857 5.85229 -0.000338632 -0.000347179 +5858 5.85325 -0.000338297 -0.000346779 +5859 5.85422 -0.000337963 -0.000346379 +5860 5.85518 -0.000337629 -0.000345980 +5861 5.85615 -0.000337295 -0.000345581 +5862 5.85711 -0.000336962 -0.000345182 +5863 5.85808 -0.000336629 -0.000344784 +5864 5.85904 -0.000336296 -0.000344387 +5865 5.86001 -0.000335964 -0.000343990 +5866 5.86097 -0.000335632 -0.000343594 +5867 5.86194 -0.000335301 -0.000343198 +5868 5.86290 -0.000334970 -0.000342803 +5869 5.86387 -0.000334639 -0.000342408 +5870 5.86483 -0.000334309 -0.000342014 +5871 5.86580 -0.000333979 -0.000341620 +5872 5.86676 -0.000333650 -0.000341227 +5873 5.86773 -0.000333321 -0.000340835 +5874 5.86869 -0.000332992 -0.000340442 +5875 5.86966 -0.000332664 -0.000340051 +5876 5.87062 -0.000332336 -0.000339660 +5877 5.87159 -0.000332008 -0.000339269 +5878 5.87255 -0.000331681 -0.000338879 +5879 5.87352 -0.000331354 -0.000338489 +5880 5.87448 -0.000331028 -0.000338100 +5881 5.87545 -0.000330701 -0.000337712 +5882 5.87641 -0.000330376 -0.000337324 +5883 5.87738 -0.000330050 -0.000336936 +5884 5.87834 -0.000329725 -0.000336549 +5885 5.87931 -0.000329401 -0.000336163 +5886 5.88027 -0.000329077 -0.000335777 +5887 5.88124 -0.000328753 -0.000335391 +5888 5.88220 -0.000328429 -0.000335006 +5889 5.88317 -0.000328106 -0.000334622 +5890 5.88413 -0.000327783 -0.000334238 +5891 5.88510 -0.000327461 -0.000333854 +5892 5.88606 -0.000327139 -0.000333471 +5893 5.88703 -0.000326817 -0.000333089 +5894 5.88799 -0.000326496 -0.000332707 +5895 5.88896 -0.000326175 -0.000332325 +5896 5.88992 -0.000325855 -0.000331945 +5897 5.89089 -0.000325535 -0.000331564 +5898 5.89185 -0.000325215 -0.000331184 +5899 5.89282 -0.000324895 -0.000330805 +5900 5.89378 -0.000324576 -0.000330426 +5901 5.89475 -0.000324258 -0.000330047 +5902 5.89571 -0.000323939 -0.000329669 +5903 5.89668 -0.000323621 -0.000329292 +5904 5.89764 -0.000323304 -0.000328915 +5905 5.89861 -0.000322986 -0.000328538 +5906 5.89957 -0.000322670 -0.000328162 +5907 5.90054 -0.000322353 -0.000327787 +5908 5.90151 -0.000322037 -0.000327412 +5909 5.90247 -0.000321721 -0.000327037 +5910 5.90344 -0.000321406 -0.000326663 +5911 5.90440 -0.000321091 -0.000326289 +5912 5.90537 -0.000320776 -0.000325916 +5913 5.90633 -0.000320462 -0.000325544 +5914 5.90730 -0.000320148 -0.000325172 +5915 5.90826 -0.000319834 -0.000324800 +5916 5.90923 -0.000319521 -0.000324429 +5917 5.91019 -0.000319208 -0.000324058 +5918 5.91116 -0.000318895 -0.000323688 +5919 5.91212 -0.000318583 -0.000323319 +5920 5.91309 -0.000318271 -0.000322949 +5921 5.91405 -0.000317960 -0.000322581 +5922 5.91502 -0.000317649 -0.000322212 +5923 5.91598 -0.000317338 -0.000321845 +5924 5.91695 -0.000317027 -0.000321477 +5925 5.91791 -0.000316717 -0.000321111 +5926 5.91888 -0.000316408 -0.000320744 +5927 5.91984 -0.000316098 -0.000320378 +5928 5.92081 -0.000315789 -0.000320013 +5929 5.92177 -0.000315481 -0.000319648 +5930 5.92274 -0.000315172 -0.000319284 +5931 5.92370 -0.000314864 -0.000318920 +5932 5.92467 -0.000314557 -0.000318556 +5933 5.92563 -0.000314249 -0.000318193 +5934 5.92660 -0.000313943 -0.000317831 +5935 5.92756 -0.000313636 -0.000317469 +5936 5.92853 -0.000313330 -0.000317107 +5937 5.92949 -0.000313024 -0.000316746 +5938 5.93046 -0.000312718 -0.000316386 +5939 5.93142 -0.000312413 -0.000316025 +5940 5.93239 -0.000312109 -0.000315666 +5941 5.93335 -0.000311804 -0.000315307 +5942 5.93432 -0.000311500 -0.000314948 +5943 5.93528 -0.000311196 -0.000314589 +5944 5.93625 -0.000310893 -0.000314232 +5945 5.93721 -0.000310590 -0.000313874 +5946 5.93818 -0.000310287 -0.000313517 +5947 5.93914 -0.000309985 -0.000313161 +5948 5.94011 -0.000309683 -0.000312805 +5949 5.94107 -0.000309381 -0.000312450 +5950 5.94204 -0.000309079 -0.000312094 +5951 5.94300 -0.000308778 -0.000311740 +5952 5.94397 -0.000308478 -0.000311386 +5953 5.94493 -0.000308177 -0.000311032 +5954 5.94590 -0.000307877 -0.000310679 +5955 5.94686 -0.000307578 -0.000310326 +5956 5.94783 -0.000307279 -0.000309974 +5957 5.94879 -0.000306980 -0.000309622 +5958 5.94976 -0.000306681 -0.000309271 +5959 5.95072 -0.000306383 -0.000308920 +5960 5.95169 -0.000306085 -0.000308569 +5961 5.95265 -0.000305787 -0.000308219 +5962 5.95362 -0.000305490 -0.000307870 +5963 5.95458 -0.000305193 -0.000307521 +5964 5.95555 -0.000304896 -0.000307172 +5965 5.95651 -0.000304600 -0.000306824 +5966 5.95748 -0.000304304 -0.000306476 +5967 5.95844 -0.000304008 -0.000306129 +5968 5.95941 -0.000303713 -0.000305782 +5969 5.96037 -0.000303418 -0.000305435 +5970 5.96134 -0.000303124 -0.000305090 +5971 5.96230 -0.000302829 -0.000304744 +5972 5.96327 -0.000302535 -0.000304399 +5973 5.96423 -0.000302242 -0.000304054 +5974 5.96520 -0.000301949 -0.000303710 +5975 5.96616 -0.000301656 -0.000303366 +5976 5.96713 -0.000301363 -0.000303023 +5977 5.96809 -0.000301071 -0.000302680 +5978 5.96906 -0.000300779 -0.000302338 +5979 5.97002 -0.000300487 -0.000301996 +5980 5.97099 -0.000300196 -0.000301655 +5981 5.97195 -0.000299905 -0.000301314 +5982 5.97292 -0.000299614 -0.000300973 +5983 5.97388 -0.000299324 -0.000300633 +5984 5.97485 -0.000299034 -0.000300293 +5985 5.97581 -0.000298745 -0.000299954 +5986 5.97678 -0.000298455 -0.000299615 +5987 5.97774 -0.000298166 -0.000299276 +5988 5.97871 -0.000297878 -0.000298938 +5989 5.97967 -0.000297589 -0.000298601 +5990 5.98064 -0.000297301 -0.000298264 +5991 5.98160 -0.000297014 -0.000297927 +5992 5.98257 -0.000296726 -0.000297591 +5993 5.98353 -0.000296439 -0.000297255 +5994 5.98450 -0.000296153 -0.000296920 +5995 5.98546 -0.000295866 -0.000296585 +5996 5.98643 -0.000295580 -0.000296250 +5997 5.98739 -0.000295294 -0.000295916 +5998 5.98836 -0.000295009 -0.000295582 +5999 5.98932 -0.000294724 -0.000295249 +6000 5.99029 -0.000294439 -0.000294916 +6001 5.99125 -0.000294155 -0.000294584 +6002 5.99222 -0.000293871 -0.000294252 +6003 5.99318 -0.000293587 -0.000293921 +6004 5.99415 -0.000293303 -0.000293589 +6005 5.99511 -0.000293020 -0.000293259 +6006 5.99608 -0.000292737 -0.000292929 +6007 5.99704 -0.000292455 -0.000292599 +6008 5.99801 -0.000292172 -0.000292269 +6009 5.99897 -0.000291891 -0.000291940 +6010 5.99994 -0.000291609 -0.000291612 +6011 6.00091 -0.000291328 -0.000291284 +6012 6.00187 -0.000291047 -0.000290956 +6013 6.00284 -0.000290766 -0.000290629 +6014 6.00380 -0.000290486 -0.000290302 +6015 6.00477 -0.000290206 -0.000289976 +6016 6.00573 -0.000289926 -0.000289650 +6017 6.00670 -0.000289647 -0.000289324 +6018 6.00766 -0.000289368 -0.000288999 +6019 6.00863 -0.000289089 -0.000288674 +6020 6.00959 -0.000288811 -0.000288350 +6021 6.01056 -0.000288532 -0.000288026 +6022 6.01152 -0.000288255 -0.000287702 +6023 6.01249 -0.000287977 -0.000287379 +6024 6.01345 -0.000287700 -0.000287056 +6025 6.01442 -0.000287423 -0.000286734 +6026 6.01538 -0.000287147 -0.000286412 +6027 6.01635 -0.000286870 -0.000286091 +6028 6.01731 -0.000286594 -0.000285770 +6029 6.01828 -0.000286319 -0.000285449 +6030 6.01924 -0.000286043 -0.000285129 +6031 6.02021 -0.000285768 -0.000284809 +6032 6.02117 -0.000285494 -0.000284490 +6033 6.02214 -0.000285219 -0.000284171 +6034 6.02310 -0.000284945 -0.000283852 +6035 6.02407 -0.000284671 -0.000283534 +6036 6.02503 -0.000284398 -0.000283216 +6037 6.02600 -0.000284125 -0.000282899 +6038 6.02696 -0.000283852 -0.000282582 +6039 6.02793 -0.000283579 -0.000282266 +6040 6.02889 -0.000283307 -0.000281949 +6041 6.02986 -0.000283035 -0.000281634 +6042 6.03082 -0.000282764 -0.000281318 +6043 6.03179 -0.000282492 -0.000281004 +6044 6.03275 -0.000282221 -0.000280689 +6045 6.03372 -0.000281950 -0.000280375 +6046 6.03468 -0.000281680 -0.000280061 +6047 6.03565 -0.000281410 -0.000279748 +6048 6.03661 -0.000281140 -0.000279435 +6049 6.03758 -0.000280871 -0.000279123 +6050 6.03854 -0.000280601 -0.000278810 +6051 6.03951 -0.000280332 -0.000278499 +6052 6.04047 -0.000280064 -0.000278187 +6053 6.04144 -0.000279796 -0.000277876 +6054 6.04240 -0.000279528 -0.000277566 +6055 6.04337 -0.000279260 -0.000277256 +6056 6.04433 -0.000278992 -0.000276946 +6057 6.04530 -0.000278725 -0.000276637 +6058 6.04626 -0.000278458 -0.000276328 +6059 6.04723 -0.000278192 -0.000276019 +6060 6.04819 -0.000277926 -0.000275711 +6061 6.04916 -0.000277660 -0.000275403 +6062 6.05012 -0.000277394 -0.000275096 +6063 6.05109 -0.000277129 -0.000274789 +6064 6.05205 -0.000276864 -0.000274483 +6065 6.05302 -0.000276599 -0.000274176 +6066 6.05398 -0.000276335 -0.000273871 +6067 6.05495 -0.000276070 -0.000273565 +6068 6.05591 -0.000275807 -0.000273260 +6069 6.05688 -0.000275543 -0.000272956 +6070 6.05784 -0.000275280 -0.000272651 +6071 6.05881 -0.000275017 -0.000272347 +6072 6.05977 -0.000274754 -0.000272044 +6073 6.06074 -0.000274492 -0.000271741 +6074 6.06170 -0.000274230 -0.000271438 +6075 6.06267 -0.000273968 -0.000271136 +6076 6.06363 -0.000273706 -0.000270834 +6077 6.06460 -0.000273445 -0.000270532 +6078 6.06556 -0.000273184 -0.000270231 +6079 6.06653 -0.000272924 -0.000269931 +6080 6.06749 -0.000272663 -0.000269630 +6081 6.06846 -0.000272403 -0.000269330 +6082 6.06942 -0.000272143 -0.000269030 +6083 6.07039 -0.000271884 -0.000268731 +6084 6.07135 -0.000271625 -0.000268432 +6085 6.07232 -0.000271366 -0.000268134 +6086 6.07328 -0.000271107 -0.000267836 +6087 6.07425 -0.000270849 -0.000267538 +6088 6.07521 -0.000270591 -0.000267241 +6089 6.07618 -0.000270333 -0.000266944 +6090 6.07714 -0.000270076 -0.000266647 +6091 6.07811 -0.000269818 -0.000266351 +6092 6.07907 -0.000269561 -0.000266055 +6093 6.08004 -0.000269305 -0.000265760 +6094 6.08100 -0.000269048 -0.000265465 +6095 6.08197 -0.000268792 -0.000265170 +6096 6.08293 -0.000268537 -0.000264875 +6097 6.08390 -0.000268281 -0.000264582 +6098 6.08486 -0.000268026 -0.000264288 +6099 6.08583 -0.000267771 -0.000263995 +6100 6.08679 -0.000267516 -0.000263702 +6101 6.08776 -0.000267262 -0.000263409 +6102 6.08872 -0.000267008 -0.000263117 +6103 6.08969 -0.000266754 -0.000262826 +6104 6.09065 -0.000266501 -0.000262534 +6105 6.09162 -0.000266248 -0.000262243 +6106 6.09258 -0.000265995 -0.000261953 +6107 6.09355 -0.000265742 -0.000261662 +6108 6.09451 -0.000265490 -0.000261372 +6109 6.09548 -0.000265238 -0.000261083 +6110 6.09644 -0.000264986 -0.000260794 +6111 6.09741 -0.000264734 -0.000260505 +6112 6.09837 -0.000264483 -0.000260216 +6113 6.09934 -0.000264232 -0.000259928 +6114 6.10031 -0.000263981 -0.000259641 +6115 6.10127 -0.000263731 -0.000259353 +6116 6.10224 -0.000263481 -0.000259066 +6117 6.10320 -0.000263231 -0.000258780 +6118 6.10417 -0.000262981 -0.000258493 +6119 6.10513 -0.000262732 -0.000258208 +6120 6.10610 -0.000262483 -0.000257922 +6121 6.10706 -0.000262234 -0.000257637 +6122 6.10803 -0.000261986 -0.000257352 +6123 6.10899 -0.000261737 -0.000257068 +6124 6.10996 -0.000261489 -0.000256784 +6125 6.11092 -0.000261242 -0.000256500 +6126 6.11189 -0.000260994 -0.000256216 +6127 6.11285 -0.000260747 -0.000255933 +6128 6.11382 -0.000260500 -0.000255651 +6129 6.11478 -0.000260254 -0.000255369 +6130 6.11575 -0.000260007 -0.000255087 +6131 6.11671 -0.000259761 -0.000254805 +6132 6.11768 -0.000259516 -0.000254524 +6133 6.11864 -0.000259270 -0.000254243 +6134 6.11961 -0.000259025 -0.000253962 +6135 6.12057 -0.000258780 -0.000253682 +6136 6.12154 -0.000258535 -0.000253402 +6137 6.12250 -0.000258291 -0.000253123 +6138 6.12347 -0.000258047 -0.000252844 +6139 6.12443 -0.000257803 -0.000252565 +6140 6.12540 -0.000257559 -0.000252287 +6141 6.12636 -0.000257316 -0.000252009 +6142 6.12733 -0.000257073 -0.000251731 +6143 6.12829 -0.000256830 -0.000251454 +6144 6.12926 -0.000256588 -0.000251177 +6145 6.13022 -0.000256345 -0.000250900 +6146 6.13119 -0.000256103 -0.000250624 +6147 6.13215 -0.000255862 -0.000250348 +6148 6.13312 -0.000255620 -0.000250072 +6149 6.13408 -0.000255379 -0.000249797 +6150 6.13505 -0.000255138 -0.000249522 +6151 6.13601 -0.000254897 -0.000249247 +6152 6.13698 -0.000254657 -0.000248973 +6153 6.13794 -0.000254417 -0.000248699 +6154 6.13891 -0.000254177 -0.000248426 +6155 6.13987 -0.000253937 -0.000248152 +6156 6.14084 -0.000253698 -0.000247879 +6157 6.14180 -0.000253459 -0.000247607 +6158 6.14277 -0.000253220 -0.000247335 +6159 6.14373 -0.000252981 -0.000247063 +6160 6.14470 -0.000252743 -0.000246791 +6161 6.14566 -0.000252505 -0.000246520 +6162 6.14663 -0.000252267 -0.000246250 +6163 6.14759 -0.000252030 -0.000245979 +6164 6.14856 -0.000251793 -0.000245709 +6165 6.14952 -0.000251556 -0.000245439 +6166 6.15049 -0.000251319 -0.000245170 +6167 6.15145 -0.000251082 -0.000244901 +6168 6.15242 -0.000250846 -0.000244632 +6169 6.15338 -0.000250610 -0.000244363 +6170 6.15435 -0.000250375 -0.000244095 +6171 6.15531 -0.000250139 -0.000243828 +6172 6.15628 -0.000249904 -0.000243560 +6173 6.15724 -0.000249669 -0.000243293 +6174 6.15821 -0.000249434 -0.000243026 +6175 6.15917 -0.000249200 -0.000242760 +6176 6.16014 -0.000248966 -0.000242494 +6177 6.16110 -0.000248732 -0.000242228 +6178 6.16207 -0.000248498 -0.000241963 +6179 6.16303 -0.000248265 -0.000241697 +6180 6.16400 -0.000248032 -0.000241433 +6181 6.16496 -0.000247799 -0.000241168 +6182 6.16593 -0.000247566 -0.000240904 +6183 6.16689 -0.000247334 -0.000240640 +6184 6.16786 -0.000247102 -0.000240377 +6185 6.16882 -0.000246870 -0.000240114 +6186 6.16979 -0.000246638 -0.000239851 +6187 6.17075 -0.000246407 -0.000239589 +6188 6.17172 -0.000246176 -0.000239326 +6189 6.17268 -0.000245945 -0.000239065 +6190 6.17365 -0.000245715 -0.000238803 +6191 6.17461 -0.000245484 -0.000238542 +6192 6.17558 -0.000245254 -0.000238281 +6193 6.17654 -0.000245024 -0.000238021 +6194 6.17751 -0.000244795 -0.000237761 +6195 6.17847 -0.000244565 -0.000237501 +6196 6.17944 -0.000244336 -0.000237241 +6197 6.18040 -0.000244108 -0.000236982 +6198 6.18137 -0.000243879 -0.000236723 +6199 6.18233 -0.000243651 -0.000236465 +6200 6.18330 -0.000243423 -0.000236206 +6201 6.18426 -0.000243195 -0.000235949 +6202 6.18523 -0.000242967 -0.000235691 +6203 6.18619 -0.000242740 -0.000235434 +6204 6.18716 -0.000242513 -0.000235177 +6205 6.18812 -0.000242286 -0.000234920 +6206 6.18909 -0.000242059 -0.000234664 +6207 6.19005 -0.000241833 -0.000234408 +6208 6.19102 -0.000241607 -0.000234152 +6209 6.19198 -0.000241381 -0.000233897 +6210 6.19295 -0.000241155 -0.000233642 +6211 6.19391 -0.000240930 -0.000233387 +6212 6.19488 -0.000240705 -0.000233133 +6213 6.19584 -0.000240480 -0.000232879 +6214 6.19681 -0.000240255 -0.000232625 +6215 6.19777 -0.000240031 -0.000232372 +6216 6.19874 -0.000239807 -0.000232118 +6217 6.19970 -0.000239583 -0.000231866 +6218 6.20067 -0.000239359 -0.000231613 +6219 6.20164 -0.000239136 -0.000231361 +6220 6.20260 -0.000238913 -0.000231109 +6221 6.20357 -0.000238690 -0.000230858 +6222 6.20453 -0.000238467 -0.000230606 +6223 6.20550 -0.000238245 -0.000230355 +6224 6.20646 -0.000238023 -0.000230105 +6225 6.20743 -0.000237801 -0.000229855 +6226 6.20839 -0.000237579 -0.000229605 +6227 6.20936 -0.000237358 -0.000229355 +6228 6.21032 -0.000237136 -0.000229105 +6229 6.21129 -0.000236915 -0.000228856 +6230 6.21225 -0.000236695 -0.000228608 +6231 6.21322 -0.000236474 -0.000228359 +6232 6.21418 -0.000236254 -0.000228111 +6233 6.21515 -0.000236034 -0.000227863 +6234 6.21611 -0.000235814 -0.000227616 +6235 6.21708 -0.000235595 -0.000227369 +6236 6.21804 -0.000235375 -0.000227122 +6237 6.21901 -0.000235156 -0.000226875 +6238 6.21997 -0.000234937 -0.000226629 +6239 6.22094 -0.000234719 -0.000226383 +6240 6.22190 -0.000234500 -0.000226137 +6241 6.22287 -0.000234282 -0.000225892 +6242 6.22383 -0.000234064 -0.000225647 +6243 6.22480 -0.000233847 -0.000225402 +6244 6.22576 -0.000233629 -0.000225157 +6245 6.22673 -0.000233412 -0.000224913 +6246 6.22769 -0.000233195 -0.000224669 +6247 6.22866 -0.000232979 -0.000224426 +6248 6.22962 -0.000232762 -0.000224183 +6249 6.23059 -0.000232546 -0.000223940 +6250 6.23155 -0.000232330 -0.000223697 +6251 6.23252 -0.000232114 -0.000223455 +6252 6.23348 -0.000231899 -0.000223213 +6253 6.23445 -0.000231683 -0.000222971 +6254 6.23541 -0.000231468 -0.000222729 +6255 6.23638 -0.000231253 -0.000222488 +6256 6.23734 -0.000231039 -0.000222247 +6257 6.23831 -0.000230824 -0.000222007 +6258 6.23927 -0.000230610 -0.000221767 +6259 6.24024 -0.000230396 -0.000221527 +6260 6.24120 -0.000230183 -0.000221287 +6261 6.24217 -0.000229969 -0.000221048 +6262 6.24313 -0.000229756 -0.000220809 +6263 6.24410 -0.000229543 -0.000220570 +6264 6.24506 -0.000229330 -0.000220331 +6265 6.24603 -0.000229118 -0.000220093 +6266 6.24699 -0.000228906 -0.000219855 +6267 6.24796 -0.000228694 -0.000219618 +6268 6.24892 -0.000228482 -0.000219380 +6269 6.24989 -0.000228270 -0.000219143 +6270 6.25085 -0.000228059 -0.000218907 +6271 6.25182 -0.000227848 -0.000218670 +6272 6.25278 -0.000227637 -0.000218434 +6273 6.25375 -0.000227426 -0.000218198 +6274 6.25471 -0.000227216 -0.000217963 +6275 6.25568 -0.000227005 -0.000217727 +6276 6.25664 -0.000226795 -0.000217492 +6277 6.25761 -0.000226586 -0.000217258 +6278 6.25857 -0.000226376 -0.000217023 +6279 6.25954 -0.000226167 -0.000216789 +6280 6.26050 -0.000225958 -0.000216555 +6281 6.26147 -0.000225749 -0.000216322 +6282 6.26243 -0.000225540 -0.000216089 +6283 6.26340 -0.000225332 -0.000215856 +6284 6.26436 -0.000225123 -0.000215623 +6285 6.26533 -0.000224915 -0.000215391 +6286 6.26629 -0.000224708 -0.000215159 +6287 6.26726 -0.000224500 -0.000214927 +6288 6.26822 -0.000224293 -0.000214695 +6289 6.26919 -0.000224086 -0.000214464 +6290 6.27015 -0.000223879 -0.000214233 +6291 6.27112 -0.000223672 -0.000214002 +6292 6.27208 -0.000223466 -0.000213772 +6293 6.27305 -0.000223260 -0.000213542 +6294 6.27401 -0.000223054 -0.000213312 +6295 6.27498 -0.000222848 -0.000213082 +6296 6.27594 -0.000222642 -0.000212853 +6297 6.27691 -0.000222437 -0.000212624 +6298 6.27787 -0.000222232 -0.000212396 +6299 6.27884 -0.000222027 -0.000212167 +6300 6.27980 -0.000221823 -0.000211939 +6301 6.28077 -0.000221618 -0.000211711 +6302 6.28173 -0.000221414 -0.000211484 +6303 6.28270 -0.000221210 -0.000211256 +6304 6.28366 -0.000221006 -0.000211029 +6305 6.28463 -0.000220803 -0.000210803 +6306 6.28559 -0.000220599 -0.000210576 +6307 6.28656 -0.000220396 -0.000210350 +6308 6.28752 -0.000220193 -0.000210124 +6309 6.28849 -0.000219991 -0.000209898 +6310 6.28945 -0.000219788 -0.000209673 +6311 6.29042 -0.000219586 -0.000209448 +6312 6.29138 -0.000219384 -0.000209223 +6313 6.29235 -0.000219182 -0.000208999 +6314 6.29331 -0.000218981 -0.000208774 +6315 6.29428 -0.000218779 -0.000208551 +6316 6.29524 -0.000218578 -0.000208327 +6317 6.29621 -0.000218377 -0.000208103 +6318 6.29717 -0.000218176 -0.000207880 +6319 6.29814 -0.000217976 -0.000207657 +6320 6.29910 -0.000217776 -0.000207435 +6321 6.30007 -0.000217576 -0.000207212 +6322 6.30104 -0.000217376 -0.000206990 +6323 6.30200 -0.000217176 -0.000206769 +6324 6.30297 -0.000216977 -0.000206547 +6325 6.30393 -0.000216777 -0.000206326 +6326 6.30490 -0.000216578 -0.000206105 +6327 6.30586 -0.000216380 -0.000205884 +6328 6.30683 -0.000216181 -0.000205664 +6329 6.30779 -0.000215983 -0.000205444 +6330 6.30876 -0.000215784 -0.000205224 +6331 6.30972 -0.000215587 -0.000205004 +6332 6.31069 -0.000215389 -0.000204785 +6333 6.31165 -0.000215191 -0.000204566 +6334 6.31262 -0.000214994 -0.000204347 +6335 6.31358 -0.000214797 -0.000204128 +6336 6.31455 -0.000214600 -0.000203910 +6337 6.31551 -0.000214403 -0.000203692 +6338 6.31648 -0.000214207 -0.000203474 +6339 6.31744 -0.000214011 -0.000203257 +6340 6.31841 -0.000213815 -0.000203040 +6341 6.31937 -0.000213619 -0.000202823 +6342 6.32034 -0.000213423 -0.000202606 +6343 6.32130 -0.000213228 -0.000202390 +6344 6.32227 -0.000213032 -0.000202173 +6345 6.32323 -0.000212837 -0.000201958 +6346 6.32420 -0.000212643 -0.000201742 +6347 6.32516 -0.000212448 -0.000201527 +6348 6.32613 -0.000212254 -0.000201311 +6349 6.32709 -0.000212059 -0.000201097 +6350 6.32806 -0.000211866 -0.000200882 +6351 6.32902 -0.000211672 -0.000200668 +6352 6.32999 -0.000211478 -0.000200454 +6353 6.33095 -0.000211285 -0.000200240 +6354 6.33192 -0.000211092 -0.000200026 +6355 6.33288 -0.000210899 -0.000199813 +6356 6.33385 -0.000210706 -0.000199600 +6357 6.33481 -0.000210514 -0.000199387 +6358 6.33578 -0.000210321 -0.000199175 +6359 6.33674 -0.000210129 -0.000198963 +6360 6.33771 -0.000209937 -0.000198751 +6361 6.33867 -0.000209746 -0.000198539 +6362 6.33964 -0.000209554 -0.000198327 +6363 6.34060 -0.000209363 -0.000198116 +6364 6.34157 -0.000209172 -0.000197905 +6365 6.34253 -0.000208981 -0.000197695 +6366 6.34350 -0.000208790 -0.000197484 +6367 6.34446 -0.000208600 -0.000197274 +6368 6.34543 -0.000208409 -0.000197064 +6369 6.34639 -0.000208219 -0.000196854 +6370 6.34736 -0.000208029 -0.000196645 +6371 6.34832 -0.000207840 -0.000196436 +6372 6.34929 -0.000207650 -0.000196227 +6373 6.35025 -0.000207461 -0.000196018 +6374 6.35122 -0.000207272 -0.000195810 +6375 6.35218 -0.000207083 -0.000195602 +6376 6.35315 -0.000206894 -0.000195394 +6377 6.35411 -0.000206706 -0.000195186 +6378 6.35508 -0.000206518 -0.000194979 +6379 6.35604 -0.000206330 -0.000194772 +6380 6.35701 -0.000206142 -0.000194565 +6381 6.35797 -0.000205954 -0.000194358 +6382 6.35894 -0.000205767 -0.000194152 +6383 6.35990 -0.000205579 -0.000193946 +6384 6.36087 -0.000205392 -0.000193740 +6385 6.36183 -0.000205205 -0.000193534 +6386 6.36280 -0.000205019 -0.000193329 +6387 6.36376 -0.000204832 -0.000193124 +6388 6.36473 -0.000204646 -0.000192919 +6389 6.36569 -0.000204460 -0.000192714 +6390 6.36666 -0.000204274 -0.000192510 +6391 6.36762 -0.000204088 -0.000192306 +6392 6.36859 -0.000203903 -0.000192102 +6393 6.36955 -0.000203718 -0.000191898 +6394 6.37052 -0.000203532 -0.000191695 +6395 6.37148 -0.000203348 -0.000191492 +6396 6.37245 -0.000203163 -0.000191289 +6397 6.37341 -0.000202978 -0.000191086 +6398 6.37438 -0.000202794 -0.000190884 +6399 6.37534 -0.000202610 -0.000190681 +6400 6.37631 -0.000202426 -0.000190479 +6401 6.37727 -0.000202242 -0.000190278 +6402 6.37824 -0.000202059 -0.000190076 +6403 6.37920 -0.000201875 -0.000189875 +6404 6.38017 -0.000201692 -0.000189674 +6405 6.38113 -0.000201509 -0.000189474 +6406 6.38210 -0.000201327 -0.000189273 +6407 6.38306 -0.000201144 -0.000189073 +6408 6.38403 -0.000200962 -0.000188873 +6409 6.38499 -0.000200779 -0.000188673 +6410 6.38596 -0.000200597 -0.000188474 +6411 6.38692 -0.000200416 -0.000188274 +6412 6.38789 -0.000200234 -0.000188075 +6413 6.38885 -0.000200053 -0.000187877 +6414 6.38982 -0.000199871 -0.000187678 +6415 6.39078 -0.000199690 -0.000187480 +6416 6.39175 -0.000199510 -0.000187282 +6417 6.39271 -0.000199329 -0.000187084 +6418 6.39368 -0.000199149 -0.000186886 +6419 6.39464 -0.000198968 -0.000186689 +6420 6.39561 -0.000198788 -0.000186492 +6421 6.39657 -0.000198608 -0.000186295 +6422 6.39754 -0.000198429 -0.000186098 +6423 6.39850 -0.000198249 -0.000185902 +6424 6.39947 -0.000198070 -0.000185706 +6425 6.40044 -0.000197891 -0.000185510 +6426 6.40140 -0.000197712 -0.000185314 +6427 6.40237 -0.000197533 -0.000185119 +6428 6.40333 -0.000197354 -0.000184924 +6429 6.40430 -0.000197176 -0.000184729 +6430 6.40526 -0.000196998 -0.000184534 +6431 6.40623 -0.000196820 -0.000184339 +6432 6.40719 -0.000196642 -0.000184145 +6433 6.40816 -0.000196465 -0.000183951 +6434 6.40912 -0.000196287 -0.000183757 +6435 6.41009 -0.000196110 -0.000183564 +6436 6.41105 -0.000195933 -0.000183370 +6437 6.41202 -0.000195756 -0.000183177 +6438 6.41298 -0.000195579 -0.000182984 +6439 6.41395 -0.000195403 -0.000182792 +6440 6.41491 -0.000195226 -0.000182599 +6441 6.41588 -0.000195050 -0.000182407 +6442 6.41684 -0.000194874 -0.000182215 +6443 6.41781 -0.000194699 -0.000182024 +6444 6.41877 -0.000194523 -0.000181832 +6445 6.41974 -0.000194348 -0.000181641 +6446 6.42070 -0.000194172 -0.000181450 +6447 6.42167 -0.000193997 -0.000181259 +6448 6.42263 -0.000193823 -0.000181068 +6449 6.42360 -0.000193648 -0.000180878 +6450 6.42456 -0.000193474 -0.000180688 +6451 6.42553 -0.000193299 -0.000180498 +6452 6.42649 -0.000193125 -0.000180308 +6453 6.42746 -0.000192951 -0.000180119 +6454 6.42842 -0.000192777 -0.000179930 +6455 6.42939 -0.000192604 -0.000179741 +6456 6.43035 -0.000192431 -0.000179552 +6457 6.43132 -0.000192257 -0.000179364 +6458 6.43228 -0.000192084 -0.000179175 +6459 6.43325 -0.000191912 -0.000178987 +6460 6.43421 -0.000191739 -0.000178799 +6461 6.43518 -0.000191566 -0.000178612 +6462 6.43614 -0.000191394 -0.000178424 +6463 6.43711 -0.000191222 -0.000178237 +6464 6.43807 -0.000191050 -0.000178050 +6465 6.43904 -0.000190878 -0.000177864 +6466 6.44000 -0.000190707 -0.000177677 +6467 6.44097 -0.000190535 -0.000177491 +6468 6.44193 -0.000190364 -0.000177305 +6469 6.44290 -0.000190193 -0.000177119 +6470 6.44386 -0.000190022 -0.000176933 +6471 6.44483 -0.000189852 -0.000176748 +6472 6.44579 -0.000189681 -0.000176563 +6473 6.44676 -0.000189511 -0.000176378 +6474 6.44772 -0.000189341 -0.000176193 +6475 6.44869 -0.000189171 -0.000176009 +6476 6.44965 -0.000189001 -0.000175824 +6477 6.45062 -0.000188832 -0.000175640 +6478 6.45158 -0.000188662 -0.000175457 +6479 6.45255 -0.000188493 -0.000175273 +6480 6.45351 -0.000188324 -0.000175090 +6481 6.45448 -0.000188155 -0.000174906 +6482 6.45544 -0.000187986 -0.000174724 +6483 6.45641 -0.000187818 -0.000174541 +6484 6.45737 -0.000187649 -0.000174358 +6485 6.45834 -0.000187481 -0.000174176 +6486 6.45930 -0.000187313 -0.000173994 +6487 6.46027 -0.000187145 -0.000173812 +6488 6.46123 -0.000186978 -0.000173630 +6489 6.46220 -0.000186810 -0.000173449 +6490 6.46316 -0.000186643 -0.000173268 +6491 6.46413 -0.000186476 -0.000173087 +6492 6.46509 -0.000186309 -0.000172906 +6493 6.46606 -0.000186142 -0.000172725 +6494 6.46702 -0.000185976 -0.000172545 +6495 6.46799 -0.000185809 -0.000172365 +6496 6.46895 -0.000185643 -0.000172185 +6497 6.46992 -0.000185477 -0.000172005 +6498 6.47088 -0.000185311 -0.000171826 +6499 6.47185 -0.000185145 -0.000171647 +6500 6.47281 -0.000184980 -0.000171468 +6501 6.47378 -0.000184814 -0.000171289 +6502 6.47474 -0.000184649 -0.000171110 +6503 6.47571 -0.000184484 -0.000170932 +6504 6.47667 -0.000184319 -0.000170753 +6505 6.47764 -0.000184154 -0.000170575 +6506 6.47860 -0.000183990 -0.000170398 +6507 6.47957 -0.000183825 -0.000170220 +6508 6.48053 -0.000183661 -0.000170043 +6509 6.48150 -0.000183497 -0.000169866 +6510 6.48246 -0.000183333 -0.000169689 +6511 6.48343 -0.000183170 -0.000169512 +6512 6.48439 -0.000183006 -0.000169335 +6513 6.48536 -0.000182843 -0.000169159 +6514 6.48632 -0.000182680 -0.000168983 +6515 6.48729 -0.000182517 -0.000168807 +6516 6.48825 -0.000182354 -0.000168631 +6517 6.48922 -0.000182191 -0.000168456 +6518 6.49018 -0.000182029 -0.000168281 +6519 6.49115 -0.000181866 -0.000168106 +6520 6.49211 -0.000181704 -0.000167931 +6521 6.49308 -0.000181542 -0.000167756 +6522 6.49404 -0.000181381 -0.000167582 +6523 6.49501 -0.000181219 -0.000167408 +6524 6.49597 -0.000181057 -0.000167233 +6525 6.49694 -0.000180896 -0.000167060 +6526 6.49790 -0.000180735 -0.000166886 +6527 6.49887 -0.000180574 -0.000166713 +6528 6.49983 -0.000180413 -0.000166539 +6529 6.50080 -0.000180253 -0.000166367 +6530 6.50177 -0.000180092 -0.000166194 +6531 6.50273 -0.000179932 -0.000166021 +6532 6.50370 -0.000179772 -0.000165849 +6533 6.50466 -0.000179612 -0.000165677 +6534 6.50563 -0.000179452 -0.000165505 +6535 6.50659 -0.000179292 -0.000165333 +6536 6.50756 -0.000179133 -0.000165161 +6537 6.50852 -0.000178973 -0.000164990 +6538 6.50949 -0.000178814 -0.000164819 +6539 6.51045 -0.000178655 -0.000164648 +6540 6.51142 -0.000178497 -0.000164477 +6541 6.51238 -0.000178338 -0.000164307 +6542 6.51335 -0.000178179 -0.000164136 +6543 6.51431 -0.000178021 -0.000163966 +6544 6.51528 -0.000177863 -0.000163796 +6545 6.51624 -0.000177705 -0.000163627 +6546 6.51721 -0.000177547 -0.000163457 +6547 6.51817 -0.000177389 -0.000163288 +6548 6.51914 -0.000177232 -0.000163119 +6549 6.52010 -0.000177075 -0.000162950 +6550 6.52107 -0.000176917 -0.000162781 +6551 6.52203 -0.000176760 -0.000162612 +6552 6.52300 -0.000176604 -0.000162444 +6553 6.52396 -0.000176447 -0.000162276 +6554 6.52493 -0.000176290 -0.000162108 +6555 6.52589 -0.000176134 -0.000161940 +6556 6.52686 -0.000175978 -0.000161773 +6557 6.52782 -0.000175822 -0.000161605 +6558 6.52879 -0.000175666 -0.000161438 +6559 6.52975 -0.000175510 -0.000161271 +6560 6.53072 -0.000175355 -0.000161105 +6561 6.53168 -0.000175199 -0.000160938 +6562 6.53265 -0.000175044 -0.000160772 +6563 6.53361 -0.000174889 -0.000160605 +6564 6.53458 -0.000174734 -0.000160440 +6565 6.53554 -0.000174579 -0.000160274 +6566 6.53651 -0.000174425 -0.000160108 +6567 6.53747 -0.000174270 -0.000159943 +6568 6.53844 -0.000174116 -0.000159778 +6569 6.53940 -0.000173962 -0.000159613 +6570 6.54037 -0.000173808 -0.000159448 +6571 6.54133 -0.000173654 -0.000159283 +6572 6.54230 -0.000173501 -0.000159119 +6573 6.54326 -0.000173347 -0.000158955 +6574 6.54423 -0.000173194 -0.000158791 +6575 6.54519 -0.000173041 -0.000158627 +6576 6.54616 -0.000172888 -0.000158463 +6577 6.54712 -0.000172735 -0.000158300 +6578 6.54809 -0.000172582 -0.000158137 +6579 6.54905 -0.000172429 -0.000157974 +6580 6.55002 -0.000172277 -0.000157811 +6581 6.55098 -0.000172125 -0.000157648 +6582 6.55195 -0.000171973 -0.000157486 +6583 6.55291 -0.000171821 -0.000157323 +6584 6.55388 -0.000171669 -0.000157161 +6585 6.55484 -0.000171518 -0.000156999 +6586 6.55581 -0.000171366 -0.000156838 +6587 6.55677 -0.000171215 -0.000156676 +6588 6.55774 -0.000171064 -0.000156515 +6589 6.55870 -0.000170913 -0.000156354 +6590 6.55967 -0.000170762 -0.000156193 +6591 6.56063 -0.000170611 -0.000156032 +6592 6.56160 -0.000170461 -0.000155871 +6593 6.56256 -0.000170310 -0.000155711 +6594 6.56353 -0.000170160 -0.000155551 +6595 6.56449 -0.000170010 -0.000155391 +6596 6.56546 -0.000169860 -0.000155231 +6597 6.56642 -0.000169711 -0.000155071 +6598 6.56739 -0.000169561 -0.000154912 +6599 6.56835 -0.000169412 -0.000154753 +6600 6.56932 -0.000169262 -0.000154594 +6601 6.57028 -0.000169113 -0.000154435 +6602 6.57125 -0.000168964 -0.000154276 +6603 6.57221 -0.000168815 -0.000154117 +6604 6.57318 -0.000168667 -0.000153959 +6605 6.57414 -0.000168518 -0.000153801 +6606 6.57511 -0.000168370 -0.000153643 +6607 6.57607 -0.000168222 -0.000153485 +6608 6.57704 -0.000168074 -0.000153328 +6609 6.57800 -0.000167926 -0.000153170 +6610 6.57897 -0.000167778 -0.000153013 +6611 6.57993 -0.000167631 -0.000152856 +6612 6.58090 -0.000167483 -0.000152699 +6613 6.58186 -0.000167336 -0.000152543 +6614 6.58283 -0.000167189 -0.000152386 +6615 6.58379 -0.000167042 -0.000152230 +6616 6.58476 -0.000166895 -0.000152074 +6617 6.58572 -0.000166748 -0.000151918 +6618 6.58669 -0.000166602 -0.000151762 +6619 6.58765 -0.000166455 -0.000151607 +6620 6.58862 -0.000166309 -0.000151451 +6621 6.58958 -0.000166163 -0.000151296 +6622 6.59055 -0.000166017 -0.000151141 +6623 6.59151 -0.000165871 -0.000150986 +6624 6.59248 -0.000165726 -0.000150831 +6625 6.59344 -0.000165580 -0.000150677 +6626 6.59441 -0.000165435 -0.000150523 +6627 6.59537 -0.000165290 -0.000150369 +6628 6.59634 -0.000165145 -0.000150215 +6629 6.59730 -0.000165000 -0.000150061 +6630 6.59827 -0.000164855 -0.000149907 +6631 6.59923 -0.000164710 -0.000149754 +6632 6.60020 -0.000164566 -0.000149601 +6633 6.60117 -0.000164422 -0.000149448 +6634 6.60213 -0.000164277 -0.000149295 +6635 6.60310 -0.000164133 -0.000149142 +6636 6.60406 -0.000163990 -0.000148990 +6637 6.60503 -0.000163846 -0.000148837 +6638 6.60599 -0.000163702 -0.000148685 +6639 6.60696 -0.000163559 -0.000148533 +6640 6.60792 -0.000163416 -0.000148382 +6641 6.60889 -0.000163272 -0.000148230 +6642 6.60985 -0.000163130 -0.000148079 +6643 6.61082 -0.000162987 -0.000147927 +6644 6.61178 -0.000162844 -0.000147776 +6645 6.61275 -0.000162701 -0.000147625 +6646 6.61371 -0.000162559 -0.000147475 +6647 6.61468 -0.000162417 -0.000147324 +6648 6.61564 -0.000162275 -0.000147174 +6649 6.61661 -0.000162133 -0.000147023 +6650 6.61757 -0.000161991 -0.000146873 +6651 6.61854 -0.000161849 -0.000146724 +6652 6.61950 -0.000161708 -0.000146574 +6653 6.62047 -0.000161566 -0.000146424 +6654 6.62143 -0.000161425 -0.000146275 +6655 6.62240 -0.000161284 -0.000146126 +6656 6.62336 -0.000161143 -0.000145977 +6657 6.62433 -0.000161002 -0.000145828 +6658 6.62529 -0.000160862 -0.000145680 +6659 6.62626 -0.000160721 -0.000145531 +6660 6.62722 -0.000160581 -0.000145383 +6661 6.62819 -0.000160441 -0.000145235 +6662 6.62915 -0.000160300 -0.000145087 +6663 6.63012 -0.000160160 -0.000144939 +6664 6.63108 -0.000160021 -0.000144791 +6665 6.63205 -0.000159881 -0.000144644 +6666 6.63301 -0.000159742 -0.000144497 +6667 6.63398 -0.000159602 -0.000144350 +6668 6.63494 -0.000159463 -0.000144203 +6669 6.63591 -0.000159324 -0.000144056 +6670 6.63687 -0.000159185 -0.000143910 +6671 6.63784 -0.000159046 -0.000143763 +6672 6.63880 -0.000158907 -0.000143617 +6673 6.63977 -0.000158769 -0.000143471 +6674 6.64073 -0.000158630 -0.000143325 +6675 6.64170 -0.000158492 -0.000143179 +6676 6.64266 -0.000158354 -0.000143034 +6677 6.64363 -0.000158216 -0.000142888 +6678 6.64459 -0.000158078 -0.000142743 +6679 6.64556 -0.000157941 -0.000142598 +6680 6.64652 -0.000157803 -0.000142453 +6681 6.64749 -0.000157666 -0.000142309 +6682 6.64845 -0.000157528 -0.000142164 +6683 6.64942 -0.000157391 -0.000142020 +6684 6.65038 -0.000157254 -0.000141875 +6685 6.65135 -0.000157117 -0.000141731 +6686 6.65231 -0.000156981 -0.000141588 +6687 6.65328 -0.000156844 -0.000141444 +6688 6.65424 -0.000156708 -0.000141300 +6689 6.65521 -0.000156571 -0.000141157 +6690 6.65617 -0.000156435 -0.000141014 +6691 6.65714 -0.000156299 -0.000140871 +6692 6.65810 -0.000156163 -0.000140728 +6693 6.65907 -0.000156028 -0.000140585 +6694 6.66003 -0.000155892 -0.000140443 +6695 6.66100 -0.000155757 -0.000140300 +6696 6.66196 -0.000155621 -0.000140158 +6697 6.66293 -0.000155486 -0.000140016 +6698 6.66389 -0.000155351 -0.000139874 +6699 6.66486 -0.000155216 -0.000139732 +6700 6.66582 -0.000155081 -0.000139591 +6701 6.66679 -0.000154947 -0.000139449 +6702 6.66775 -0.000154812 -0.000139308 +6703 6.66872 -0.000154678 -0.000139167 +6704 6.66968 -0.000154544 -0.000139026 +6705 6.67065 -0.000154409 -0.000138886 +6706 6.67161 -0.000154276 -0.000138745 +6707 6.67258 -0.000154142 -0.000138605 +6708 6.67354 -0.000154008 -0.000138464 +6709 6.67451 -0.000153874 -0.000138324 +6710 6.67547 -0.000153741 -0.000138184 +6711 6.67644 -0.000153608 -0.000138045 +6712 6.67740 -0.000153475 -0.000137905 +6713 6.67837 -0.000153342 -0.000137766 +6714 6.67933 -0.000153209 -0.000137626 +6715 6.68030 -0.000153076 -0.000137487 +6716 6.68126 -0.000152943 -0.000137348 +6717 6.68223 -0.000152811 -0.000137209 +6718 6.68319 -0.000152678 -0.000137071 +6719 6.68416 -0.000152546 -0.000136932 +6720 6.68512 -0.000152414 -0.000136794 +6721 6.68609 -0.000152282 -0.000136656 +6722 6.68705 -0.000152150 -0.000136518 +6723 6.68802 -0.000152019 -0.000136380 +6724 6.68898 -0.000151887 -0.000136242 +6725 6.68995 -0.000151756 -0.000136105 +6726 6.69091 -0.000151625 -0.000135968 +6727 6.69188 -0.000151493 -0.000135830 +6728 6.69284 -0.000151362 -0.000135693 +6729 6.69381 -0.000151231 -0.000135556 +6730 6.69477 -0.000151101 -0.000135420 +6731 6.69574 -0.000150970 -0.000135283 +6732 6.69670 -0.000150840 -0.000135147 +6733 6.69767 -0.000150709 -0.000135010 +6734 6.69863 -0.000150579 -0.000134874 +6735 6.69960 -0.000150449 -0.000134738 +6736 6.70057 -0.000150319 -0.000134603 +6737 6.70153 -0.000150189 -0.000134467 +6738 6.70250 -0.000150059 -0.000134332 +6739 6.70346 -0.000149930 -0.000134196 +6740 6.70443 -0.000149800 -0.000134061 +6741 6.70539 -0.000149671 -0.000133926 +6742 6.70636 -0.000149542 -0.000133791 +6743 6.70732 -0.000149413 -0.000133657 +6744 6.70829 -0.000149284 -0.000133522 +6745 6.70925 -0.000149155 -0.000133388 +6746 6.71022 -0.000149027 -0.000133253 +6747 6.71118 -0.000148898 -0.000133119 +6748 6.71215 -0.000148770 -0.000132985 +6749 6.71311 -0.000148641 -0.000132852 +6750 6.71408 -0.000148513 -0.000132718 +6751 6.71504 -0.000148385 -0.000132585 +6752 6.71601 -0.000148257 -0.000132451 +6753 6.71697 -0.000148129 -0.000132318 +6754 6.71794 -0.000148002 -0.000132185 +6755 6.71890 -0.000147874 -0.000132052 +6756 6.71987 -0.000147747 -0.000131920 +6757 6.72083 -0.000147620 -0.000131787 +6758 6.72180 -0.000147493 -0.000131655 +6759 6.72276 -0.000147366 -0.000131522 +6760 6.72373 -0.000147239 -0.000131390 +6761 6.72469 -0.000147112 -0.000131258 +6762 6.72566 -0.000146985 -0.000131127 +6763 6.72662 -0.000146859 -0.000130995 +6764 6.72759 -0.000146733 -0.000130864 +6765 6.72855 -0.000146606 -0.000130732 +6766 6.72952 -0.000146480 -0.000130601 +6767 6.73048 -0.000146354 -0.000130470 +6768 6.73145 -0.000146228 -0.000130339 +6769 6.73241 -0.000146103 -0.000130208 +6770 6.73338 -0.000145977 -0.000130078 +6771 6.73434 -0.000145852 -0.000129947 +6772 6.73531 -0.000145726 -0.000129817 +6773 6.73627 -0.000145601 -0.000129687 +6774 6.73724 -0.000145476 -0.000129557 +6775 6.73820 -0.000145351 -0.000129427 +6776 6.73917 -0.000145226 -0.000129297 +6777 6.74013 -0.000145102 -0.000129168 +6778 6.74110 -0.000144977 -0.000129039 +6779 6.74206 -0.000144852 -0.000128909 +6780 6.74303 -0.000144728 -0.000128780 +6781 6.74399 -0.000144604 -0.000128651 +6782 6.74496 -0.000144480 -0.000128523 +6783 6.74592 -0.000144356 -0.000128394 +6784 6.74689 -0.000144232 -0.000128265 +6785 6.74785 -0.000144108 -0.000128137 +6786 6.74882 -0.000143985 -0.000128009 +6787 6.74978 -0.000143861 -0.000127881 +6788 6.75075 -0.000143738 -0.000127753 +6789 6.75171 -0.000143615 -0.000127625 +6790 6.75268 -0.000143492 -0.000127497 +6791 6.75364 -0.000143369 -0.000127370 +6792 6.75461 -0.000143246 -0.000127243 +6793 6.75557 -0.000143123 -0.000127115 +6794 6.75654 -0.000143000 -0.000126988 +6795 6.75750 -0.000142878 -0.000126862 +6796 6.75847 -0.000142755 -0.000126735 +6797 6.75943 -0.000142633 -0.000126608 +6798 6.76040 -0.000142511 -0.000126482 +6799 6.76136 -0.000142389 -0.000126355 +6800 6.76233 -0.000142267 -0.000126229 +6801 6.76329 -0.000142145 -0.000126103 +6802 6.76426 -0.000142024 -0.000125977 +6803 6.76522 -0.000141902 -0.000125852 +6804 6.76619 -0.000141781 -0.000125726 +6805 6.76715 -0.000141660 -0.000125601 +6806 6.76812 -0.000141539 -0.000125475 +6807 6.76908 -0.000141417 -0.000125350 +6808 6.77005 -0.000141297 -0.000125225 +6809 6.77101 -0.000141176 -0.000125100 +6810 6.77198 -0.000141055 -0.000124975 +6811 6.77294 -0.000140935 -0.000124851 +6812 6.77391 -0.000140814 -0.000124726 +6813 6.77487 -0.000140694 -0.000124602 +6814 6.77584 -0.000140574 -0.000124478 +6815 6.77680 -0.000140454 -0.000124354 +6816 6.77777 -0.000140334 -0.000124230 +6817 6.77873 -0.000140214 -0.000124106 +6818 6.77970 -0.000140094 -0.000123983 +6819 6.78066 -0.000139975 -0.000123859 +6820 6.78163 -0.000139855 -0.000123736 +6821 6.78259 -0.000139736 -0.000123613 +6822 6.78356 -0.000139616 -0.000123490 +6823 6.78452 -0.000139497 -0.000123367 +6824 6.78549 -0.000139378 -0.000123244 +6825 6.78645 -0.000139259 -0.000123121 +6826 6.78742 -0.000139141 -0.000122999 +6827 6.78838 -0.000139022 -0.000122876 +6828 6.78935 -0.000138904 -0.000122754 +6829 6.79031 -0.000138785 -0.000122632 +6830 6.79128 -0.000138667 -0.000122510 +6831 6.79224 -0.000138549 -0.000122388 +6832 6.79321 -0.000138431 -0.000122267 +6833 6.79417 -0.000138313 -0.000122145 +6834 6.79514 -0.000138195 -0.000122024 +6835 6.79610 -0.000138077 -0.000121903 +6836 6.79707 -0.000137960 -0.000121782 +6837 6.79803 -0.000137842 -0.000121661 +6838 6.79900 -0.000137725 -0.000121540 +6839 6.79996 -0.000137608 -0.000121419 +6840 6.80093 -0.000137490 -0.000121298 +6841 6.80190 -0.000137373 -0.000121178 +6842 6.80286 -0.000137257 -0.000121058 +6843 6.80383 -0.000137140 -0.000120938 +6844 6.80479 -0.000137023 -0.000120818 +6845 6.80576 -0.000136907 -0.000120698 +6846 6.80672 -0.000136790 -0.000120578 +6847 6.80769 -0.000136674 -0.000120458 +6848 6.80865 -0.000136558 -0.000120339 +6849 6.80962 -0.000136442 -0.000120220 +6850 6.81058 -0.000136326 -0.000120100 +6851 6.81155 -0.000136210 -0.000119981 +6852 6.81251 -0.000136094 -0.000119862 +6853 6.81348 -0.000135978 -0.000119744 +6854 6.81444 -0.000135863 -0.000119625 +6855 6.81541 -0.000135748 -0.000119507 +6856 6.81637 -0.000135632 -0.000119388 +6857 6.81734 -0.000135517 -0.000119270 +6858 6.81830 -0.000135402 -0.000119152 +6859 6.81927 -0.000135287 -0.000119034 +6860 6.82023 -0.000135172 -0.000118916 +6861 6.82120 -0.000135058 -0.000118798 +6862 6.82216 -0.000134943 -0.000118681 +6863 6.82313 -0.000134829 -0.000118563 +6864 6.82409 -0.000134714 -0.000118446 +6865 6.82506 -0.000134600 -0.000118329 +6866 6.82602 -0.000134486 -0.000118212 +6867 6.82699 -0.000134372 -0.000118095 +6868 6.82795 -0.000134258 -0.000117978 +6869 6.82892 -0.000134144 -0.000117861 +6870 6.82988 -0.000134030 -0.000117745 +6871 6.83085 -0.000133917 -0.000117628 +6872 6.83181 -0.000133803 -0.000117512 +6873 6.83278 -0.000133690 -0.000117396 +6874 6.83374 -0.000133577 -0.000117280 +6875 6.83471 -0.000133464 -0.000117164 +6876 6.83567 -0.000133351 -0.000117048 +6877 6.83664 -0.000133238 -0.000116933 +6878 6.83760 -0.000133125 -0.000116817 +6879 6.83857 -0.000133012 -0.000116702 +6880 6.83953 -0.000132900 -0.000116587 +6881 6.84050 -0.000132787 -0.000116472 +6882 6.84146 -0.000132675 -0.000116357 +6883 6.84243 -0.000132563 -0.000116242 +6884 6.84339 -0.000132451 -0.000116127 +6885 6.84436 -0.000132339 -0.000116013 +6886 6.84532 -0.000132227 -0.000115898 +6887 6.84629 -0.000132115 -0.000115784 +6888 6.84725 -0.000132003 -0.000115670 +6889 6.84822 -0.000131892 -0.000115556 +6890 6.84918 -0.000131780 -0.000115442 +6891 6.85015 -0.000131669 -0.000115328 +6892 6.85111 -0.000131558 -0.000115214 +6893 6.85208 -0.000131446 -0.000115101 +6894 6.85304 -0.000131335 -0.000114987 +6895 6.85401 -0.000131224 -0.000114874 +6896 6.85497 -0.000131114 -0.000114761 +6897 6.85594 -0.000131003 -0.000114648 +6898 6.85690 -0.000130892 -0.000114535 +6899 6.85787 -0.000130782 -0.000114422 +6900 6.85883 -0.000130672 -0.000114309 +6901 6.85980 -0.000130561 -0.000114197 +6902 6.86076 -0.000130451 -0.000114084 +6903 6.86173 -0.000130341 -0.000113972 +6904 6.86269 -0.000130231 -0.000113860 +6905 6.86366 -0.000130121 -0.000113748 +6906 6.86462 -0.000130012 -0.000113636 +6907 6.86559 -0.000129902 -0.000113524 +6908 6.86655 -0.000129792 -0.000113413 +6909 6.86752 -0.000129683 -0.000113301 +6910 6.86848 -0.000129574 -0.000113190 +6911 6.86945 -0.000129465 -0.000113079 +6912 6.87041 -0.000129356 -0.000112967 +6913 6.87138 -0.000129247 -0.000112856 +6914 6.87234 -0.000129138 -0.000112746 +6915 6.87331 -0.000129029 -0.000112635 +6916 6.87427 -0.000128920 -0.000112524 +6917 6.87524 -0.000128812 -0.000112414 +6918 6.87620 -0.000128703 -0.000112303 +6919 6.87717 -0.000128595 -0.000112193 +6920 6.87813 -0.000128487 -0.000112083 +6921 6.87910 -0.000128379 -0.000111973 +6922 6.88006 -0.000128271 -0.000111863 +6923 6.88103 -0.000128163 -0.000111753 +6924 6.88199 -0.000128055 -0.000111644 +6925 6.88296 -0.000127947 -0.000111534 +6926 6.88392 -0.000127840 -0.000111425 +6927 6.88489 -0.000127732 -0.000111315 +6928 6.88585 -0.000127625 -0.000111206 +6929 6.88682 -0.000127518 -0.000111097 +6930 6.88778 -0.000127410 -0.000110988 +6931 6.88875 -0.000127303 -0.000110879 +6932 6.88971 -0.000127196 -0.000110771 +6933 6.89068 -0.000127090 -0.000110662 +6934 6.89164 -0.000126983 -0.000110554 +6935 6.89261 -0.000126876 -0.000110445 +6936 6.89357 -0.000126770 -0.000110337 +6937 6.89454 -0.000126663 -0.000110229 +6938 6.89550 -0.000126557 -0.000110121 +6939 6.89647 -0.000126451 -0.000110013 +6940 6.89743 -0.000126345 -0.000109906 +6941 6.89840 -0.000126239 -0.000109798 +6942 6.89936 -0.000126133 -0.000109691 +6943 6.90033 -0.000126027 -0.000109583 +6944 6.90130 -0.000125921 -0.000109476 +6945 6.90226 -0.000125816 -0.000109369 +6946 6.90323 -0.000125710 -0.000109262 +6947 6.90419 -0.000125605 -0.000109155 +6948 6.90516 -0.000125499 -0.000109048 +6949 6.90612 -0.000125394 -0.000108942 +6950 6.90709 -0.000125289 -0.000108835 +6951 6.90805 -0.000125184 -0.000108729 +6952 6.90902 -0.000125079 -0.000108623 +6953 6.90998 -0.000124974 -0.000108517 +6954 6.91095 -0.000124870 -0.000108410 +6955 6.91191 -0.000124765 -0.000108305 +6956 6.91288 -0.000124661 -0.000108199 +6957 6.91384 -0.000124556 -0.000108093 +6958 6.91481 -0.000124452 -0.000107988 +6959 6.91577 -0.000124348 -0.000107882 +6960 6.91674 -0.000124244 -0.000107777 +6961 6.91770 -0.000124140 -0.000107672 +6962 6.91867 -0.000124036 -0.000107567 +6963 6.91963 -0.000123932 -0.000107462 +6964 6.92060 -0.000123829 -0.000107357 +6965 6.92156 -0.000123725 -0.000107252 +6966 6.92253 -0.000123622 -0.000107147 +6967 6.92349 -0.000123518 -0.000107043 +6968 6.92446 -0.000123415 -0.000106938 +6969 6.92542 -0.000123312 -0.000106834 +6970 6.92639 -0.000123209 -0.000106730 +6971 6.92735 -0.000123106 -0.000106626 +6972 6.92832 -0.000123003 -0.000106522 +6973 6.92928 -0.000122900 -0.000106418 +6974 6.93025 -0.000122798 -0.000106315 +6975 6.93121 -0.000122695 -0.000106211 +6976 6.93218 -0.000122593 -0.000106108 +6977 6.93314 -0.000122490 -0.000106004 +6978 6.93411 -0.000122388 -0.000105901 +6979 6.93507 -0.000122286 -0.000105798 +6980 6.93604 -0.000122184 -0.000105695 +6981 6.93700 -0.000122082 -0.000105592 +6982 6.93797 -0.000121980 -0.000105489 +6983 6.93893 -0.000121878 -0.000105387 +6984 6.93990 -0.000121777 -0.000105284 +6985 6.94086 -0.000121675 -0.000105182 +6986 6.94183 -0.000121574 -0.000105079 +6987 6.94279 -0.000121472 -0.000104977 +6988 6.94376 -0.000121371 -0.000104875 +6989 6.94472 -0.000121270 -0.000104773 +6990 6.94569 -0.000121169 -0.000104671 +6991 6.94665 -0.000121068 -0.000104569 +6992 6.94762 -0.000120967 -0.000104468 +6993 6.94858 -0.000120866 -0.000104366 +6994 6.94955 -0.000120766 -0.000104265 +6995 6.95051 -0.000120665 -0.000104164 +6996 6.95148 -0.000120565 -0.000104062 +6997 6.95244 -0.000120464 -0.000103961 +6998 6.95341 -0.000120364 -0.000103860 +6999 6.95437 -0.000120264 -0.000103759 +7000 6.95534 -0.000120164 -0.000103659 +7001 6.95630 -0.000120064 -0.000103558 +7002 6.95727 -0.000119964 -0.000103458 +7003 6.95823 -0.000119864 -0.000103357 +7004 6.95920 -0.000119764 -0.000103257 +7005 6.96016 -0.000119665 -0.000103157 +7006 6.96113 -0.000119565 -0.000103057 +7007 6.96209 -0.000119466 -0.000102957 +7008 6.96306 -0.000119366 -0.000102857 +7009 6.96402 -0.000119267 -0.000102757 +7010 6.96499 -0.000119168 -0.000102658 +7011 6.96595 -0.000119069 -0.000102558 +7012 6.96692 -0.000118970 -0.000102459 +7013 6.96788 -0.000118871 -0.000102359 +7014 6.96885 -0.000118773 -0.000102260 +7015 6.96981 -0.000118674 -0.000102161 +7016 6.97078 -0.000118575 -0.000102062 +7017 6.97174 -0.000118477 -0.000101963 +7018 6.97271 -0.000118379 -0.000101865 +7019 6.97367 -0.000118280 -0.000101766 +7020 6.97464 -0.000118182 -0.000101667 +7021 6.97560 -0.000118084 -0.000101569 +7022 6.97657 -0.000117986 -0.000101471 +7023 6.97753 -0.000117888 -0.000101372 +7024 6.97850 -0.000117790 -0.000101274 +7025 6.97946 -0.000117693 -0.000101176 +7026 6.98043 -0.000117595 -0.000101078 +7027 6.98139 -0.000117498 -0.000100981 +7028 6.98236 -0.000117400 -0.000100883 +7029 6.98332 -0.000117303 -0.000100786 +7030 6.98429 -0.000117206 -0.000100688 +7031 6.98525 -0.000117109 -0.000100591 +7032 6.98622 -0.000117012 -0.000100494 +7033 6.98718 -0.000116915 -0.000100396 +7034 6.98815 -0.000116818 -0.000100299 +7035 6.98911 -0.000116721 -0.000100202 +7036 6.99008 -0.000116624 -0.000100106 +7037 6.99104 -0.000116528 -0.000100009 +7038 6.99201 -0.000116431 -0.000099912 +7039 6.99297 -0.000116335 -0.000099816 +7040 6.99394 -0.000116239 -0.000099720 +7041 6.99490 -0.000116143 -0.000099623 +7042 6.99587 -0.000116046 -0.000099527 +7043 6.99683 -0.000115950 -0.000099431 +7044 6.99780 -0.000115855 -0.000099335 +7045 6.99876 -0.000115759 -0.000099239 +7046 6.99973 -0.000115663 -0.000099144 +7047 7.00070 -0.000115567 -0.000099048 +7048 7.00166 -0.000115472 -0.000098952 +7049 7.00263 -0.000115376 -0.000098857 +7050 7.00359 -0.000115281 -0.000098762 +7051 7.00456 -0.000115186 -0.000098666 +7052 7.00552 -0.000115091 -0.000098571 +7053 7.00649 -0.000114996 -0.000098476 +7054 7.00745 -0.000114901 -0.000098381 +7055 7.00842 -0.000114806 -0.000098287 +7056 7.00938 -0.000114711 -0.000098192 +7057 7.01035 -0.000114616 -0.000098097 +7058 7.01131 -0.000114522 -0.000098003 +7059 7.01228 -0.000114427 -0.000097909 +7060 7.01324 -0.000114333 -0.000097814 +7061 7.01421 -0.000114238 -0.000097720 +7062 7.01517 -0.000114144 -0.000097626 +7063 7.01614 -0.000114050 -0.000097532 +7064 7.01710 -0.000113956 -0.000097438 +7065 7.01807 -0.000113862 -0.000097345 +7066 7.01903 -0.000113768 -0.000097251 +7067 7.02000 -0.000113674 -0.000097157 +7068 7.02096 -0.000113580 -0.000097064 +7069 7.02193 -0.000113487 -0.000096971 +7070 7.02289 -0.000113393 -0.000096877 +7071 7.02386 -0.000113300 -0.000096784 +7072 7.02482 -0.000113206 -0.000096691 +7073 7.02579 -0.000113113 -0.000096598 +7074 7.02675 -0.000113020 -0.000096505 +7075 7.02772 -0.000112927 -0.000096413 +7076 7.02868 -0.000112834 -0.000096320 +7077 7.02965 -0.000112741 -0.000096227 +7078 7.03061 -0.000112648 -0.000096135 +7079 7.03158 -0.000112555 -0.000096043 +7080 7.03254 -0.000112463 -0.000095951 +7081 7.03351 -0.000112370 -0.000095858 +7082 7.03447 -0.000112278 -0.000095766 +7083 7.03544 -0.000112185 -0.000095674 +7084 7.03640 -0.000112093 -0.000095583 +7085 7.03737 -0.000112001 -0.000095491 +7086 7.03833 -0.000111909 -0.000095399 +7087 7.03930 -0.000111817 -0.000095308 +7088 7.04026 -0.000111725 -0.000095216 +7089 7.04123 -0.000111633 -0.000095125 +7090 7.04219 -0.000111541 -0.000095034 +7091 7.04316 -0.000111449 -0.000094943 +7092 7.04412 -0.000111358 -0.000094852 +7093 7.04509 -0.000111266 -0.000094761 +7094 7.04605 -0.000111175 -0.000094670 +7095 7.04702 -0.000111084 -0.000094579 +7096 7.04798 -0.000110992 -0.000094489 +7097 7.04895 -0.000110901 -0.000094398 +7098 7.04991 -0.000110810 -0.000094308 +7099 7.05088 -0.000110719 -0.000094217 +7100 7.05184 -0.000110628 -0.000094127 +7101 7.05281 -0.000110538 -0.000094037 +7102 7.05377 -0.000110447 -0.000093947 +7103 7.05474 -0.000110356 -0.000093857 +7104 7.05570 -0.000110266 -0.000093767 +7105 7.05667 -0.000110175 -0.000093678 +7106 7.05763 -0.000110085 -0.000093588 +7107 7.05860 -0.000109995 -0.000093498 +7108 7.05956 -0.000109904 -0.000093409 +7109 7.06053 -0.000109814 -0.000093320 +7110 7.06149 -0.000109724 -0.000093230 +7111 7.06246 -0.000109634 -0.000093141 +7112 7.06342 -0.000109545 -0.000093052 +7113 7.06439 -0.000109455 -0.000092963 +7114 7.06535 -0.000109365 -0.000092874 +7115 7.06632 -0.000109276 -0.000092786 +7116 7.06728 -0.000109186 -0.000092697 +7117 7.06825 -0.000109097 -0.000092609 +7118 7.06921 -0.000109007 -0.000092520 +7119 7.07018 -0.000108918 -0.000092432 +7120 7.07114 -0.000108829 -0.000092343 +7121 7.07211 -0.000108740 -0.000092255 +7122 7.07307 -0.000108651 -0.000092167 +7123 7.07404 -0.000108562 -0.000092079 +7124 7.07500 -0.000108473 -0.000091991 +7125 7.07597 -0.000108384 -0.000091904 +7126 7.07693 -0.000108296 -0.000091816 +7127 7.07790 -0.000108207 -0.000091728 +7128 7.07886 -0.000108119 -0.000091641 +7129 7.07983 -0.000108030 -0.000091553 +7130 7.08079 -0.000107942 -0.000091466 +7131 7.08176 -0.000107854 -0.000091379 +7132 7.08272 -0.000107766 -0.000091292 +7133 7.08369 -0.000107678 -0.000091205 +7134 7.08465 -0.000107590 -0.000091118 +7135 7.08562 -0.000107502 -0.000091031 +7136 7.08658 -0.000107414 -0.000090944 +7137 7.08755 -0.000107326 -0.000090858 +7138 7.08851 -0.000107239 -0.000090771 +7139 7.08948 -0.000107151 -0.000090685 +7140 7.09044 -0.000107064 -0.000090598 +7141 7.09141 -0.000106976 -0.000090512 +7142 7.09237 -0.000106889 -0.000090426 +7143 7.09334 -0.000106802 -0.000090340 +7144 7.09430 -0.000106714 -0.000090254 +7145 7.09527 -0.000106627 -0.000090168 +7146 7.09623 -0.000106540 -0.000090082 +7147 7.09720 -0.000106454 -0.000089996 +7148 7.09816 -0.000106367 -0.000089911 +7149 7.09913 -0.000106280 -0.000089825 +7150 7.10010 -0.000106193 -0.000089740 +7151 7.10106 -0.000106107 -0.000089654 +7152 7.10203 -0.000106020 -0.000089569 +7153 7.10299 -0.000105934 -0.000089484 +7154 7.10396 -0.000105848 -0.000089399 +7155 7.10492 -0.000105761 -0.000089314 +7156 7.10589 -0.000105675 -0.000089229 +7157 7.10685 -0.000105589 -0.000089144 +7158 7.10782 -0.000105503 -0.000089060 +7159 7.10878 -0.000105417 -0.000088975 +7160 7.10975 -0.000105331 -0.000088890 +7161 7.11071 -0.000105246 -0.000088806 +7162 7.11168 -0.000105160 -0.000088722 +7163 7.11264 -0.000105074 -0.000088638 +7164 7.11361 -0.000104989 -0.000088553 +7165 7.11457 -0.000104904 -0.000088469 +7166 7.11554 -0.000104818 -0.000088385 +7167 7.11650 -0.000104733 -0.000088301 +7168 7.11747 -0.000104648 -0.000088218 +7169 7.11843 -0.000104563 -0.000088134 +7170 7.11940 -0.000104478 -0.000088050 +7171 7.12036 -0.000104393 -0.000087967 +7172 7.12133 -0.000104308 -0.000087884 +7173 7.12229 -0.000104223 -0.000087800 +7174 7.12326 -0.000104138 -0.000087717 +7175 7.12422 -0.000104054 -0.000087634 +7176 7.12519 -0.000103969 -0.000087551 +7177 7.12615 -0.000103885 -0.000087468 +7178 7.12712 -0.000103800 -0.000087385 +7179 7.12808 -0.000103716 -0.000087302 +7180 7.12905 -0.000103632 -0.000087219 +7181 7.13001 -0.000103548 -0.000087137 +7182 7.13098 -0.000103464 -0.000087054 +7183 7.13194 -0.000103380 -0.000086972 +7184 7.13291 -0.000103296 -0.000086890 +7185 7.13387 -0.000103212 -0.000086807 +7186 7.13484 -0.000103128 -0.000086725 +7187 7.13580 -0.000103045 -0.000086643 +7188 7.13677 -0.000102961 -0.000086561 +7189 7.13773 -0.000102878 -0.000086479 +7190 7.13870 -0.000102794 -0.000086397 +7191 7.13966 -0.000102711 -0.000086316 +7192 7.14063 -0.000102628 -0.000086234 +7193 7.14159 -0.000102544 -0.000086153 +7194 7.14256 -0.000102461 -0.000086071 +7195 7.14352 -0.000102378 -0.000085990 +7196 7.14449 -0.000102295 -0.000085908 +7197 7.14545 -0.000102212 -0.000085827 +7198 7.14642 -0.000102130 -0.000085746 +7199 7.14738 -0.000102047 -0.000085665 +7200 7.14835 -0.000101964 -0.000085584 +7201 7.14931 -0.000101882 -0.000085503 +7202 7.15028 -0.000101799 -0.000085423 +7203 7.15124 -0.000101717 -0.000085342 +7204 7.15221 -0.000101635 -0.000085261 +7205 7.15317 -0.000101552 -0.000085181 +7206 7.15414 -0.000101470 -0.000085101 +7207 7.15510 -0.000101388 -0.000085020 +7208 7.15607 -0.000101306 -0.000084940 +7209 7.15703 -0.000101224 -0.000084860 +7210 7.15800 -0.000101142 -0.000084780 +7211 7.15896 -0.000101061 -0.000084700 +7212 7.15993 -0.000100979 -0.000084620 +7213 7.16089 -0.000100897 -0.000084540 +7214 7.16186 -0.000100816 -0.000084460 +7215 7.16282 -0.000100734 -0.000084381 +7216 7.16379 -0.000100653 -0.000084301 +7217 7.16475 -0.000100571 -0.000084222 +7218 7.16572 -0.000100490 -0.000084143 +7219 7.16668 -0.000100409 -0.000084063 +7220 7.16765 -0.000100328 -0.000083984 +7221 7.16861 -0.000100247 -0.000083905 +7222 7.16958 -0.000100166 -0.000083826 +7223 7.17054 -0.000100085 -0.000083747 +7224 7.17151 -0.000100004 -0.000083668 +7225 7.17247 -0.000099924 -0.000083589 +7226 7.17344 -0.000099843 -0.000083511 +7227 7.17440 -0.000099763 -0.000083432 +7228 7.17537 -0.000099682 -0.000083354 +7229 7.17633 -0.000099602 -0.000083275 +7230 7.17730 -0.000099521 -0.000083197 +7231 7.17826 -0.000099441 -0.000083118 +7232 7.17923 -0.000099361 -0.000083040 +7233 7.18019 -0.000099281 -0.000082962 +7234 7.18116 -0.000099201 -0.000082884 +7235 7.18212 -0.000099121 -0.000082806 +7236 7.18309 -0.000099041 -0.000082728 +7237 7.18405 -0.000098961 -0.000082651 +7238 7.18502 -0.000098881 -0.000082573 +7239 7.18598 -0.000098802 -0.000082495 +7240 7.18695 -0.000098722 -0.000082418 +7241 7.18791 -0.000098643 -0.000082340 +7242 7.18888 -0.000098563 -0.000082263 +7243 7.18984 -0.000098484 -0.000082186 +7244 7.19081 -0.000098405 -0.000082109 +7245 7.19177 -0.000098325 -0.000082032 +7246 7.19274 -0.000098246 -0.000081955 +7247 7.19370 -0.000098167 -0.000081878 +7248 7.19467 -0.000098088 -0.000081801 +7249 7.19563 -0.000098009 -0.000081724 +7250 7.19660 -0.000097931 -0.000081647 +7251 7.19756 -0.000097852 -0.000081571 +7252 7.19853 -0.000097773 -0.000081494 +7253 7.19949 -0.000097695 -0.000081418 +7254 7.20046 -0.000097616 -0.000081341 +7255 7.20143 -0.000097538 -0.000081265 +7256 7.20239 -0.000097459 -0.000081189 +7257 7.20336 -0.000097381 -0.000081113 +7258 7.20432 -0.000097303 -0.000081037 +7259 7.20529 -0.000097224 -0.000080961 +7260 7.20625 -0.000097146 -0.000080885 +7261 7.20722 -0.000097068 -0.000080809 +7262 7.20818 -0.000096990 -0.000080734 +7263 7.20915 -0.000096912 -0.000080658 +7264 7.21011 -0.000096835 -0.000080582 +7265 7.21108 -0.000096757 -0.000080507 +7266 7.21204 -0.000096679 -0.000080432 +7267 7.21301 -0.000096602 -0.000080356 +7268 7.21397 -0.000096524 -0.000080281 +7269 7.21494 -0.000096447 -0.000080206 +7270 7.21590 -0.000096369 -0.000080131 +7271 7.21687 -0.000096292 -0.000080056 +7272 7.21783 -0.000096215 -0.000079981 +7273 7.21880 -0.000096138 -0.000079906 +7274 7.21976 -0.000096061 -0.000079831 +7275 7.22073 -0.000095984 -0.000079757 +7276 7.22169 -0.000095907 -0.000079682 +7277 7.22266 -0.000095830 -0.000079608 +7278 7.22362 -0.000095753 -0.000079533 +7279 7.22459 -0.000095676 -0.000079459 +7280 7.22555 -0.000095600 -0.000079385 +7281 7.22652 -0.000095523 -0.000079310 +7282 7.22748 -0.000095447 -0.000079236 +7283 7.22845 -0.000095370 -0.000079162 +7284 7.22941 -0.000095294 -0.000079088 +7285 7.23038 -0.000095218 -0.000079015 +7286 7.23134 -0.000095141 -0.000078941 +7287 7.23231 -0.000095065 -0.000078867 +7288 7.23327 -0.000094989 -0.000078793 +7289 7.23424 -0.000094913 -0.000078720 +7290 7.23520 -0.000094837 -0.000078646 +7291 7.23617 -0.000094761 -0.000078573 +7292 7.23713 -0.000094685 -0.000078500 +7293 7.23810 -0.000094610 -0.000078426 +7294 7.23906 -0.000094534 -0.000078353 +7295 7.24003 -0.000094459 -0.000078280 +7296 7.24099 -0.000094383 -0.000078207 +7297 7.24196 -0.000094308 -0.000078134 +7298 7.24292 -0.000094232 -0.000078061 +7299 7.24389 -0.000094157 -0.000077989 +7300 7.24485 -0.000094082 -0.000077916 +7301 7.24582 -0.000094007 -0.000077843 +7302 7.24678 -0.000093931 -0.000077771 +7303 7.24775 -0.000093856 -0.000077698 +7304 7.24871 -0.000093781 -0.000077626 +7305 7.24968 -0.000093707 -0.000077554 +7306 7.25064 -0.000093632 -0.000077482 +7307 7.25161 -0.000093557 -0.000077409 +7308 7.25257 -0.000093482 -0.000077337 +7309 7.25354 -0.000093408 -0.000077265 +7310 7.25450 -0.000093333 -0.000077193 +7311 7.25547 -0.000093259 -0.000077122 +7312 7.25643 -0.000093184 -0.000077050 +7313 7.25740 -0.000093110 -0.000076978 +7314 7.25836 -0.000093036 -0.000076906 +7315 7.25933 -0.000092962 -0.000076835 +7316 7.26029 -0.000092888 -0.000076763 +7317 7.26126 -0.000092813 -0.000076692 +7318 7.26222 -0.000092740 -0.000076621 +7319 7.26319 -0.000092666 -0.000076550 +7320 7.26415 -0.000092592 -0.000076478 +7321 7.26512 -0.000092518 -0.000076407 +7322 7.26608 -0.000092444 -0.000076336 +7323 7.26705 -0.000092371 -0.000076265 +7324 7.26801 -0.000092297 -0.000076194 +7325 7.26898 -0.000092224 -0.000076124 +7326 7.26994 -0.000092150 -0.000076053 +7327 7.27091 -0.000092077 -0.000075982 +7328 7.27187 -0.000092004 -0.000075912 +7329 7.27284 -0.000091930 -0.000075841 +7330 7.27380 -0.000091857 -0.000075771 +7331 7.27477 -0.000091784 -0.000075701 +7332 7.27573 -0.000091711 -0.000075630 +7333 7.27670 -0.000091638 -0.000075560 +7334 7.27766 -0.000091565 -0.000075490 +7335 7.27863 -0.000091492 -0.000075420 +7336 7.27959 -0.000091420 -0.000075350 +7337 7.28056 -0.000091347 -0.000075280 +7338 7.28152 -0.000091274 -0.000075210 +7339 7.28249 -0.000091202 -0.000075141 +7340 7.28345 -0.000091129 -0.000075071 +7341 7.28442 -0.000091057 -0.000075001 +7342 7.28538 -0.000090985 -0.000074932 +7343 7.28635 -0.000090912 -0.000074862 +7344 7.28731 -0.000090840 -0.000074793 +7345 7.28828 -0.000090768 -0.000074724 +7346 7.28924 -0.000090696 -0.000074654 +7347 7.29021 -0.000090624 -0.000074585 +7348 7.29117 -0.000090552 -0.000074516 +7349 7.29214 -0.000090480 -0.000074447 +7350 7.29310 -0.000090408 -0.000074378 +7351 7.29407 -0.000090336 -0.000074309 +7352 7.29503 -0.000090265 -0.000074241 +7353 7.29600 -0.000090193 -0.000074172 +7354 7.29696 -0.000090122 -0.000074103 +7355 7.29793 -0.000090050 -0.000074035 +7356 7.29889 -0.000089979 -0.000073966 +7357 7.29986 -0.000089907 -0.000073898 +7358 7.30083 -0.000089836 -0.000073830 +7359 7.30179 -0.000089765 -0.000073761 +7360 7.30276 -0.000089694 -0.000073693 +7361 7.30372 -0.000089623 -0.000073625 +7362 7.30469 -0.000089552 -0.000073557 +7363 7.30565 -0.000089481 -0.000073489 +7364 7.30662 -0.000089410 -0.000073421 +7365 7.30758 -0.000089339 -0.000073353 +7366 7.30855 -0.000089268 -0.000073285 +7367 7.30951 -0.000089197 -0.000073218 +7368 7.31048 -0.000089127 -0.000073150 +7369 7.31144 -0.000089056 -0.000073082 +7370 7.31241 -0.000088986 -0.000073015 +7371 7.31337 -0.000088915 -0.000072948 +7372 7.31434 -0.000088845 -0.000072880 +7373 7.31530 -0.000088775 -0.000072813 +7374 7.31627 -0.000088704 -0.000072746 +7375 7.31723 -0.000088634 -0.000072679 +7376 7.31820 -0.000088564 -0.000072611 +7377 7.31916 -0.000088494 -0.000072544 +7378 7.32013 -0.000088424 -0.000072478 +7379 7.32109 -0.000088354 -0.000072411 +7380 7.32206 -0.000088284 -0.000072344 +7381 7.32302 -0.000088215 -0.000072277 +7382 7.32399 -0.000088145 -0.000072211 +7383 7.32495 -0.000088075 -0.000072144 +7384 7.32592 -0.000088006 -0.000072078 +7385 7.32688 -0.000087936 -0.000072011 +7386 7.32785 -0.000087867 -0.000071945 +7387 7.32881 -0.000087797 -0.000071878 +7388 7.32978 -0.000087728 -0.000071812 +7389 7.33074 -0.000087659 -0.000071746 +7390 7.33171 -0.000087589 -0.000071680 +7391 7.33267 -0.000087520 -0.000071614 +7392 7.33364 -0.000087451 -0.000071548 +7393 7.33460 -0.000087382 -0.000071482 +7394 7.33557 -0.000087313 -0.000071416 +7395 7.33653 -0.000087244 -0.000071351 +7396 7.33750 -0.000087176 -0.000071285 +7397 7.33846 -0.000087107 -0.000071219 +7398 7.33943 -0.000087038 -0.000071154 +7399 7.34039 -0.000086969 -0.000071088 +7400 7.34136 -0.000086901 -0.000071023 +7401 7.34232 -0.000086832 -0.000070958 +7402 7.34329 -0.000086764 -0.000070892 +7403 7.34425 -0.000086696 -0.000070827 +7404 7.34522 -0.000086627 -0.000070762 +7405 7.34618 -0.000086559 -0.000070697 +7406 7.34715 -0.000086491 -0.000070632 +7407 7.34811 -0.000086423 -0.000070567 +7408 7.34908 -0.000086355 -0.000070502 +7409 7.35004 -0.000086287 -0.000070438 +7410 7.35101 -0.000086219 -0.000070373 +7411 7.35197 -0.000086151 -0.000070308 +7412 7.35294 -0.000086083 -0.000070244 +7413 7.35390 -0.000086015 -0.000070179 +7414 7.35487 -0.000085947 -0.000070115 +7415 7.35583 -0.000085880 -0.000070050 +7416 7.35680 -0.000085812 -0.000069986 +7417 7.35776 -0.000085745 -0.000069922 +7418 7.35873 -0.000085677 -0.000069858 +7419 7.35969 -0.000085610 -0.000069794 +7420 7.36066 -0.000085543 -0.000069730 +7421 7.36162 -0.000085475 -0.000069666 +7422 7.36259 -0.000085408 -0.000069602 +7423 7.36355 -0.000085341 -0.000069538 +7424 7.36452 -0.000085274 -0.000069474 +7425 7.36548 -0.000085207 -0.000069410 +7426 7.36645 -0.000085140 -0.000069347 +7427 7.36741 -0.000085073 -0.000069283 +7428 7.36838 -0.000085006 -0.000069220 +7429 7.36934 -0.000084939 -0.000069156 +7430 7.37031 -0.000084873 -0.000069093 +7431 7.37127 -0.000084806 -0.000069030 +7432 7.37224 -0.000084740 -0.000068966 +7433 7.37320 -0.000084673 -0.000068903 +7434 7.37417 -0.000084607 -0.000068840 +7435 7.37513 -0.000084540 -0.000068777 +7436 7.37610 -0.000084474 -0.000068714 +7437 7.37706 -0.000084408 -0.000068651 +7438 7.37803 -0.000084341 -0.000068588 +7439 7.37899 -0.000084275 -0.000068526 +7440 7.37996 -0.000084209 -0.000068463 +7441 7.38092 -0.000084143 -0.000068400 +7442 7.38189 -0.000084077 -0.000068338 +7443 7.38285 -0.000084011 -0.000068275 +7444 7.38382 -0.000083945 -0.000068213 +7445 7.38478 -0.000083879 -0.000068150 +7446 7.38575 -0.000083814 -0.000068088 +7447 7.38671 -0.000083748 -0.000068026 +7448 7.38768 -0.000083682 -0.000067964 +7449 7.38864 -0.000083617 -0.000067902 +7450 7.38961 -0.000083551 -0.000067840 +7451 7.39057 -0.000083486 -0.000067778 +7452 7.39154 -0.000083421 -0.000067716 +7453 7.39250 -0.000083355 -0.000067654 +7454 7.39347 -0.000083290 -0.000067592 +7455 7.39443 -0.000083225 -0.000067530 +7456 7.39540 -0.000083160 -0.000067469 +7457 7.39636 -0.000083095 -0.000067407 +7458 7.39733 -0.000083029 -0.000067346 +7459 7.39829 -0.000082965 -0.000067284 +7460 7.39926 -0.000082900 -0.000067223 +7461 7.40023 -0.000082835 -0.000067161 +7462 7.40119 -0.000082770 -0.000067100 +7463 7.40216 -0.000082705 -0.000067039 +7464 7.40312 -0.000082641 -0.000066978 +7465 7.40409 -0.000082576 -0.000066917 +7466 7.40505 -0.000082511 -0.000066856 +7467 7.40602 -0.000082447 -0.000066795 +7468 7.40698 -0.000082383 -0.000066734 +7469 7.40795 -0.000082318 -0.000066673 +7470 7.40891 -0.000082254 -0.000066612 +7471 7.40988 -0.000082190 -0.000066551 +7472 7.41084 -0.000082125 -0.000066491 +7473 7.41181 -0.000082061 -0.000066430 +7474 7.41277 -0.000081997 -0.000066370 +7475 7.41374 -0.000081933 -0.000066309 +7476 7.41470 -0.000081869 -0.000066249 +7477 7.41567 -0.000081805 -0.000066188 +7478 7.41663 -0.000081741 -0.000066128 +7479 7.41760 -0.000081678 -0.000066068 +7480 7.41856 -0.000081614 -0.000066008 +7481 7.41953 -0.000081550 -0.000065948 +7482 7.42049 -0.000081487 -0.000065888 +7483 7.42146 -0.000081423 -0.000065828 +7484 7.42242 -0.000081360 -0.000065768 +7485 7.42339 -0.000081296 -0.000065708 +7486 7.42435 -0.000081233 -0.000065648 +7487 7.42532 -0.000081169 -0.000065589 +7488 7.42628 -0.000081106 -0.000065529 +7489 7.42725 -0.000081043 -0.000065469 +7490 7.42821 -0.000080980 -0.000065410 +7491 7.42918 -0.000080917 -0.000065350 +7492 7.43014 -0.000080854 -0.000065291 +7493 7.43111 -0.000080791 -0.000065232 +7494 7.43207 -0.000080728 -0.000065172 +7495 7.43304 -0.000080665 -0.000065113 +7496 7.43400 -0.000080602 -0.000065054 +7497 7.43497 -0.000080539 -0.000064995 +7498 7.43593 -0.000080477 -0.000064936 +7499 7.43690 -0.000080414 -0.000064877 +7500 7.43786 -0.000080351 -0.000064818 +7501 7.43883 -0.000080289 -0.000064759 +7502 7.43979 -0.000080226 -0.000064701 +7503 7.44076 -0.000080164 -0.000064642 +7504 7.44172 -0.000080102 -0.000064583 +7505 7.44269 -0.000080039 -0.000064525 +7506 7.44365 -0.000079977 -0.000064466 +7507 7.44462 -0.000079915 -0.000064408 +7508 7.44558 -0.000079853 -0.000064349 +7509 7.44655 -0.000079791 -0.000064291 +7510 7.44751 -0.000079729 -0.000064233 +7511 7.44848 -0.000079667 -0.000064174 +7512 7.44944 -0.000079605 -0.000064116 +7513 7.45041 -0.000079543 -0.000064058 +7514 7.45137 -0.000079481 -0.000064000 +7515 7.45234 -0.000079420 -0.000063942 +7516 7.45330 -0.000079358 -0.000063884 +7517 7.45427 -0.000079296 -0.000063826 +7518 7.45523 -0.000079235 -0.000063768 +7519 7.45620 -0.000079173 -0.000063711 +7520 7.45716 -0.000079112 -0.000063653 +7521 7.45813 -0.000079050 -0.000063595 +7522 7.45909 -0.000078989 -0.000063538 +7523 7.46006 -0.000078928 -0.000063480 +7524 7.46102 -0.000078866 -0.000063423 +7525 7.46199 -0.000078805 -0.000063365 +7526 7.46295 -0.000078744 -0.000063308 +7527 7.46392 -0.000078683 -0.000063251 +7528 7.46488 -0.000078622 -0.000063194 +7529 7.46585 -0.000078561 -0.000063136 +7530 7.46681 -0.000078500 -0.000063079 +7531 7.46778 -0.000078439 -0.000063022 +7532 7.46874 -0.000078379 -0.000062965 +7533 7.46971 -0.000078318 -0.000062908 +7534 7.47067 -0.000078257 -0.000062851 +7535 7.47164 -0.000078196 -0.000062795 +7536 7.47260 -0.000078136 -0.000062738 +7537 7.47357 -0.000078075 -0.000062681 +7538 7.47453 -0.000078015 -0.000062625 +7539 7.47550 -0.000077955 -0.000062568 +7540 7.47646 -0.000077894 -0.000062512 +7541 7.47743 -0.000077834 -0.000062455 +7542 7.47839 -0.000077774 -0.000062399 +7543 7.47936 -0.000077713 -0.000062342 +7544 7.48032 -0.000077653 -0.000062286 +7545 7.48129 -0.000077593 -0.000062230 +7546 7.48225 -0.000077533 -0.000062174 +7547 7.48322 -0.000077473 -0.000062118 +7548 7.48418 -0.000077413 -0.000062062 +7549 7.48515 -0.000077353 -0.000062006 +7550 7.48611 -0.000077294 -0.000061950 +7551 7.48708 -0.000077234 -0.000061894 +7552 7.48804 -0.000077174 -0.000061838 +7553 7.48901 -0.000077115 -0.000061782 +7554 7.48997 -0.000077055 -0.000061726 +7555 7.49094 -0.000076995 -0.000061671 +7556 7.49190 -0.000076936 -0.000061615 +7557 7.49287 -0.000076876 -0.000061560 +7558 7.49383 -0.000076817 -0.000061504 +7559 7.49480 -0.000076758 -0.000061449 +7560 7.49576 -0.000076698 -0.000061393 +7561 7.49673 -0.000076639 -0.000061338 +7562 7.49769 -0.000076580 -0.000061283 +7563 7.49866 -0.000076521 -0.000061228 +7564 7.49962 -0.000076462 -0.000061173 +7565 7.50059 -0.000076403 -0.000061117 +7566 7.50156 -0.000076344 -0.000061062 +7567 7.50252 -0.000076285 -0.000061008 +7568 7.50349 -0.000076226 -0.000060953 +7569 7.50445 -0.000076167 -0.000060898 +7570 7.50542 -0.000076109 -0.000060843 +7571 7.50638 -0.000076050 -0.000060788 +7572 7.50735 -0.000075991 -0.000060734 +7573 7.50831 -0.000075933 -0.000060679 +7574 7.50928 -0.000075874 -0.000060624 +7575 7.51024 -0.000075816 -0.000060570 +7576 7.51121 -0.000075757 -0.000060515 +7577 7.51217 -0.000075699 -0.000060461 +7578 7.51314 -0.000075641 -0.000060407 +7579 7.51410 -0.000075582 -0.000060352 +7580 7.51507 -0.000075524 -0.000060298 +7581 7.51603 -0.000075466 -0.000060244 +7582 7.51700 -0.000075408 -0.000060190 +7583 7.51796 -0.000075350 -0.000060136 +7584 7.51893 -0.000075292 -0.000060082 +7585 7.51989 -0.000075234 -0.000060028 +7586 7.52086 -0.000075176 -0.000059974 +7587 7.52182 -0.000075118 -0.000059920 +7588 7.52279 -0.000075060 -0.000059866 +7589 7.52375 -0.000075003 -0.000059813 +7590 7.52472 -0.000074945 -0.000059759 +7591 7.52568 -0.000074887 -0.000059705 +7592 7.52665 -0.000074830 -0.000059652 +7593 7.52761 -0.000074772 -0.000059598 +7594 7.52858 -0.000074715 -0.000059545 +7595 7.52954 -0.000074657 -0.000059491 +7596 7.53051 -0.000074600 -0.000059438 +7597 7.53147 -0.000074542 -0.000059385 +7598 7.53244 -0.000074485 -0.000059331 +7599 7.53340 -0.000074428 -0.000059278 +7600 7.53437 -0.000074371 -0.000059225 +7601 7.53533 -0.000074314 -0.000059172 +7602 7.53630 -0.000074256 -0.000059119 +7603 7.53726 -0.000074199 -0.000059066 +7604 7.53823 -0.000074142 -0.000059013 +7605 7.53919 -0.000074086 -0.000058960 +7606 7.54016 -0.000074029 -0.000058908 +7607 7.54112 -0.000073972 -0.000058855 +7608 7.54209 -0.000073915 -0.000058802 +7609 7.54305 -0.000073858 -0.000058749 +7610 7.54402 -0.000073802 -0.000058697 +7611 7.54498 -0.000073745 -0.000058644 +7612 7.54595 -0.000073689 -0.000058592 +7613 7.54691 -0.000073632 -0.000058539 +7614 7.54788 -0.000073576 -0.000058487 +7615 7.54884 -0.000073519 -0.000058435 +7616 7.54981 -0.000073463 -0.000058382 +7617 7.55077 -0.000073406 -0.000058330 +7618 7.55174 -0.000073350 -0.000058278 +7619 7.55270 -0.000073294 -0.000058226 +7620 7.55367 -0.000073238 -0.000058174 +7621 7.55463 -0.000073182 -0.000058122 +7622 7.55560 -0.000073126 -0.000058070 +7623 7.55656 -0.000073070 -0.000058018 +7624 7.55753 -0.000073014 -0.000057966 +7625 7.55849 -0.000072958 -0.000057914 +7626 7.55946 -0.000072902 -0.000057863 +7627 7.56042 -0.000072846 -0.000057811 +7628 7.56139 -0.000072790 -0.000057759 +7629 7.56235 -0.000072735 -0.000057708 +7630 7.56332 -0.000072679 -0.000057656 +7631 7.56428 -0.000072623 -0.000057605 +7632 7.56525 -0.000072568 -0.000057553 +7633 7.56621 -0.000072512 -0.000057502 +7634 7.56718 -0.000072457 -0.000057451 +7635 7.56814 -0.000072401 -0.000057399 +7636 7.56911 -0.000072346 -0.000057348 +7637 7.57007 -0.000072291 -0.000057297 +7638 7.57104 -0.000072235 -0.000057246 +7639 7.57200 -0.000072180 -0.000057195 +7640 7.57297 -0.000072125 -0.000057144 +7641 7.57393 -0.000072070 -0.000057093 +7642 7.57490 -0.000072015 -0.000057042 +7643 7.57586 -0.000071960 -0.000056991 +7644 7.57683 -0.000071905 -0.000056940 +7645 7.57779 -0.000071850 -0.000056890 +7646 7.57876 -0.000071795 -0.000056839 +7647 7.57972 -0.000071740 -0.000056788 +7648 7.58069 -0.000071685 -0.000056738 +7649 7.58165 -0.000071631 -0.000056687 +7650 7.58262 -0.000071576 -0.000056637 +7651 7.58358 -0.000071521 -0.000056586 +7652 7.58455 -0.000071467 -0.000056536 +7653 7.58551 -0.000071412 -0.000056486 +7654 7.58648 -0.000071358 -0.000056435 +7655 7.58744 -0.000071303 -0.000056385 +7656 7.58841 -0.000071249 -0.000056335 +7657 7.58937 -0.000071194 -0.000056285 +7658 7.59034 -0.000071140 -0.000056235 +7659 7.59130 -0.000071086 -0.000056185 +7660 7.59227 -0.000071032 -0.000056135 +7661 7.59323 -0.000070978 -0.000056085 +7662 7.59420 -0.000070924 -0.000056035 +7663 7.59516 -0.000070869 -0.000055985 +7664 7.59613 -0.000070815 -0.000055935 +7665 7.59709 -0.000070762 -0.000055886 +7666 7.59806 -0.000070708 -0.000055836 +7667 7.59902 -0.000070654 -0.000055786 +7668 7.59999 -0.000070600 -0.000055737 +7669 7.60096 -0.000070546 -0.000055687 +7670 7.60192 -0.000070492 -0.000055638 +7671 7.60289 -0.000070439 -0.000055588 +7672 7.60385 -0.000070385 -0.000055539 +7673 7.60482 -0.000070332 -0.000055490 +7674 7.60578 -0.000070278 -0.000055441 +7675 7.60675 -0.000070225 -0.000055391 +7676 7.60771 -0.000070171 -0.000055342 +7677 7.60868 -0.000070118 -0.000055293 +7678 7.60964 -0.000070064 -0.000055244 +7679 7.61061 -0.000070011 -0.000055195 +7680 7.61157 -0.000069958 -0.000055146 +7681 7.61254 -0.000069905 -0.000055097 +7682 7.61350 -0.000069852 -0.000055048 +7683 7.61447 -0.000069798 -0.000054999 +7684 7.61543 -0.000069745 -0.000054951 +7685 7.61640 -0.000069692 -0.000054902 +7686 7.61736 -0.000069639 -0.000054853 +7687 7.61833 -0.000069587 -0.000054805 +7688 7.61929 -0.000069534 -0.000054756 +7689 7.62026 -0.000069481 -0.000054707 +7690 7.62122 -0.000069428 -0.000054659 +7691 7.62219 -0.000069375 -0.000054611 +7692 7.62315 -0.000069323 -0.000054562 +7693 7.62412 -0.000069270 -0.000054514 +7694 7.62508 -0.000069217 -0.000054466 +7695 7.62605 -0.000069165 -0.000054417 +7696 7.62701 -0.000069112 -0.000054369 +7697 7.62798 -0.000069060 -0.000054321 +7698 7.62894 -0.000069008 -0.000054273 +7699 7.62991 -0.000068955 -0.000054225 +7700 7.63087 -0.000068903 -0.000054177 +7701 7.63184 -0.000068851 -0.000054129 +7702 7.63280 -0.000068798 -0.000054081 +7703 7.63377 -0.000068746 -0.000054033 +7704 7.63473 -0.000068694 -0.000053985 +7705 7.63570 -0.000068642 -0.000053938 +7706 7.63666 -0.000068590 -0.000053890 +7707 7.63763 -0.000068538 -0.000053842 +7708 7.63859 -0.000068486 -0.000053795 +7709 7.63956 -0.000068434 -0.000053747 +7710 7.64052 -0.000068382 -0.000053700 +7711 7.64149 -0.000068331 -0.000053652 +7712 7.64245 -0.000068279 -0.000053605 +7713 7.64342 -0.000068227 -0.000053558 +7714 7.64438 -0.000068175 -0.000053510 +7715 7.64535 -0.000068124 -0.000053463 +7716 7.64631 -0.000068072 -0.000053416 +7717 7.64728 -0.000068021 -0.000053369 +7718 7.64824 -0.000067969 -0.000053321 +7719 7.64921 -0.000067918 -0.000053274 +7720 7.65017 -0.000067866 -0.000053227 +7721 7.65114 -0.000067815 -0.000053180 +7722 7.65210 -0.000067764 -0.000053133 +7723 7.65307 -0.000067713 -0.000053087 +7724 7.65403 -0.000067661 -0.000053040 +7725 7.65500 -0.000067610 -0.000052993 +7726 7.65596 -0.000067559 -0.000052946 +7727 7.65693 -0.000067508 -0.000052900 +7728 7.65789 -0.000067457 -0.000052853 +7729 7.65886 -0.000067406 -0.000052806 +7730 7.65982 -0.000067355 -0.000052760 +7731 7.66079 -0.000067304 -0.000052713 +7732 7.66175 -0.000067253 -0.000052667 +7733 7.66272 -0.000067202 -0.000052620 +7734 7.66368 -0.000067152 -0.000052574 +7735 7.66465 -0.000067101 -0.000052528 +7736 7.66561 -0.000067050 -0.000052481 +7737 7.66658 -0.000067000 -0.000052435 +7738 7.66754 -0.000066949 -0.000052389 +7739 7.66851 -0.000066899 -0.000052343 +7740 7.66947 -0.000066848 -0.000052297 +7741 7.67044 -0.000066798 -0.000052251 +7742 7.67140 -0.000066747 -0.000052205 +7743 7.67237 -0.000066697 -0.000052159 +7744 7.67333 -0.000066647 -0.000052113 +7745 7.67430 -0.000066596 -0.000052067 +7746 7.67526 -0.000066546 -0.000052021 +7747 7.67623 -0.000066496 -0.000051975 +7748 7.67719 -0.000066446 -0.000051930 +7749 7.67816 -0.000066396 -0.000051884 +7750 7.67912 -0.000066346 -0.000051838 +7751 7.68009 -0.000066296 -0.000051793 +7752 7.68105 -0.000066246 -0.000051747 +7753 7.68202 -0.000066196 -0.000051702 +7754 7.68298 -0.000066146 -0.000051656 +7755 7.68395 -0.000066096 -0.000051611 +7756 7.68491 -0.000066046 -0.000051566 +7757 7.68588 -0.000065997 -0.000051520 +7758 7.68684 -0.000065947 -0.000051475 +7759 7.68781 -0.000065897 -0.000051430 +7760 7.68877 -0.000065848 -0.000051385 +7761 7.68974 -0.000065798 -0.000051340 +7762 7.69070 -0.000065748 -0.000051294 +7763 7.69167 -0.000065699 -0.000051249 +7764 7.69263 -0.000065650 -0.000051204 +7765 7.69360 -0.000065600 -0.000051160 +7766 7.69456 -0.000065551 -0.000051115 +7767 7.69553 -0.000065501 -0.000051070 +7768 7.69649 -0.000065452 -0.000051025 +7769 7.69746 -0.000065403 -0.000050980 +7770 7.69842 -0.000065354 -0.000050935 +7771 7.69939 -0.000065305 -0.000050891 +7772 7.70036 -0.000065256 -0.000050846 +7773 7.70132 -0.000065207 -0.000050802 +7774 7.70229 -0.000065158 -0.000050757 +7775 7.70325 -0.000065109 -0.000050713 +7776 7.70422 -0.000065060 -0.000050668 +7777 7.70518 -0.000065011 -0.000050624 +7778 7.70615 -0.000064962 -0.000050579 +7779 7.70711 -0.000064913 -0.000050535 +7780 7.70808 -0.000064864 -0.000050491 +7781 7.70904 -0.000064816 -0.000050447 +7782 7.71001 -0.000064767 -0.000050402 +7783 7.71097 -0.000064718 -0.000050358 +7784 7.71194 -0.000064670 -0.000050314 +7785 7.71290 -0.000064621 -0.000050270 +7786 7.71387 -0.000064573 -0.000050226 +7787 7.71483 -0.000064524 -0.000050182 +7788 7.71580 -0.000064476 -0.000050138 +7789 7.71676 -0.000064428 -0.000050094 +7790 7.71773 -0.000064379 -0.000050050 +7791 7.71869 -0.000064331 -0.000050007 +7792 7.71966 -0.000064283 -0.000049963 +7793 7.72062 -0.000064235 -0.000049919 +7794 7.72159 -0.000064186 -0.000049876 +7795 7.72255 -0.000064138 -0.000049832 +7796 7.72352 -0.000064090 -0.000049788 +7797 7.72448 -0.000064042 -0.000049745 +7798 7.72545 -0.000063994 -0.000049701 +7799 7.72641 -0.000063946 -0.000049658 +7800 7.72738 -0.000063898 -0.000049615 +7801 7.72834 -0.000063851 -0.000049571 +7802 7.72931 -0.000063803 -0.000049528 +7803 7.73027 -0.000063755 -0.000049485 +7804 7.73124 -0.000063707 -0.000049441 +7805 7.73220 -0.000063660 -0.000049398 +7806 7.73317 -0.000063612 -0.000049355 +7807 7.73413 -0.000063564 -0.000049312 +7808 7.73510 -0.000063517 -0.000049269 +7809 7.73606 -0.000063469 -0.000049226 +7810 7.73703 -0.000063422 -0.000049183 +7811 7.73799 -0.000063374 -0.000049140 +7812 7.73896 -0.000063327 -0.000049097 +7813 7.73992 -0.000063279 -0.000049054 +7814 7.74089 -0.000063232 -0.000049012 +7815 7.74185 -0.000063185 -0.000048969 +7816 7.74282 -0.000063138 -0.000048926 +7817 7.74378 -0.000063090 -0.000048883 +7818 7.74475 -0.000063043 -0.000048841 +7819 7.74571 -0.000062996 -0.000048798 +7820 7.74668 -0.000062949 -0.000048756 +7821 7.74764 -0.000062902 -0.000048713 +7822 7.74861 -0.000062855 -0.000048671 +7823 7.74957 -0.000062808 -0.000048628 +7824 7.75054 -0.000062761 -0.000048586 +7825 7.75150 -0.000062714 -0.000048544 +7826 7.75247 -0.000062668 -0.000048501 +7827 7.75343 -0.000062621 -0.000048459 +7828 7.75440 -0.000062574 -0.000048417 +7829 7.75536 -0.000062527 -0.000048375 +7830 7.75633 -0.000062481 -0.000048333 +7831 7.75729 -0.000062434 -0.000048291 +7832 7.75826 -0.000062387 -0.000048249 +7833 7.75922 -0.000062341 -0.000048207 +7834 7.76019 -0.000062294 -0.000048165 +7835 7.76115 -0.000062248 -0.000048123 +7836 7.76212 -0.000062201 -0.000048081 +7837 7.76308 -0.000062155 -0.000048039 +7838 7.76405 -0.000062109 -0.000047997 +7839 7.76501 -0.000062062 -0.000047955 +7840 7.76598 -0.000062016 -0.000047914 +7841 7.76694 -0.000061970 -0.000047872 +7842 7.76791 -0.000061924 -0.000047830 +7843 7.76887 -0.000061878 -0.000047789 +7844 7.76984 -0.000061832 -0.000047747 +7845 7.77080 -0.000061786 -0.000047706 +7846 7.77177 -0.000061740 -0.000047664 +7847 7.77273 -0.000061694 -0.000047623 +7848 7.77370 -0.000061648 -0.000047582 +7849 7.77466 -0.000061602 -0.000047540 +7850 7.77563 -0.000061556 -0.000047499 +7851 7.77659 -0.000061510 -0.000047458 +7852 7.77756 -0.000061464 -0.000047417 +7853 7.77852 -0.000061418 -0.000047375 +7854 7.77949 -0.000061373 -0.000047334 +7855 7.78045 -0.000061327 -0.000047293 +7856 7.78142 -0.000061282 -0.000047252 +7857 7.78238 -0.000061236 -0.000047211 +7858 7.78335 -0.000061190 -0.000047170 +7859 7.78431 -0.000061145 -0.000047129 +7860 7.78528 -0.000061099 -0.000047088 +7861 7.78624 -0.000061054 -0.000047048 +7862 7.78721 -0.000061009 -0.000047007 +7863 7.78817 -0.000060963 -0.000046966 +7864 7.78914 -0.000060918 -0.000046925 +7865 7.79010 -0.000060873 -0.000046885 +7866 7.79107 -0.000060827 -0.000046844 +7867 7.79203 -0.000060782 -0.000046803 +7868 7.79300 -0.000060737 -0.000046763 +7869 7.79396 -0.000060692 -0.000046722 +7870 7.79493 -0.000060647 -0.000046682 +7871 7.79589 -0.000060602 -0.000046641 +7872 7.79686 -0.000060557 -0.000046601 +7873 7.79782 -0.000060512 -0.000046561 +7874 7.79879 -0.000060467 -0.000046520 +7875 7.79975 -0.000060422 -0.000046480 +7876 7.80072 -0.000060377 -0.000046440 +7877 7.80169 -0.000060333 -0.000046400 +7878 7.80265 -0.000060288 -0.000046359 +7879 7.80362 -0.000060243 -0.000046319 +7880 7.80458 -0.000060198 -0.000046279 +7881 7.80555 -0.000060154 -0.000046239 +7882 7.80651 -0.000060109 -0.000046199 +7883 7.80748 -0.000060065 -0.000046159 +7884 7.80844 -0.000060020 -0.000046119 +7885 7.80941 -0.000059976 -0.000046080 +7886 7.81037 -0.000059931 -0.000046040 +7887 7.81134 -0.000059887 -0.000046000 +7888 7.81230 -0.000059842 -0.000045960 +7889 7.81327 -0.000059798 -0.000045920 +7890 7.81423 -0.000059754 -0.000045881 +7891 7.81520 -0.000059709 -0.000045841 +7892 7.81616 -0.000059665 -0.000045801 +7893 7.81713 -0.000059621 -0.000045762 +7894 7.81809 -0.000059577 -0.000045722 +7895 7.81906 -0.000059533 -0.000045683 +7896 7.82002 -0.000059489 -0.000045643 +7897 7.82099 -0.000059445 -0.000045604 +7898 7.82195 -0.000059401 -0.000045565 +7899 7.82292 -0.000059357 -0.000045525 +7900 7.82388 -0.000059313 -0.000045486 +7901 7.82485 -0.000059269 -0.000045447 +7902 7.82581 -0.000059225 -0.000045408 +7903 7.82678 -0.000059181 -0.000045368 +7904 7.82774 -0.000059138 -0.000045329 +7905 7.82871 -0.000059094 -0.000045290 +7906 7.82967 -0.000059050 -0.000045251 +7907 7.83064 -0.000059007 -0.000045212 +7908 7.83160 -0.000058963 -0.000045173 +7909 7.83257 -0.000058919 -0.000045134 +7910 7.83353 -0.000058876 -0.000045095 +7911 7.83450 -0.000058832 -0.000045056 +7912 7.83546 -0.000058789 -0.000045018 +7913 7.83643 -0.000058745 -0.000044979 +7914 7.83739 -0.000058702 -0.000044940 +7915 7.83836 -0.000058659 -0.000044901 +7916 7.83932 -0.000058615 -0.000044863 +7917 7.84029 -0.000058572 -0.000044824 +7918 7.84125 -0.000058529 -0.000044785 +7919 7.84222 -0.000058486 -0.000044747 +7920 7.84318 -0.000058442 -0.000044708 +7921 7.84415 -0.000058399 -0.000044670 +7922 7.84511 -0.000058356 -0.000044631 +7923 7.84608 -0.000058313 -0.000044593 +7924 7.84704 -0.000058270 -0.000044555 +7925 7.84801 -0.000058227 -0.000044516 +7926 7.84897 -0.000058184 -0.000044478 +7927 7.84994 -0.000058141 -0.000044440 +7928 7.85090 -0.000058098 -0.000044401 +7929 7.85187 -0.000058056 -0.000044363 +7930 7.85283 -0.000058013 -0.000044325 +7931 7.85380 -0.000057970 -0.000044287 +7932 7.85476 -0.000057927 -0.000044249 +7933 7.85573 -0.000057885 -0.000044211 +7934 7.85669 -0.000057842 -0.000044173 +7935 7.85766 -0.000057799 -0.000044135 +7936 7.85862 -0.000057757 -0.000044097 +7937 7.85959 -0.000057714 -0.000044059 +7938 7.86055 -0.000057672 -0.000044021 +7939 7.86152 -0.000057629 -0.000043983 +7940 7.86248 -0.000057587 -0.000043946 +7941 7.86345 -0.000057545 -0.000043908 +7942 7.86441 -0.000057502 -0.000043870 +7943 7.86538 -0.000057460 -0.000043833 +7944 7.86634 -0.000057418 -0.000043795 +7945 7.86731 -0.000057375 -0.000043757 +7946 7.86827 -0.000057333 -0.000043720 +7947 7.86924 -0.000057291 -0.000043682 +7948 7.87020 -0.000057249 -0.000043645 +7949 7.87117 -0.000057207 -0.000043607 +7950 7.87213 -0.000057165 -0.000043570 +7951 7.87310 -0.000057123 -0.000043533 +7952 7.87406 -0.000057081 -0.000043495 +7953 7.87503 -0.000057039 -0.000043458 +7954 7.87599 -0.000056997 -0.000043421 +7955 7.87696 -0.000056955 -0.000043383 +7956 7.87792 -0.000056913 -0.000043346 +7957 7.87889 -0.000056871 -0.000043309 +7958 7.87985 -0.000056829 -0.000043272 +7959 7.88082 -0.000056788 -0.000043235 +7960 7.88178 -0.000056746 -0.000043198 +7961 7.88275 -0.000056704 -0.000043161 +7962 7.88371 -0.000056663 -0.000043124 +7963 7.88468 -0.000056621 -0.000043087 +7964 7.88564 -0.000056580 -0.000043050 +7965 7.88661 -0.000056538 -0.000043013 +7966 7.88757 -0.000056496 -0.000042976 +7967 7.88854 -0.000056455 -0.000042940 +7968 7.88950 -0.000056414 -0.000042903 +7969 7.89047 -0.000056372 -0.000042866 +7970 7.89143 -0.000056331 -0.000042829 +7971 7.89240 -0.000056290 -0.000042793 +7972 7.89336 -0.000056248 -0.000042756 +7973 7.89433 -0.000056207 -0.000042720 +7974 7.89529 -0.000056166 -0.000042683 +7975 7.89626 -0.000056125 -0.000042647 +7976 7.89722 -0.000056084 -0.000042610 +7977 7.89819 -0.000056042 -0.000042574 +7978 7.89915 -0.000056001 -0.000042537 +7979 7.90012 -0.000055960 -0.000042501 +7980 7.90109 -0.000055919 -0.000042465 +7981 7.90205 -0.000055878 -0.000042428 +7982 7.90302 -0.000055837 -0.000042392 +7983 7.90398 -0.000055797 -0.000042356 +7984 7.90495 -0.000055756 -0.000042320 +7985 7.90591 -0.000055715 -0.000042283 +7986 7.90688 -0.000055674 -0.000042247 +7987 7.90784 -0.000055633 -0.000042211 +7988 7.90881 -0.000055593 -0.000042175 +7989 7.90977 -0.000055552 -0.000042139 +7990 7.91074 -0.000055511 -0.000042103 +7991 7.91170 -0.000055471 -0.000042067 +7992 7.91267 -0.000055430 -0.000042031 +7993 7.91363 -0.000055390 -0.000041996 +7994 7.91460 -0.000055349 -0.000041960 +7995 7.91556 -0.000055309 -0.000041924 +7996 7.91653 -0.000055268 -0.000041888 +7997 7.91749 -0.000055228 -0.000041852 +7998 7.91846 -0.000055187 -0.000041817 +7999 7.91942 -0.000055147 -0.000041781 +8000 7.92039 -0.000055107 -0.000041745 +8001 7.92135 -0.000055066 -0.000041710 +8002 7.92232 -0.000055026 -0.000041674 +8003 7.92328 -0.000054986 -0.000041639 +8004 7.92425 -0.000054946 -0.000041603 +8005 7.92521 -0.000054906 -0.000041568 +8006 7.92618 -0.000054866 -0.000041532 +8007 7.92714 -0.000054826 -0.000041497 +8008 7.92811 -0.000054785 -0.000041462 +8009 7.92907 -0.000054745 -0.000041426 +8010 7.93004 -0.000054706 -0.000041391 +8011 7.93100 -0.000054666 -0.000041356 +8012 7.93197 -0.000054626 -0.000041321 +8013 7.93293 -0.000054586 -0.000041286 +8014 7.93390 -0.000054546 -0.000041250 +8015 7.93486 -0.000054506 -0.000041215 +8016 7.93583 -0.000054466 -0.000041180 +8017 7.93679 -0.000054427 -0.000041145 +8018 7.93776 -0.000054387 -0.000041110 +8019 7.93872 -0.000054347 -0.000041075 +8020 7.93969 -0.000054308 -0.000041040 +8021 7.94065 -0.000054268 -0.000041005 +8022 7.94162 -0.000054229 -0.000040970 +8023 7.94258 -0.000054189 -0.000040936 +8024 7.94355 -0.000054150 -0.000040901 +8025 7.94451 -0.000054110 -0.000040866 +8026 7.94548 -0.000054071 -0.000040831 +8027 7.94644 -0.000054031 -0.000040797 +8028 7.94741 -0.000053992 -0.000040762 +8029 7.94837 -0.000053953 -0.000040727 +8030 7.94934 -0.000053913 -0.000040693 +8031 7.95030 -0.000053874 -0.000040658 +8032 7.95127 -0.000053835 -0.000040624 +8033 7.95223 -0.000053796 -0.000040589 +8034 7.95320 -0.000053757 -0.000040555 +8035 7.95416 -0.000053717 -0.000040520 +8036 7.95513 -0.000053678 -0.000040486 +8037 7.95609 -0.000053639 -0.000040452 +8038 7.95706 -0.000053600 -0.000040417 +8039 7.95802 -0.000053561 -0.000040383 +8040 7.95899 -0.000053522 -0.000040349 +8041 7.95995 -0.000053483 -0.000040314 +8042 7.96092 -0.000053445 -0.000040280 +8043 7.96188 -0.000053406 -0.000040246 +8044 7.96285 -0.000053367 -0.000040212 +8045 7.96381 -0.000053328 -0.000040178 +8046 7.96478 -0.000053289 -0.000040144 +8047 7.96574 -0.000053251 -0.000040110 +8048 7.96671 -0.000053212 -0.000040076 +8049 7.96767 -0.000053173 -0.000040042 +8050 7.96864 -0.000053135 -0.000040008 +8051 7.96960 -0.000053096 -0.000039974 +8052 7.97057 -0.000053057 -0.000039940 +8053 7.97153 -0.000053019 -0.000039906 +8054 7.97250 -0.000052980 -0.000039872 +8055 7.97346 -0.000052942 -0.000039839 +8056 7.97443 -0.000052904 -0.000039805 +8057 7.97539 -0.000052865 -0.000039771 +8058 7.97636 -0.000052827 -0.000039738 +8059 7.97732 -0.000052788 -0.000039704 +8060 7.97829 -0.000052750 -0.000039670 +8061 7.97925 -0.000052712 -0.000039637 +8062 7.98022 -0.000052674 -0.000039603 +8063 7.98118 -0.000052635 -0.000039570 +8064 7.98215 -0.000052597 -0.000039536 +8065 7.98311 -0.000052559 -0.000039503 +8066 7.98408 -0.000052521 -0.000039469 +8067 7.98504 -0.000052483 -0.000039436 +8068 7.98601 -0.000052445 -0.000039403 +8069 7.98697 -0.000052407 -0.000039369 +8070 7.98794 -0.000052369 -0.000039336 +8071 7.98890 -0.000052331 -0.000039303 +8072 7.98987 -0.000052293 -0.000039270 +8073 7.99083 -0.000052255 -0.000039236 +8074 7.99180 -0.000052217 -0.000039203 +8075 7.99276 -0.000052180 -0.000039170 +8076 7.99373 -0.000052142 -0.000039137 +8077 7.99469 -0.000052104 -0.000039104 +8078 7.99566 -0.000052066 -0.000039071 +8079 7.99662 -0.000052029 -0.000039038 +8080 7.99759 -0.000051991 -0.000039005 +8081 7.99855 -0.000051953 -0.000038972 +8082 7.99952 -0.000051916 -0.000038939 +8083 8.00049 -0.000051878 -0.000038906 +8084 8.00145 -0.000051841 -0.000038873 +8085 8.00242 -0.000051803 -0.000038841 +8086 8.00338 -0.000051766 -0.000038808 +8087 8.00435 -0.000051728 -0.000038775 +8088 8.00531 -0.000051691 -0.000038742 +8089 8.00628 -0.000051653 -0.000038710 +8090 8.00724 -0.000051616 -0.000038677 +8091 8.00821 -0.000051579 -0.000038644 +8092 8.00917 -0.000051542 -0.000038612 +8093 8.01014 -0.000051504 -0.000038579 +8094 8.01110 -0.000051467 -0.000038547 +8095 8.01207 -0.000051430 -0.000038514 +8096 8.01303 -0.000051393 -0.000038482 +8097 8.01400 -0.000051356 -0.000038449 +8098 8.01496 -0.000051318 -0.000038417 +8099 8.01593 -0.000051281 -0.000038385 +8100 8.01689 -0.000051244 -0.000038352 +8101 8.01786 -0.000051207 -0.000038320 +8102 8.01882 -0.000051170 -0.000038288 +8103 8.01979 -0.000051134 -0.000038256 +8104 8.02075 -0.000051097 -0.000038223 +8105 8.02172 -0.000051060 -0.000038191 +8106 8.02268 -0.000051023 -0.000038159 +8107 8.02365 -0.000050986 -0.000038127 +8108 8.02461 -0.000050949 -0.000038095 +8109 8.02558 -0.000050913 -0.000038063 +8110 8.02654 -0.000050876 -0.000038031 +8111 8.02751 -0.000050839 -0.000037999 +8112 8.02847 -0.000050803 -0.000037967 +8113 8.02944 -0.000050766 -0.000037935 +8114 8.03040 -0.000050729 -0.000037903 +8115 8.03137 -0.000050693 -0.000037871 +8116 8.03233 -0.000050656 -0.000037839 +8117 8.03330 -0.000050620 -0.000037807 +8118 8.03426 -0.000050583 -0.000037776 +8119 8.03523 -0.000050547 -0.000037744 +8120 8.03619 -0.000050510 -0.000037712 +8121 8.03716 -0.000050474 -0.000037680 +8122 8.03812 -0.000050438 -0.000037649 +8123 8.03909 -0.000050401 -0.000037617 +8124 8.04005 -0.000050365 -0.000037586 +8125 8.04102 -0.000050329 -0.000037554 +8126 8.04198 -0.000050293 -0.000037523 +8127 8.04295 -0.000050256 -0.000037491 +8128 8.04391 -0.000050220 -0.000037460 +8129 8.04488 -0.000050184 -0.000037428 +8130 8.04584 -0.000050148 -0.000037397 +8131 8.04681 -0.000050112 -0.000037365 +8132 8.04777 -0.000050076 -0.000037334 +8133 8.04874 -0.000050040 -0.000037303 +8134 8.04970 -0.000050004 -0.000037271 +8135 8.05067 -0.000049968 -0.000037240 +8136 8.05163 -0.000049932 -0.000037209 +8137 8.05260 -0.000049896 -0.000037178 +8138 8.05356 -0.000049860 -0.000037146 +8139 8.05453 -0.000049824 -0.000037115 +8140 8.05549 -0.000049789 -0.000037084 +8141 8.05646 -0.000049753 -0.000037053 +8142 8.05742 -0.000049717 -0.000037022 +8143 8.05839 -0.000049681 -0.000036991 +8144 8.05935 -0.000049646 -0.000036960 +8145 8.06032 -0.000049610 -0.000036929 +8146 8.06128 -0.000049574 -0.000036898 +8147 8.06225 -0.000049539 -0.000036867 +8148 8.06321 -0.000049503 -0.000036836 +8149 8.06418 -0.000049468 -0.000036806 +8150 8.06514 -0.000049432 -0.000036775 +8151 8.06611 -0.000049397 -0.000036744 +8152 8.06707 -0.000049361 -0.000036713 +8153 8.06804 -0.000049326 -0.000036682 +8154 8.06900 -0.000049290 -0.000036652 +8155 8.06997 -0.000049255 -0.000036621 +8156 8.07093 -0.000049220 -0.000036590 +8157 8.07190 -0.000049184 -0.000036560 +8158 8.07286 -0.000049149 -0.000036529 +8159 8.07383 -0.000049114 -0.000036499 +8160 8.07479 -0.000049079 -0.000036468 +8161 8.07576 -0.000049044 -0.000036438 +8162 8.07672 -0.000049008 -0.000036407 +8163 8.07769 -0.000048973 -0.000036377 +8164 8.07865 -0.000048938 -0.000036346 +8165 8.07962 -0.000048903 -0.000036316 +8166 8.08058 -0.000048868 -0.000036286 +8167 8.08155 -0.000048833 -0.000036255 +8168 8.08251 -0.000048798 -0.000036225 +8169 8.08348 -0.000048763 -0.000036195 +8170 8.08444 -0.000048728 -0.000036164 +8171 8.08541 -0.000048693 -0.000036134 +8172 8.08637 -0.000048659 -0.000036104 +8173 8.08734 -0.000048624 -0.000036074 +8174 8.08830 -0.000048589 -0.000036044 +8175 8.08927 -0.000048554 -0.000036014 +8176 8.09023 -0.000048519 -0.000035984 +8177 8.09120 -0.000048485 -0.000035954 +8178 8.09216 -0.000048450 -0.000035924 +8179 8.09313 -0.000048415 -0.000035894 +8180 8.09409 -0.000048381 -0.000035864 +8181 8.09506 -0.000048346 -0.000035834 +8182 8.09602 -0.000048312 -0.000035804 +8183 8.09699 -0.000048277 -0.000035774 +8184 8.09795 -0.000048243 -0.000035744 +8185 8.09892 -0.000048208 -0.000035714 +8186 8.09988 -0.000048174 -0.000035685 +8187 8.10085 -0.000048139 -0.000035655 +8188 8.10182 -0.000048105 -0.000035625 +8189 8.10278 -0.000048070 -0.000035596 +8190 8.10375 -0.000048036 -0.000035566 +8191 8.10471 -0.000048002 -0.000035536 +8192 8.10568 -0.000047968 -0.000035507 +8193 8.10664 -0.000047933 -0.000035477 +8194 8.10761 -0.000047899 -0.000035447 +8195 8.10857 -0.000047865 -0.000035418 +8196 8.10954 -0.000047831 -0.000035388 +8197 8.11050 -0.000047797 -0.000035359 +8198 8.11147 -0.000047762 -0.000035330 +8199 8.11243 -0.000047728 -0.000035300 +8200 8.11340 -0.000047694 -0.000035271 +8201 8.11436 -0.000047660 -0.000035241 +8202 8.11533 -0.000047626 -0.000035212 +8203 8.11629 -0.000047592 -0.000035183 +8204 8.11726 -0.000047558 -0.000035154 +8205 8.11822 -0.000047524 -0.000035124 +8206 8.11919 -0.000047491 -0.000035095 +8207 8.12015 -0.000047457 -0.000035066 +8208 8.12112 -0.000047423 -0.000035037 +8209 8.12208 -0.000047389 -0.000035008 +8210 8.12305 -0.000047355 -0.000034978 +8211 8.12401 -0.000047322 -0.000034949 +8212 8.12498 -0.000047288 -0.000034920 +8213 8.12594 -0.000047254 -0.000034891 +8214 8.12691 -0.000047221 -0.000034862 +8215 8.12787 -0.000047187 -0.000034833 +8216 8.12884 -0.000047153 -0.000034804 +8217 8.12980 -0.000047120 -0.000034776 +8218 8.13077 -0.000047086 -0.000034747 +8219 8.13173 -0.000047053 -0.000034718 +8220 8.13270 -0.000047019 -0.000034689 +8221 8.13366 -0.000046986 -0.000034660 +8222 8.13463 -0.000046952 -0.000034631 +8223 8.13559 -0.000046919 -0.000034603 +8224 8.13656 -0.000046885 -0.000034574 +8225 8.13752 -0.000046852 -0.000034545 +8226 8.13849 -0.000046819 -0.000034517 +8227 8.13945 -0.000046786 -0.000034488 +8228 8.14042 -0.000046752 -0.000034459 +8229 8.14138 -0.000046719 -0.000034431 +8230 8.14235 -0.000046686 -0.000034402 +8231 8.14331 -0.000046653 -0.000034374 +8232 8.14428 -0.000046619 -0.000034345 +8233 8.14524 -0.000046586 -0.000034317 +8234 8.14621 -0.000046553 -0.000034288 +8235 8.14717 -0.000046520 -0.000034260 +8236 8.14814 -0.000046487 -0.000034231 +8237 8.14910 -0.000046454 -0.000034203 +8238 8.15007 -0.000046421 -0.000034175 +8239 8.15103 -0.000046388 -0.000034146 +8240 8.15200 -0.000046355 -0.000034118 +8241 8.15296 -0.000046322 -0.000034090 +8242 8.15393 -0.000046289 -0.000034062 +8243 8.15489 -0.000046257 -0.000034033 +8244 8.15586 -0.000046224 -0.000034005 +8245 8.15682 -0.000046191 -0.000033977 +8246 8.15779 -0.000046158 -0.000033949 +8247 8.15875 -0.000046125 -0.000033921 +8248 8.15972 -0.000046093 -0.000033893 +8249 8.16068 -0.000046060 -0.000033865 +8250 8.16165 -0.000046027 -0.000033837 +8251 8.16261 -0.000045995 -0.000033809 +8252 8.16358 -0.000045962 -0.000033781 +8253 8.16454 -0.000045929 -0.000033753 +8254 8.16551 -0.000045897 -0.000033725 +8255 8.16647 -0.000045864 -0.000033697 +8256 8.16744 -0.000045832 -0.000033669 +8257 8.16840 -0.000045799 -0.000033641 +8258 8.16937 -0.000045767 -0.000033614 +8259 8.17033 -0.000045734 -0.000033586 +8260 8.17130 -0.000045702 -0.000033558 +8261 8.17226 -0.000045670 -0.000033530 +8262 8.17323 -0.000045637 -0.000033503 +8263 8.17419 -0.000045605 -0.000033475 +8264 8.17516 -0.000045573 -0.000033447 +8265 8.17612 -0.000045540 -0.000033420 +8266 8.17709 -0.000045508 -0.000033392 +8267 8.17805 -0.000045476 -0.000033364 +8268 8.17902 -0.000045444 -0.000033337 +8269 8.17998 -0.000045412 -0.000033309 +8270 8.18095 -0.000045380 -0.000033282 +8271 8.18191 -0.000045347 -0.000033254 +8272 8.18288 -0.000045315 -0.000033227 +8273 8.18384 -0.000045283 -0.000033200 +8274 8.18481 -0.000045251 -0.000033172 +8275 8.18577 -0.000045219 -0.000033145 +8276 8.18674 -0.000045187 -0.000033117 +8277 8.18770 -0.000045155 -0.000033090 +8278 8.18867 -0.000045123 -0.000033063 +8279 8.18963 -0.000045092 -0.000033036 +8280 8.19060 -0.000045060 -0.000033008 +8281 8.19156 -0.000045028 -0.000032981 +8282 8.19253 -0.000044996 -0.000032954 +8283 8.19349 -0.000044964 -0.000032927 +8284 8.19446 -0.000044932 -0.000032900 +8285 8.19542 -0.000044901 -0.000032873 +8286 8.19639 -0.000044869 -0.000032845 +8287 8.19735 -0.000044837 -0.000032818 +8288 8.19832 -0.000044806 -0.000032791 +8289 8.19928 -0.000044774 -0.000032764 +8290 8.20025 -0.000044742 -0.000032737 +8291 8.20122 -0.000044711 -0.000032710 +8292 8.20218 -0.000044679 -0.000032683 +8293 8.20315 -0.000044648 -0.000032657 +8294 8.20411 -0.000044616 -0.000032630 +8295 8.20508 -0.000044585 -0.000032603 +8296 8.20604 -0.000044553 -0.000032576 +8297 8.20701 -0.000044522 -0.000032549 +8298 8.20797 -0.000044491 -0.000032522 +8299 8.20894 -0.000044459 -0.000032496 +8300 8.20990 -0.000044428 -0.000032469 +8301 8.21087 -0.000044396 -0.000032442 +8302 8.21183 -0.000044365 -0.000032416 +8303 8.21280 -0.000044334 -0.000032389 +8304 8.21376 -0.000044303 -0.000032362 +8305 8.21473 -0.000044271 -0.000032336 +8306 8.21569 -0.000044240 -0.000032309 +8307 8.21666 -0.000044209 -0.000032283 +8308 8.21762 -0.000044178 -0.000032256 +8309 8.21859 -0.000044147 -0.000032230 +8310 8.21955 -0.000044116 -0.000032203 +8311 8.22052 -0.000044085 -0.000032177 +8312 8.22148 -0.000044054 -0.000032150 +8313 8.22245 -0.000044023 -0.000032124 +8314 8.22341 -0.000043992 -0.000032097 +8315 8.22438 -0.000043961 -0.000032071 +8316 8.22534 -0.000043930 -0.000032045 +8317 8.22631 -0.000043899 -0.000032018 +8318 8.22727 -0.000043868 -0.000031992 +8319 8.22824 -0.000043837 -0.000031966 +8320 8.22920 -0.000043806 -0.000031940 +8321 8.23017 -0.000043775 -0.000031913 +8322 8.23113 -0.000043745 -0.000031887 +8323 8.23210 -0.000043714 -0.000031861 +8324 8.23306 -0.000043683 -0.000031835 +8325 8.23403 -0.000043652 -0.000031809 +8326 8.23499 -0.000043622 -0.000031783 +8327 8.23596 -0.000043591 -0.000031757 +8328 8.23692 -0.000043560 -0.000031731 +8329 8.23789 -0.000043530 -0.000031705 +8330 8.23885 -0.000043499 -0.000031679 +8331 8.23982 -0.000043469 -0.000031653 +8332 8.24078 -0.000043438 -0.000031627 +8333 8.24175 -0.000043408 -0.000031601 +8334 8.24271 -0.000043377 -0.000031575 +8335 8.24368 -0.000043347 -0.000031549 +8336 8.24464 -0.000043316 -0.000031523 +8337 8.24561 -0.000043286 -0.000031497 +8338 8.24657 -0.000043256 -0.000031472 +8339 8.24754 -0.000043225 -0.000031446 +8340 8.24850 -0.000043195 -0.000031420 +8341 8.24947 -0.000043165 -0.000031394 +8342 8.25043 -0.000043134 -0.000031369 +8343 8.25140 -0.000043104 -0.000031343 +8344 8.25236 -0.000043074 -0.000031317 +8345 8.25333 -0.000043044 -0.000031292 +8346 8.25429 -0.000043013 -0.000031266 +8347 8.25526 -0.000042983 -0.000031241 +8348 8.25622 -0.000042953 -0.000031215 +8349 8.25719 -0.000042923 -0.000031190 +8350 8.25815 -0.000042893 -0.000031164 +8351 8.25912 -0.000042863 -0.000031139 +8352 8.26008 -0.000042833 -0.000031113 +8353 8.26105 -0.000042803 -0.000031088 +8354 8.26201 -0.000042773 -0.000031062 +8355 8.26298 -0.000042743 -0.000031037 +8356 8.26394 -0.000042713 -0.000031011 +8357 8.26491 -0.000042683 -0.000030986 +8358 8.26587 -0.000042653 -0.000030961 +8359 8.26684 -0.000042623 -0.000030936 +8360 8.26780 -0.000042593 -0.000030910 +8361 8.26877 -0.000042564 -0.000030885 +8362 8.26973 -0.000042534 -0.000030860 +8363 8.27070 -0.000042504 -0.000030835 +8364 8.27166 -0.000042474 -0.000030809 +8365 8.27263 -0.000042444 -0.000030784 +8366 8.27359 -0.000042415 -0.000030759 +8367 8.27456 -0.000042385 -0.000030734 +8368 8.27552 -0.000042355 -0.000030709 +8369 8.27649 -0.000042326 -0.000030684 +8370 8.27745 -0.000042296 -0.000030659 +8371 8.27842 -0.000042267 -0.000030634 +8372 8.27938 -0.000042237 -0.000030609 +8373 8.28035 -0.000042208 -0.000030584 +8374 8.28131 -0.000042178 -0.000030559 +8375 8.28228 -0.000042149 -0.000030534 +8376 8.28324 -0.000042119 -0.000030509 +8377 8.28421 -0.000042090 -0.000030484 +8378 8.28517 -0.000042060 -0.000030459 +8379 8.28614 -0.000042031 -0.000030435 +8380 8.28710 -0.000042002 -0.000030410 +8381 8.28807 -0.000041972 -0.000030385 +8382 8.28903 -0.000041943 -0.000030360 +8383 8.29000 -0.000041914 -0.000030336 +8384 8.29096 -0.000041884 -0.000030311 +8385 8.29193 -0.000041855 -0.000030286 +8386 8.29289 -0.000041826 -0.000030262 +8387 8.29386 -0.000041797 -0.000030237 +8388 8.29482 -0.000041768 -0.000030212 +8389 8.29579 -0.000041738 -0.000030188 +8390 8.29675 -0.000041709 -0.000030163 +8391 8.29772 -0.000041680 -0.000030139 +8392 8.29868 -0.000041651 -0.000030114 +8393 8.29965 -0.000041622 -0.000030090 +8394 8.30062 -0.000041593 -0.000030065 +8395 8.30158 -0.000041564 -0.000030041 +8396 8.30255 -0.000041535 -0.000030016 +8397 8.30351 -0.000041506 -0.000029992 +8398 8.30448 -0.000041477 -0.000029967 +8399 8.30544 -0.000041448 -0.000029943 +8400 8.30641 -0.000041419 -0.000029919 +8401 8.30737 -0.000041391 -0.000029894 +8402 8.30834 -0.000041362 -0.000029870 +8403 8.30930 -0.000041333 -0.000029846 +8404 8.31027 -0.000041304 -0.000029821 +8405 8.31123 -0.000041275 -0.000029797 +8406 8.31220 -0.000041247 -0.000029773 +8407 8.31316 -0.000041218 -0.000029749 +8408 8.31413 -0.000041189 -0.000029725 +8409 8.31509 -0.000041160 -0.000029701 +8410 8.31606 -0.000041132 -0.000029676 +8411 8.31702 -0.000041103 -0.000029652 +8412 8.31799 -0.000041075 -0.000029628 +8413 8.31895 -0.000041046 -0.000029604 +8414 8.31992 -0.000041017 -0.000029580 +8415 8.32088 -0.000040989 -0.000029556 +8416 8.32185 -0.000040960 -0.000029532 +8417 8.32281 -0.000040932 -0.000029508 +8418 8.32378 -0.000040903 -0.000029484 +8419 8.32474 -0.000040875 -0.000029460 +8420 8.32571 -0.000040847 -0.000029436 +8421 8.32667 -0.000040818 -0.000029413 +8422 8.32764 -0.000040790 -0.000029389 +8423 8.32860 -0.000040761 -0.000029365 +8424 8.32957 -0.000040733 -0.000029341 +8425 8.33053 -0.000040705 -0.000029317 +8426 8.33150 -0.000040677 -0.000029294 +8427 8.33246 -0.000040648 -0.000029270 +8428 8.33343 -0.000040620 -0.000029246 +8429 8.33439 -0.000040592 -0.000029222 +8430 8.33536 -0.000040564 -0.000029199 +8431 8.33632 -0.000040535 -0.000029175 +8432 8.33729 -0.000040507 -0.000029151 +8433 8.33825 -0.000040479 -0.000029128 +8434 8.33922 -0.000040451 -0.000029104 +8435 8.34018 -0.000040423 -0.000029081 +8436 8.34115 -0.000040395 -0.000029057 +8437 8.34211 -0.000040367 -0.000029034 +8438 8.34308 -0.000040339 -0.000029010 +8439 8.34404 -0.000040311 -0.000028987 +8440 8.34501 -0.000040283 -0.000028963 +8441 8.34597 -0.000040255 -0.000028940 +8442 8.34694 -0.000040227 -0.000028916 +8443 8.34790 -0.000040199 -0.000028893 +8444 8.34887 -0.000040171 -0.000028870 +8445 8.34983 -0.000040144 -0.000028846 +8446 8.35080 -0.000040116 -0.000028823 +8447 8.35176 -0.000040088 -0.000028800 +8448 8.35273 -0.000040060 -0.000028776 +8449 8.35369 -0.000040032 -0.000028753 +8450 8.35466 -0.000040005 -0.000028730 +8451 8.35562 -0.000039977 -0.000028707 +8452 8.35659 -0.000039949 -0.000028683 +8453 8.35755 -0.000039922 -0.000028660 +8454 8.35852 -0.000039894 -0.000028637 +8455 8.35948 -0.000039866 -0.000028614 +8456 8.36045 -0.000039839 -0.000028591 +8457 8.36141 -0.000039811 -0.000028568 +8458 8.36238 -0.000039784 -0.000028545 +8459 8.36334 -0.000039756 -0.000028522 +8460 8.36431 -0.000039728 -0.000028499 +8461 8.36527 -0.000039701 -0.000028476 +8462 8.36624 -0.000039674 -0.000028453 +8463 8.36720 -0.000039646 -0.000028430 +8464 8.36817 -0.000039619 -0.000028407 +8465 8.36913 -0.000039591 -0.000028384 +8466 8.37010 -0.000039564 -0.000028361 +8467 8.37106 -0.000039537 -0.000028338 +8468 8.37203 -0.000039509 -0.000028315 +8469 8.37299 -0.000039482 -0.000028292 +8470 8.37396 -0.000039455 -0.000028269 +8471 8.37492 -0.000039427 -0.000028247 +8472 8.37589 -0.000039400 -0.000028224 +8473 8.37685 -0.000039373 -0.000028201 +8474 8.37782 -0.000039346 -0.000028178 +8475 8.37878 -0.000039318 -0.000028156 +8476 8.37975 -0.000039291 -0.000028133 +8477 8.38071 -0.000039264 -0.000028110 +8478 8.38168 -0.000039237 -0.000028088 +8479 8.38264 -0.000039210 -0.000028065 +8480 8.38361 -0.000039183 -0.000028042 +8481 8.38457 -0.000039156 -0.000028020 +8482 8.38554 -0.000039129 -0.000027997 +8483 8.38650 -0.000039102 -0.000027975 +8484 8.38747 -0.000039075 -0.000027952 +8485 8.38843 -0.000039048 -0.000027930 +8486 8.38940 -0.000039021 -0.000027907 +8487 8.39036 -0.000038994 -0.000027885 +8488 8.39133 -0.000038967 -0.000027862 +8489 8.39229 -0.000038940 -0.000027840 +8490 8.39326 -0.000038913 -0.000027818 +8491 8.39422 -0.000038886 -0.000027795 +8492 8.39519 -0.000038860 -0.000027773 +8493 8.39615 -0.000038833 -0.000027750 +8494 8.39712 -0.000038806 -0.000027728 +8495 8.39808 -0.000038779 -0.000027706 +8496 8.39905 -0.000038753 -0.000027684 +8497 8.40002 -0.000038726 -0.000027661 +8498 8.40098 -0.000038699 -0.000027639 +8499 8.40195 -0.000038673 -0.000027617 +8500 8.40291 -0.000038646 -0.000027595 +8501 8.40388 -0.000038619 -0.000027573 +8502 8.40484 -0.000038593 -0.000027550 +8503 8.40581 -0.000038566 -0.000027528 +8504 8.40677 -0.000038540 -0.000027506 +8505 8.40774 -0.000038513 -0.000027484 +8506 8.40870 -0.000038487 -0.000027462 +8507 8.40967 -0.000038460 -0.000027440 +8508 8.41063 -0.000038434 -0.000027418 +8509 8.41160 -0.000038407 -0.000027396 +8510 8.41256 -0.000038381 -0.000027374 +8511 8.41353 -0.000038354 -0.000027352 +8512 8.41449 -0.000038328 -0.000027330 +8513 8.41546 -0.000038302 -0.000027308 +8514 8.41642 -0.000038275 -0.000027286 +8515 8.41739 -0.000038249 -0.000027264 +8516 8.41835 -0.000038223 -0.000027242 +8517 8.41932 -0.000038196 -0.000027220 +8518 8.42028 -0.000038170 -0.000027199 +8519 8.42125 -0.000038144 -0.000027177 +8520 8.42221 -0.000038118 -0.000027155 +8521 8.42318 -0.000038091 -0.000027133 +8522 8.42414 -0.000038065 -0.000027112 +8523 8.42511 -0.000038039 -0.000027090 +8524 8.42607 -0.000038013 -0.000027068 +8525 8.42704 -0.000037987 -0.000027046 +8526 8.42800 -0.000037961 -0.000027025 +8527 8.42897 -0.000037935 -0.000027003 +8528 8.42993 -0.000037909 -0.000026981 +8529 8.43090 -0.000037883 -0.000026960 +8530 8.43186 -0.000037857 -0.000026938 +8531 8.43283 -0.000037831 -0.000026917 +8532 8.43379 -0.000037805 -0.000026895 +8533 8.43476 -0.000037779 -0.000026874 +8534 8.43572 -0.000037753 -0.000026852 +8535 8.43669 -0.000037727 -0.000026831 +8536 8.43765 -0.000037701 -0.000026809 +8537 8.43862 -0.000037675 -0.000026788 +8538 8.43958 -0.000037649 -0.000026766 +8539 8.44055 -0.000037623 -0.000026745 +8540 8.44151 -0.000037598 -0.000026723 +8541 8.44248 -0.000037572 -0.000026702 +8542 8.44344 -0.000037546 -0.000026681 +8543 8.44441 -0.000037520 -0.000026659 +8544 8.44537 -0.000037495 -0.000026638 +8545 8.44634 -0.000037469 -0.000026617 +8546 8.44730 -0.000037443 -0.000026595 +8547 8.44827 -0.000037418 -0.000026574 +8548 8.44923 -0.000037392 -0.000026553 +8549 8.45020 -0.000037366 -0.000026532 +8550 8.45116 -0.000037341 -0.000026511 +8551 8.45213 -0.000037315 -0.000026489 +8552 8.45309 -0.000037290 -0.000026468 +8553 8.45406 -0.000037264 -0.000026447 +8554 8.45502 -0.000037239 -0.000026426 +8555 8.45599 -0.000037213 -0.000026405 +8556 8.45695 -0.000037188 -0.000026384 +8557 8.45792 -0.000037162 -0.000026363 +8558 8.45888 -0.000037137 -0.000026342 +8559 8.45985 -0.000037111 -0.000026321 +8560 8.46081 -0.000037086 -0.000026300 +8561 8.46178 -0.000037061 -0.000026279 +8562 8.46274 -0.000037035 -0.000026258 +8563 8.46371 -0.000037010 -0.000026237 +8564 8.46467 -0.000036985 -0.000026216 +8565 8.46564 -0.000036959 -0.000026195 +8566 8.46660 -0.000036934 -0.000026174 +8567 8.46757 -0.000036909 -0.000026153 +8568 8.46853 -0.000036884 -0.000026132 +8569 8.46950 -0.000036858 -0.000026111 +8570 8.47046 -0.000036833 -0.000026091 +8571 8.47143 -0.000036808 -0.000026070 +8572 8.47239 -0.000036783 -0.000026049 +8573 8.47336 -0.000036758 -0.000026028 +8574 8.47432 -0.000036733 -0.000026007 +8575 8.47529 -0.000036708 -0.000025987 +8576 8.47625 -0.000036682 -0.000025966 +8577 8.47722 -0.000036657 -0.000025945 +8578 8.47818 -0.000036632 -0.000025925 +8579 8.47915 -0.000036607 -0.000025904 +8580 8.48011 -0.000036582 -0.000025883 +8581 8.48108 -0.000036557 -0.000025863 +8582 8.48204 -0.000036532 -0.000025842 +8583 8.48301 -0.000036508 -0.000025822 +8584 8.48397 -0.000036483 -0.000025801 +8585 8.48494 -0.000036458 -0.000025781 +8586 8.48590 -0.000036433 -0.000025760 +8587 8.48687 -0.000036408 -0.000025740 +8588 8.48783 -0.000036383 -0.000025719 +8589 8.48880 -0.000036358 -0.000025699 +8590 8.48976 -0.000036334 -0.000025678 +8591 8.49073 -0.000036309 -0.000025658 +8592 8.49169 -0.000036284 -0.000025637 +8593 8.49266 -0.000036259 -0.000025617 +8594 8.49362 -0.000036235 -0.000025597 +8595 8.49459 -0.000036210 -0.000025576 +8596 8.49555 -0.000036185 -0.000025556 +8597 8.49652 -0.000036161 -0.000025536 +8598 8.49748 -0.000036136 -0.000025515 +8599 8.49845 -0.000036111 -0.000025495 +8600 8.49941 -0.000036087 -0.000025475 +8601 8.50038 -0.000036062 -0.000025455 +8602 8.50135 -0.000036038 -0.000025434 +8603 8.50231 -0.000036013 -0.000025414 +8604 8.50328 -0.000035989 -0.000025394 +8605 8.50424 -0.000035964 -0.000025374 +8606 8.50521 -0.000035940 -0.000025354 +8607 8.50617 -0.000035915 -0.000025333 +8608 8.50714 -0.000035891 -0.000025313 +8609 8.50810 -0.000035866 -0.000025293 +8610 8.50907 -0.000035842 -0.000025273 +8611 8.51003 -0.000035818 -0.000025253 +8612 8.51100 -0.000035793 -0.000025233 +8613 8.51196 -0.000035769 -0.000025213 +8614 8.51293 -0.000035745 -0.000025193 +8615 8.51389 -0.000035720 -0.000025173 +8616 8.51486 -0.000035696 -0.000025153 +8617 8.51582 -0.000035672 -0.000025133 +8618 8.51679 -0.000035647 -0.000025113 +8619 8.51775 -0.000035623 -0.000025093 +8620 8.51872 -0.000035599 -0.000025073 +8621 8.51968 -0.000035575 -0.000025054 +8622 8.52065 -0.000035551 -0.000025034 +8623 8.52161 -0.000035526 -0.000025014 +8624 8.52258 -0.000035502 -0.000024994 +8625 8.52354 -0.000035478 -0.000024974 +8626 8.52451 -0.000035454 -0.000024955 +8627 8.52547 -0.000035430 -0.000024935 +8628 8.52644 -0.000035406 -0.000024915 +8629 8.52740 -0.000035382 -0.000024895 +8630 8.52837 -0.000035358 -0.000024876 +8631 8.52933 -0.000035334 -0.000024856 +8632 8.53030 -0.000035310 -0.000024836 +8633 8.53126 -0.000035286 -0.000024817 +8634 8.53223 -0.000035262 -0.000024797 +8635 8.53319 -0.000035238 -0.000024777 +8636 8.53416 -0.000035214 -0.000024758 +8637 8.53512 -0.000035190 -0.000024738 +8638 8.53609 -0.000035167 -0.000024718 +8639 8.53705 -0.000035143 -0.000024699 +8640 8.53802 -0.000035119 -0.000024679 +8641 8.53898 -0.000035095 -0.000024660 +8642 8.53995 -0.000035071 -0.000024640 +8643 8.54091 -0.000035047 -0.000024621 +8644 8.54188 -0.000035024 -0.000024601 +8645 8.54284 -0.000035000 -0.000024582 +8646 8.54381 -0.000034976 -0.000024563 +8647 8.54477 -0.000034953 -0.000024543 +8648 8.54574 -0.000034929 -0.000024524 +8649 8.54670 -0.000034905 -0.000024504 +8650 8.54767 -0.000034882 -0.000024485 +8651 8.54863 -0.000034858 -0.000024466 +8652 8.54960 -0.000034834 -0.000024446 +8653 8.55056 -0.000034811 -0.000024427 +8654 8.55153 -0.000034787 -0.000024408 +8655 8.55249 -0.000034764 -0.000024388 +8656 8.55346 -0.000034740 -0.000024369 +8657 8.55442 -0.000034717 -0.000024350 +8658 8.55539 -0.000034693 -0.000024331 +8659 8.55635 -0.000034670 -0.000024312 +8660 8.55732 -0.000034646 -0.000024292 +8661 8.55828 -0.000034623 -0.000024273 +8662 8.55925 -0.000034599 -0.000024254 +8663 8.56021 -0.000034576 -0.000024235 +8664 8.56118 -0.000034553 -0.000024216 +8665 8.56214 -0.000034529 -0.000024197 +8666 8.56311 -0.000034506 -0.000024178 +8667 8.56407 -0.000034483 -0.000024159 +8668 8.56504 -0.000034459 -0.000024140 +8669 8.56600 -0.000034436 -0.000024120 +8670 8.56697 -0.000034413 -0.000024101 +8671 8.56793 -0.000034389 -0.000024082 +8672 8.56890 -0.000034366 -0.000024063 +8673 8.56986 -0.000034343 -0.000024045 +8674 8.57083 -0.000034320 -0.000024026 +8675 8.57179 -0.000034297 -0.000024007 +8676 8.57276 -0.000034274 -0.000023988 +8677 8.57372 -0.000034250 -0.000023969 +8678 8.57469 -0.000034227 -0.000023950 +8679 8.57565 -0.000034204 -0.000023931 +8680 8.57662 -0.000034181 -0.000023912 +8681 8.57758 -0.000034158 -0.000023893 +8682 8.57855 -0.000034135 -0.000023875 +8683 8.57951 -0.000034112 -0.000023856 +8684 8.58048 -0.000034089 -0.000023837 +8685 8.58144 -0.000034066 -0.000023818 +8686 8.58241 -0.000034043 -0.000023800 +8687 8.58337 -0.000034020 -0.000023781 +8688 8.58434 -0.000033997 -0.000023762 +8689 8.58530 -0.000033974 -0.000023743 +8690 8.58627 -0.000033951 -0.000023725 +8691 8.58723 -0.000033928 -0.000023706 +8692 8.58820 -0.000033905 -0.000023687 +8693 8.58916 -0.000033883 -0.000023669 +8694 8.59013 -0.000033860 -0.000023650 +8695 8.59109 -0.000033837 -0.000023632 +8696 8.59206 -0.000033814 -0.000023613 +8697 8.59302 -0.000033791 -0.000023595 +8698 8.59399 -0.000033769 -0.000023576 +8699 8.59495 -0.000033746 -0.000023557 +8700 8.59592 -0.000033723 -0.000023539 +8701 8.59688 -0.000033700 -0.000023520 +8702 8.59785 -0.000033678 -0.000023502 +8703 8.59881 -0.000033655 -0.000023484 +8704 8.59978 -0.000033632 -0.000023465 +8705 8.60075 -0.000033610 -0.000023447 +8706 8.60171 -0.000033587 -0.000023428 +8707 8.60268 -0.000033565 -0.000023410 +8708 8.60364 -0.000033542 -0.000023391 +8709 8.60461 -0.000033519 -0.000023373 +8710 8.60557 -0.000033497 -0.000023355 +8711 8.60654 -0.000033474 -0.000023336 +8712 8.60750 -0.000033452 -0.000023318 +8713 8.60847 -0.000033429 -0.000023300 +8714 8.60943 -0.000033407 -0.000023282 +8715 8.61040 -0.000033384 -0.000023263 +8716 8.61136 -0.000033362 -0.000023245 +8717 8.61233 -0.000033340 -0.000023227 +8718 8.61329 -0.000033317 -0.000023209 +8719 8.61426 -0.000033295 -0.000023190 +8720 8.61522 -0.000033272 -0.000023172 +8721 8.61619 -0.000033250 -0.000023154 +8722 8.61715 -0.000033228 -0.000023136 +8723 8.61812 -0.000033205 -0.000023118 +8724 8.61908 -0.000033183 -0.000023100 +8725 8.62005 -0.000033161 -0.000023082 +8726 8.62101 -0.000033139 -0.000023064 +8727 8.62198 -0.000033116 -0.000023045 +8728 8.62294 -0.000033094 -0.000023027 +8729 8.62391 -0.000033072 -0.000023009 +8730 8.62487 -0.000033050 -0.000022991 +8731 8.62584 -0.000033027 -0.000022973 +8732 8.62680 -0.000033005 -0.000022955 +8733 8.62777 -0.000032983 -0.000022937 +8734 8.62873 -0.000032961 -0.000022919 +8735 8.62970 -0.000032939 -0.000022902 +8736 8.63066 -0.000032917 -0.000022884 +8737 8.63163 -0.000032895 -0.000022866 +8738 8.63259 -0.000032873 -0.000022848 +8739 8.63356 -0.000032851 -0.000022830 +8740 8.63452 -0.000032829 -0.000022812 +8741 8.63549 -0.000032807 -0.000022794 +8742 8.63645 -0.000032785 -0.000022776 +8743 8.63742 -0.000032763 -0.000022759 +8744 8.63838 -0.000032741 -0.000022741 +8745 8.63935 -0.000032719 -0.000022723 +8746 8.64031 -0.000032697 -0.000022705 +8747 8.64128 -0.000032675 -0.000022688 +8748 8.64224 -0.000032653 -0.000022670 +8749 8.64321 -0.000032631 -0.000022652 +8750 8.64417 -0.000032609 -0.000022634 +8751 8.64514 -0.000032587 -0.000022617 +8752 8.64610 -0.000032566 -0.000022599 +8753 8.64707 -0.000032544 -0.000022581 +8754 8.64803 -0.000032522 -0.000022564 +8755 8.64900 -0.000032500 -0.000022546 +8756 8.64996 -0.000032479 -0.000022529 +8757 8.65093 -0.000032457 -0.000022511 +8758 8.65189 -0.000032435 -0.000022493 +8759 8.65286 -0.000032413 -0.000022476 +8760 8.65382 -0.000032392 -0.000022458 +8761 8.65479 -0.000032370 -0.000022441 +8762 8.65575 -0.000032348 -0.000022423 +8763 8.65672 -0.000032327 -0.000022406 +8764 8.65768 -0.000032305 -0.000022388 +8765 8.65865 -0.000032284 -0.000022371 +8766 8.65961 -0.000032262 -0.000022353 +8767 8.66058 -0.000032240 -0.000022336 +8768 8.66154 -0.000032219 -0.000022319 +8769 8.66251 -0.000032197 -0.000022301 +8770 8.66347 -0.000032176 -0.000022284 +8771 8.66444 -0.000032154 -0.000022266 +8772 8.66540 -0.000032133 -0.000022249 +8773 8.66637 -0.000032111 -0.000022232 +8774 8.66733 -0.000032090 -0.000022214 +8775 8.66830 -0.000032069 -0.000022197 +8776 8.66926 -0.000032047 -0.000022180 +8777 8.67023 -0.000032026 -0.000022163 +8778 8.67119 -0.000032004 -0.000022145 +8779 8.67216 -0.000031983 -0.000022128 +8780 8.67312 -0.000031962 -0.000022111 +8781 8.67409 -0.000031940 -0.000022094 +8782 8.67505 -0.000031919 -0.000022076 +8783 8.67602 -0.000031898 -0.000022059 +8784 8.67698 -0.000031876 -0.000022042 +8785 8.67795 -0.000031855 -0.000022025 +8786 8.67891 -0.000031834 -0.000022008 +8787 8.67988 -0.000031813 -0.000021991 +8788 8.68084 -0.000031791 -0.000021974 +8789 8.68181 -0.000031770 -0.000021956 +8790 8.68277 -0.000031749 -0.000021939 +8791 8.68374 -0.000031728 -0.000021922 +8792 8.68470 -0.000031707 -0.000021905 +8793 8.68567 -0.000031686 -0.000021888 +8794 8.68663 -0.000031665 -0.000021871 +8795 8.68760 -0.000031643 -0.000021854 +8796 8.68856 -0.000031622 -0.000021837 +8797 8.68953 -0.000031601 -0.000021820 +8798 8.69049 -0.000031580 -0.000021803 +8799 8.69146 -0.000031559 -0.000021786 +8800 8.69242 -0.000031538 -0.000021769 +8801 8.69339 -0.000031517 -0.000021753 +8802 8.69435 -0.000031496 -0.000021736 +8803 8.69532 -0.000031475 -0.000021719 +8804 8.69628 -0.000031454 -0.000021702 +8805 8.69725 -0.000031433 -0.000021685 +8806 8.69821 -0.000031412 -0.000021668 +8807 8.69918 -0.000031392 -0.000021651 +8808 8.70015 -0.000031371 -0.000021635 +8809 8.70111 -0.000031350 -0.000021618 +8810 8.70208 -0.000031329 -0.000021601 +8811 8.70304 -0.000031308 -0.000021584 +8812 8.70401 -0.000031287 -0.000021567 +8813 8.70497 -0.000031266 -0.000021551 +8814 8.70594 -0.000031246 -0.000021534 +8815 8.70690 -0.000031225 -0.000021517 +8816 8.70787 -0.000031204 -0.000021501 +8817 8.70883 -0.000031183 -0.000021484 +8818 8.70980 -0.000031163 -0.000021467 +8819 8.71076 -0.000031142 -0.000021451 +8820 8.71173 -0.000031121 -0.000021434 +8821 8.71269 -0.000031101 -0.000021417 +8822 8.71366 -0.000031080 -0.000021401 +8823 8.71462 -0.000031059 -0.000021384 +8824 8.71559 -0.000031039 -0.000021368 +8825 8.71655 -0.000031018 -0.000021351 +8826 8.71752 -0.000030997 -0.000021335 +8827 8.71848 -0.000030977 -0.000021318 +8828 8.71945 -0.000030956 -0.000021302 +8829 8.72041 -0.000030936 -0.000021285 +8830 8.72138 -0.000030915 -0.000021269 +8831 8.72234 -0.000030895 -0.000021252 +8832 8.72331 -0.000030874 -0.000021236 +8833 8.72427 -0.000030854 -0.000021219 +8834 8.72524 -0.000030833 -0.000021203 +8835 8.72620 -0.000030813 -0.000021186 +8836 8.72717 -0.000030792 -0.000021170 +8837 8.72813 -0.000030772 -0.000021154 +8838 8.72910 -0.000030752 -0.000021137 +8839 8.73006 -0.000030731 -0.000021121 +8840 8.73103 -0.000030711 -0.000021105 +8841 8.73199 -0.000030690 -0.000021088 +8842 8.73296 -0.000030670 -0.000021072 +8843 8.73392 -0.000030650 -0.000021056 +8844 8.73489 -0.000030629 -0.000021039 +8845 8.73585 -0.000030609 -0.000021023 +8846 8.73682 -0.000030589 -0.000021007 +8847 8.73778 -0.000030569 -0.000020991 +8848 8.73875 -0.000030548 -0.000020974 +8849 8.73971 -0.000030528 -0.000020958 +8850 8.74068 -0.000030508 -0.000020942 +8851 8.74164 -0.000030488 -0.000020926 +8852 8.74261 -0.000030467 -0.000020910 +8853 8.74357 -0.000030447 -0.000020894 +8854 8.74454 -0.000030427 -0.000020877 +8855 8.74550 -0.000030407 -0.000020861 +8856 8.74647 -0.000030387 -0.000020845 +8857 8.74743 -0.000030367 -0.000020829 +8858 8.74840 -0.000030347 -0.000020813 +8859 8.74936 -0.000030327 -0.000020797 +8860 8.75033 -0.000030307 -0.000020781 +8861 8.75129 -0.000030287 -0.000020765 +8862 8.75226 -0.000030266 -0.000020749 +8863 8.75322 -0.000030246 -0.000020733 +8864 8.75419 -0.000030226 -0.000020717 +8865 8.75515 -0.000030206 -0.000020701 +8866 8.75612 -0.000030186 -0.000020685 +8867 8.75708 -0.000030167 -0.000020669 +8868 8.75805 -0.000030147 -0.000020653 +8869 8.75901 -0.000030127 -0.000020637 +8870 8.75998 -0.000030107 -0.000020621 +8871 8.76094 -0.000030087 -0.000020605 +8872 8.76191 -0.000030067 -0.000020589 +8873 8.76287 -0.000030047 -0.000020573 +8874 8.76384 -0.000030027 -0.000020558 +8875 8.76480 -0.000030007 -0.000020542 +8876 8.76577 -0.000029988 -0.000020526 +8877 8.76673 -0.000029968 -0.000020510 +8878 8.76770 -0.000029948 -0.000020494 +8879 8.76866 -0.000029928 -0.000020479 +8880 8.76963 -0.000029909 -0.000020463 +8881 8.77059 -0.000029889 -0.000020447 +8882 8.77156 -0.000029869 -0.000020431 +8883 8.77252 -0.000029849 -0.000020416 +8884 8.77349 -0.000029830 -0.000020400 +8885 8.77445 -0.000029810 -0.000020384 +8886 8.77542 -0.000029790 -0.000020368 +8887 8.77638 -0.000029771 -0.000020353 +8888 8.77735 -0.000029751 -0.000020337 +8889 8.77831 -0.000029731 -0.000020322 +8890 8.77928 -0.000029712 -0.000020306 +8891 8.78024 -0.000029692 -0.000020290 +8892 8.78121 -0.000029673 -0.000020275 +8893 8.78217 -0.000029653 -0.000020259 +8894 8.78314 -0.000029634 -0.000020243 +8895 8.78410 -0.000029614 -0.000020228 +8896 8.78507 -0.000029595 -0.000020212 +8897 8.78603 -0.000029575 -0.000020197 +8898 8.78700 -0.000029556 -0.000020181 +8899 8.78796 -0.000029536 -0.000020166 +8900 8.78893 -0.000029517 -0.000020150 +8901 8.78989 -0.000029497 -0.000020135 +8902 8.79086 -0.000029478 -0.000020119 +8903 8.79182 -0.000029458 -0.000020104 +8904 8.79279 -0.000029439 -0.000020088 +8905 8.79375 -0.000029420 -0.000020073 +8906 8.79472 -0.000029400 -0.000020058 +8907 8.79568 -0.000029381 -0.000020042 +8908 8.79665 -0.000029362 -0.000020027 +8909 8.79761 -0.000029342 -0.000020011 +8910 8.79858 -0.000029323 -0.000019996 +8911 8.79954 -0.000029304 -0.000019981 +8912 8.80051 -0.000029284 -0.000019965 +8913 8.80148 -0.000029265 -0.000019950 +8914 8.80244 -0.000029246 -0.000019935 +8915 8.80341 -0.000029227 -0.000019920 +8916 8.80437 -0.000029207 -0.000019904 +8917 8.80534 -0.000029188 -0.000019889 +8918 8.80630 -0.000029169 -0.000019874 +8919 8.80727 -0.000029150 -0.000019858 +8920 8.80823 -0.000029131 -0.000019843 +8921 8.80920 -0.000029112 -0.000019828 +8922 8.81016 -0.000029092 -0.000019813 +8923 8.81113 -0.000029073 -0.000019798 +8924 8.81209 -0.000029054 -0.000019782 +8925 8.81306 -0.000029035 -0.000019767 +8926 8.81402 -0.000029016 -0.000019752 +8927 8.81499 -0.000028997 -0.000019737 +8928 8.81595 -0.000028978 -0.000019722 +8929 8.81692 -0.000028959 -0.000019707 +8930 8.81788 -0.000028940 -0.000019692 +8931 8.81885 -0.000028921 -0.000019677 +8932 8.81981 -0.000028902 -0.000019662 +8933 8.82078 -0.000028883 -0.000019647 +8934 8.82174 -0.000028864 -0.000019631 +8935 8.82271 -0.000028845 -0.000019616 +8936 8.82367 -0.000028826 -0.000019601 +8937 8.82464 -0.000028807 -0.000019586 +8938 8.82560 -0.000028788 -0.000019571 +8939 8.82657 -0.000028769 -0.000019556 +8940 8.82753 -0.000028751 -0.000019542 +8941 8.82850 -0.000028732 -0.000019527 +8942 8.82946 -0.000028713 -0.000019512 +8943 8.83043 -0.000028694 -0.000019497 +8944 8.83139 -0.000028675 -0.000019482 +8945 8.83236 -0.000028656 -0.000019467 +8946 8.83332 -0.000028638 -0.000019452 +8947 8.83429 -0.000028619 -0.000019437 +8948 8.83525 -0.000028600 -0.000019422 +8949 8.83622 -0.000028581 -0.000019407 +8950 8.83718 -0.000028563 -0.000019393 +8951 8.83815 -0.000028544 -0.000019378 +8952 8.83911 -0.000028525 -0.000019363 +8953 8.84008 -0.000028507 -0.000019348 +8954 8.84104 -0.000028488 -0.000019333 +8955 8.84201 -0.000028469 -0.000019319 +8956 8.84297 -0.000028451 -0.000019304 +8957 8.84394 -0.000028432 -0.000019289 +8958 8.84490 -0.000028413 -0.000019274 +8959 8.84587 -0.000028395 -0.000019260 +8960 8.84683 -0.000028376 -0.000019245 +8961 8.84780 -0.000028358 -0.000019230 +8962 8.84876 -0.000028339 -0.000019216 +8963 8.84973 -0.000028321 -0.000019201 +8964 8.85069 -0.000028302 -0.000019186 +8965 8.85166 -0.000028284 -0.000019172 +8966 8.85262 -0.000028265 -0.000019157 +8967 8.85359 -0.000028247 -0.000019142 +8968 8.85455 -0.000028228 -0.000019128 +8969 8.85552 -0.000028210 -0.000019113 +8970 8.85648 -0.000028191 -0.000019099 +8971 8.85745 -0.000028173 -0.000019084 +8972 8.85841 -0.000028154 -0.000019070 +8973 8.85938 -0.000028136 -0.000019055 +8974 8.86034 -0.000028118 -0.000019041 +8975 8.86131 -0.000028099 -0.000019026 +8976 8.86227 -0.000028081 -0.000019012 +8977 8.86324 -0.000028063 -0.000018997 +8978 8.86420 -0.000028044 -0.000018983 +8979 8.86517 -0.000028026 -0.000018968 +8980 8.86613 -0.000028008 -0.000018954 +8981 8.86710 -0.000027989 -0.000018939 +8982 8.86806 -0.000027971 -0.000018925 +8983 8.86903 -0.000027953 -0.000018910 +8984 8.86999 -0.000027935 -0.000018896 +8985 8.87096 -0.000027916 -0.000018882 +8986 8.87192 -0.000027898 -0.000018867 +8987 8.87289 -0.000027880 -0.000018853 +8988 8.87385 -0.000027862 -0.000018839 +8989 8.87482 -0.000027844 -0.000018824 +8990 8.87578 -0.000027825 -0.000018810 +8991 8.87675 -0.000027807 -0.000018796 +8992 8.87771 -0.000027789 -0.000018781 +8993 8.87868 -0.000027771 -0.000018767 +8994 8.87964 -0.000027753 -0.000018753 +8995 8.88061 -0.000027735 -0.000018738 +8996 8.88157 -0.000027717 -0.000018724 +8997 8.88254 -0.000027699 -0.000018710 +8998 8.88350 -0.000027681 -0.000018696 +8999 8.88447 -0.000027663 -0.000018682 +9000 8.88543 -0.000027645 -0.000018667 +9001 8.88640 -0.000027627 -0.000018653 +9002 8.88736 -0.000027609 -0.000018639 +9003 8.88833 -0.000027591 -0.000018625 +9004 8.88929 -0.000027573 -0.000018611 +9005 8.89026 -0.000027555 -0.000018597 +9006 8.89122 -0.000027537 -0.000018582 +9007 8.89219 -0.000027519 -0.000018568 +9008 8.89315 -0.000027501 -0.000018554 +9009 8.89412 -0.000027483 -0.000018540 +9010 8.89508 -0.000027465 -0.000018526 +9011 8.89605 -0.000027447 -0.000018512 +9012 8.89701 -0.000027429 -0.000018498 +9013 8.89798 -0.000027412 -0.000018484 +9014 8.89894 -0.000027394 -0.000018470 +9015 8.89991 -0.000027376 -0.000018456 +9016 8.90088 -0.000027358 -0.000018442 +9017 8.90184 -0.000027340 -0.000018428 +9018 8.90281 -0.000027323 -0.000018414 +9019 8.90377 -0.000027305 -0.000018400 +9020 8.90474 -0.000027287 -0.000018386 +9021 8.90570 -0.000027269 -0.000018372 +9022 8.90667 -0.000027252 -0.000018358 +9023 8.90763 -0.000027234 -0.000018344 +9024 8.90860 -0.000027216 -0.000018330 +9025 8.90956 -0.000027198 -0.000018316 +9026 8.91053 -0.000027181 -0.000018302 +9027 8.91149 -0.000027163 -0.000018289 +9028 8.91246 -0.000027146 -0.000018275 +9029 8.91342 -0.000027128 -0.000018261 +9030 8.91439 -0.000027110 -0.000018247 +9031 8.91535 -0.000027093 -0.000018233 +9032 8.91632 -0.000027075 -0.000018219 +9033 8.91728 -0.000027057 -0.000018206 +9034 8.91825 -0.000027040 -0.000018192 +9035 8.91921 -0.000027022 -0.000018178 +9036 8.92018 -0.000027005 -0.000018164 +9037 8.92114 -0.000026987 -0.000018151 +9038 8.92211 -0.000026970 -0.000018137 +9039 8.92307 -0.000026952 -0.000018123 +9040 8.92404 -0.000026935 -0.000018109 +9041 8.92500 -0.000026917 -0.000018096 +9042 8.92597 -0.000026900 -0.000018082 +9043 8.92693 -0.000026882 -0.000018068 +9044 8.92790 -0.000026865 -0.000018055 +9045 8.92886 -0.000026848 -0.000018041 +9046 8.92983 -0.000026830 -0.000018027 +9047 8.93079 -0.000026813 -0.000018014 +9048 8.93176 -0.000026795 -0.000018000 +9049 8.93272 -0.000026778 -0.000017987 +9050 8.93369 -0.000026761 -0.000017973 +9051 8.93465 -0.000026743 -0.000017959 +9052 8.93562 -0.000026726 -0.000017946 +9053 8.93658 -0.000026709 -0.000017932 +9054 8.93755 -0.000026691 -0.000017919 +9055 8.93851 -0.000026674 -0.000017905 +9056 8.93948 -0.000026657 -0.000017892 +9057 8.94044 -0.000026640 -0.000017878 +9058 8.94141 -0.000026622 -0.000017865 +9059 8.94237 -0.000026605 -0.000017851 +9060 8.94334 -0.000026588 -0.000017838 +9061 8.94430 -0.000026571 -0.000017824 +9062 8.94527 -0.000026554 -0.000017811 +9063 8.94623 -0.000026536 -0.000017797 +9064 8.94720 -0.000026519 -0.000017784 +9065 8.94816 -0.000026502 -0.000017770 +9066 8.94913 -0.000026485 -0.000017757 +9067 8.95009 -0.000026468 -0.000017744 +9068 8.95106 -0.000026451 -0.000017730 +9069 8.95202 -0.000026434 -0.000017717 +9070 8.95299 -0.000026416 -0.000017703 +9071 8.95395 -0.000026399 -0.000017690 +9072 8.95492 -0.000026382 -0.000017677 +9073 8.95588 -0.000026365 -0.000017663 +9074 8.95685 -0.000026348 -0.000017650 +9075 8.95781 -0.000026331 -0.000017637 +9076 8.95878 -0.000026314 -0.000017624 +9077 8.95974 -0.000026297 -0.000017610 +9078 8.96071 -0.000026280 -0.000017597 +9079 8.96167 -0.000026263 -0.000017584 +9080 8.96264 -0.000026246 -0.000017570 +9081 8.96360 -0.000026229 -0.000017557 +9082 8.96457 -0.000026212 -0.000017544 +9083 8.96553 -0.000026195 -0.000017531 +9084 8.96650 -0.000026179 -0.000017518 +9085 8.96746 -0.000026162 -0.000017504 +9086 8.96843 -0.000026145 -0.000017491 +9087 8.96939 -0.000026128 -0.000017478 +9088 8.97036 -0.000026111 -0.000017465 +9089 8.97132 -0.000026094 -0.000017452 +9090 8.97229 -0.000026077 -0.000017439 +9091 8.97325 -0.000026060 -0.000017425 +9092 8.97422 -0.000026044 -0.000017412 +9093 8.97518 -0.000026027 -0.000017399 +9094 8.97615 -0.000026010 -0.000017386 +9095 8.97711 -0.000025993 -0.000017373 +9096 8.97808 -0.000025977 -0.000017360 +9097 8.97904 -0.000025960 -0.000017347 +9098 8.98001 -0.000025943 -0.000017334 +9099 8.98097 -0.000025926 -0.000017321 +9100 8.98194 -0.000025910 -0.000017308 +9101 8.98290 -0.000025893 -0.000017295 +9102 8.98387 -0.000025876 -0.000017282 +9103 8.98483 -0.000025860 -0.000017269 +9104 8.98580 -0.000025843 -0.000017256 +9105 8.98676 -0.000025826 -0.000017243 +9106 8.98773 -0.000025810 -0.000017230 +9107 8.98869 -0.000025793 -0.000017217 +9108 8.98966 -0.000025776 -0.000017204 +9109 8.99062 -0.000025760 -0.000017191 +9110 8.99159 -0.000025743 -0.000017178 +9111 8.99255 -0.000025727 -0.000017165 +9112 8.99352 -0.000025710 -0.000017152 +9113 8.99448 -0.000025694 -0.000017140 +9114 8.99545 -0.000025677 -0.000017127 +9115 8.99641 -0.000025661 -0.000017114 +9116 8.99738 -0.000025644 -0.000017101 +9117 8.99834 -0.000025628 -0.000017088 +9118 8.99931 -0.000025611 -0.000017075 +9119 9.00028 -0.000025595 -0.000017063 +9120 9.00124 -0.000025578 -0.000017050 +9121 9.00221 -0.000025562 -0.000017037 +9122 9.00317 -0.000025545 -0.000017024 +9123 9.00414 -0.000025529 -0.000017011 +9124 9.00510 -0.000025512 -0.000016999 +9125 9.00607 -0.000025496 -0.000016986 +9126 9.00703 -0.000025480 -0.000016973 +9127 9.00800 -0.000025463 -0.000016960 +9128 9.00896 -0.000025447 -0.000016948 +9129 9.00993 -0.000025431 -0.000016935 +9130 9.01089 -0.000025414 -0.000016922 +9131 9.01186 -0.000025398 -0.000016910 +9132 9.01282 -0.000025382 -0.000016897 +9133 9.01379 -0.000025365 -0.000016884 +9134 9.01475 -0.000025349 -0.000016872 +9135 9.01572 -0.000025333 -0.000016859 +9136 9.01668 -0.000025316 -0.000016846 +9137 9.01765 -0.000025300 -0.000016834 +9138 9.01861 -0.000025284 -0.000016821 +9139 9.01958 -0.000025268 -0.000016809 +9140 9.02054 -0.000025251 -0.000016796 +9141 9.02151 -0.000025235 -0.000016783 +9142 9.02247 -0.000025219 -0.000016771 +9143 9.02344 -0.000025203 -0.000016758 +9144 9.02440 -0.000025187 -0.000016746 +9145 9.02537 -0.000025171 -0.000016733 +9146 9.02633 -0.000025154 -0.000016721 +9147 9.02730 -0.000025138 -0.000016708 +9148 9.02826 -0.000025122 -0.000016696 +9149 9.02923 -0.000025106 -0.000016683 +9150 9.03019 -0.000025090 -0.000016671 +9151 9.03116 -0.000025074 -0.000016658 +9152 9.03212 -0.000025058 -0.000016646 +9153 9.03309 -0.000025042 -0.000016633 +9154 9.03405 -0.000025026 -0.000016621 +9155 9.03502 -0.000025010 -0.000016609 +9156 9.03598 -0.000024994 -0.000016596 +9157 9.03695 -0.000024978 -0.000016584 +9158 9.03791 -0.000024962 -0.000016571 +9159 9.03888 -0.000024946 -0.000016559 +9160 9.03984 -0.000024930 -0.000016547 +9161 9.04081 -0.000024914 -0.000016534 +9162 9.04177 -0.000024898 -0.000016522 +9163 9.04274 -0.000024882 -0.000016510 +9164 9.04370 -0.000024866 -0.000016497 +9165 9.04467 -0.000024850 -0.000016485 +9166 9.04563 -0.000024834 -0.000016473 +9167 9.04660 -0.000024818 -0.000016460 +9168 9.04756 -0.000024802 -0.000016448 +9169 9.04853 -0.000024786 -0.000016436 +9170 9.04949 -0.000024771 -0.000016423 +9171 9.05046 -0.000024755 -0.000016411 +9172 9.05142 -0.000024739 -0.000016399 +9173 9.05239 -0.000024723 -0.000016387 +9174 9.05335 -0.000024707 -0.000016374 +9175 9.05432 -0.000024692 -0.000016362 +9176 9.05528 -0.000024676 -0.000016350 +9177 9.05625 -0.000024660 -0.000016338 +9178 9.05721 -0.000024644 -0.000016326 +9179 9.05818 -0.000024628 -0.000016314 +9180 9.05914 -0.000024613 -0.000016301 +9181 9.06011 -0.000024597 -0.000016289 +9182 9.06107 -0.000024581 -0.000016277 +9183 9.06204 -0.000024566 -0.000016265 +9184 9.06300 -0.000024550 -0.000016253 +9185 9.06397 -0.000024534 -0.000016241 +9186 9.06493 -0.000024519 -0.000016229 +9187 9.06590 -0.000024503 -0.000016217 +9188 9.06686 -0.000024487 -0.000016204 +9189 9.06783 -0.000024472 -0.000016192 +9190 9.06879 -0.000024456 -0.000016180 +9191 9.06976 -0.000024440 -0.000016168 +9192 9.07072 -0.000024425 -0.000016156 +9193 9.07169 -0.000024409 -0.000016144 +9194 9.07265 -0.000024394 -0.000016132 +9195 9.07362 -0.000024378 -0.000016120 +9196 9.07458 -0.000024363 -0.000016108 +9197 9.07555 -0.000024347 -0.000016096 +9198 9.07651 -0.000024331 -0.000016084 +9199 9.07748 -0.000024316 -0.000016072 +9200 9.07844 -0.000024300 -0.000016060 +9201 9.07941 -0.000024285 -0.000016048 +9202 9.08037 -0.000024269 -0.000016036 +9203 9.08134 -0.000024254 -0.000016024 +9204 9.08230 -0.000024239 -0.000016013 +9205 9.08327 -0.000024223 -0.000016001 +9206 9.08423 -0.000024208 -0.000015989 +9207 9.08520 -0.000024192 -0.000015977 +9208 9.08616 -0.000024177 -0.000015965 +9209 9.08713 -0.000024161 -0.000015953 +9210 9.08809 -0.000024146 -0.000015941 +9211 9.08906 -0.000024131 -0.000015929 +9212 9.09002 -0.000024115 -0.000015918 +9213 9.09099 -0.000024100 -0.000015906 +9214 9.09195 -0.000024085 -0.000015894 +9215 9.09292 -0.000024069 -0.000015882 +9216 9.09388 -0.000024054 -0.000015870 +9217 9.09485 -0.000024039 -0.000015859 +9218 9.09581 -0.000024023 -0.000015847 +9219 9.09678 -0.000024008 -0.000015835 +9220 9.09774 -0.000023993 -0.000015823 +9221 9.09871 -0.000023977 -0.000015812 +9222 9.09967 -0.000023962 -0.000015800 +9223 9.10064 -0.000023947 -0.000015788 +9224 9.10161 -0.000023932 -0.000015776 +9225 9.10257 -0.000023917 -0.000015765 +9226 9.10354 -0.000023901 -0.000015753 +9227 9.10450 -0.000023886 -0.000015741 +9228 9.10547 -0.000023871 -0.000015730 +9229 9.10643 -0.000023856 -0.000015718 +9230 9.10740 -0.000023841 -0.000015706 +9231 9.10836 -0.000023825 -0.000015695 +9232 9.10933 -0.000023810 -0.000015683 +9233 9.11029 -0.000023795 -0.000015671 +9234 9.11126 -0.000023780 -0.000015660 +9235 9.11222 -0.000023765 -0.000015648 +9236 9.11319 -0.000023750 -0.000015637 +9237 9.11415 -0.000023735 -0.000015625 +9238 9.11512 -0.000023720 -0.000015613 +9239 9.11608 -0.000023705 -0.000015602 +9240 9.11705 -0.000023690 -0.000015590 +9241 9.11801 -0.000023675 -0.000015579 +9242 9.11898 -0.000023660 -0.000015567 +9243 9.11994 -0.000023645 -0.000015556 +9244 9.12091 -0.000023629 -0.000015544 +9245 9.12187 -0.000023614 -0.000015533 +9246 9.12284 -0.000023600 -0.000015521 +9247 9.12380 -0.000023585 -0.000015510 +9248 9.12477 -0.000023570 -0.000015498 +9249 9.12573 -0.000023555 -0.000015487 +9250 9.12670 -0.000023540 -0.000015475 +9251 9.12766 -0.000023525 -0.000015464 +9252 9.12863 -0.000023510 -0.000015452 +9253 9.12959 -0.000023495 -0.000015441 +9254 9.13056 -0.000023480 -0.000015430 +9255 9.13152 -0.000023465 -0.000015418 +9256 9.13249 -0.000023450 -0.000015407 +9257 9.13345 -0.000023435 -0.000015395 +9258 9.13442 -0.000023421 -0.000015384 +9259 9.13538 -0.000023406 -0.000015373 +9260 9.13635 -0.000023391 -0.000015361 +9261 9.13731 -0.000023376 -0.000015350 +9262 9.13828 -0.000023361 -0.000015339 +9263 9.13924 -0.000023346 -0.000015327 +9264 9.14021 -0.000023332 -0.000015316 +9265 9.14117 -0.000023317 -0.000015305 +9266 9.14214 -0.000023302 -0.000015293 +9267 9.14310 -0.000023287 -0.000015282 +9268 9.14407 -0.000023273 -0.000015271 +9269 9.14503 -0.000023258 -0.000015259 +9270 9.14600 -0.000023243 -0.000015248 +9271 9.14696 -0.000023228 -0.000015237 +9272 9.14793 -0.000023214 -0.000015226 +9273 9.14889 -0.000023199 -0.000015214 +9274 9.14986 -0.000023184 -0.000015203 +9275 9.15082 -0.000023170 -0.000015192 +9276 9.15179 -0.000023155 -0.000015181 +9277 9.15275 -0.000023140 -0.000015170 +9278 9.15372 -0.000023126 -0.000015158 +9279 9.15468 -0.000023111 -0.000015147 +9280 9.15565 -0.000023097 -0.000015136 +9281 9.15661 -0.000023082 -0.000015125 +9282 9.15758 -0.000023067 -0.000015114 +9283 9.15854 -0.000023053 -0.000015103 +9284 9.15951 -0.000023038 -0.000015091 +9285 9.16047 -0.000023024 -0.000015080 +9286 9.16144 -0.000023009 -0.000015069 +9287 9.16240 -0.000022995 -0.000015058 +9288 9.16337 -0.000022980 -0.000015047 +9289 9.16433 -0.000022966 -0.000015036 +9290 9.16530 -0.000022951 -0.000015025 +9291 9.16626 -0.000022937 -0.000015014 +9292 9.16723 -0.000022922 -0.000015003 +9293 9.16819 -0.000022908 -0.000014992 +9294 9.16916 -0.000022893 -0.000014981 +9295 9.17012 -0.000022879 -0.000014969 +9296 9.17109 -0.000022864 -0.000014958 +9297 9.17205 -0.000022850 -0.000014947 +9298 9.17302 -0.000022835 -0.000014936 +9299 9.17398 -0.000022821 -0.000014925 +9300 9.17495 -0.000022807 -0.000014914 +9301 9.17591 -0.000022792 -0.000014904 +9302 9.17688 -0.000022778 -0.000014893 +9303 9.17784 -0.000022763 -0.000014882 +9304 9.17881 -0.000022749 -0.000014871 +9305 9.17977 -0.000022735 -0.000014860 +9306 9.18074 -0.000022720 -0.000014849 +9307 9.18170 -0.000022706 -0.000014838 +9308 9.18267 -0.000022692 -0.000014827 +9309 9.18363 -0.000022677 -0.000014816 +9310 9.18460 -0.000022663 -0.000014805 +9311 9.18556 -0.000022649 -0.000014794 +9312 9.18653 -0.000022635 -0.000014783 +9313 9.18749 -0.000022620 -0.000014772 +9314 9.18846 -0.000022606 -0.000014762 +9315 9.18942 -0.000022592 -0.000014751 +9316 9.19039 -0.000022578 -0.000014740 +9317 9.19135 -0.000022563 -0.000014729 +9318 9.19232 -0.000022549 -0.000014718 +9319 9.19328 -0.000022535 -0.000014707 +9320 9.19425 -0.000022521 -0.000014697 +9321 9.19521 -0.000022507 -0.000014686 +9322 9.19618 -0.000022492 -0.000014675 +9323 9.19714 -0.000022478 -0.000014664 +9324 9.19811 -0.000022464 -0.000014654 +9325 9.19907 -0.000022450 -0.000014643 +9326 9.20004 -0.000022436 -0.000014632 +9327 9.20101 -0.000022422 -0.000014621 +9328 9.20197 -0.000022408 -0.000014611 +9329 9.20294 -0.000022394 -0.000014600 +9330 9.20390 -0.000022380 -0.000014589 +9331 9.20487 -0.000022365 -0.000014578 +9332 9.20583 -0.000022351 -0.000014568 +9333 9.20680 -0.000022337 -0.000014557 +9334 9.20776 -0.000022323 -0.000014546 +9335 9.20873 -0.000022309 -0.000014536 +9336 9.20969 -0.000022295 -0.000014525 +9337 9.21066 -0.000022281 -0.000014514 +9338 9.21162 -0.000022267 -0.000014504 +9339 9.21259 -0.000022253 -0.000014493 +9340 9.21355 -0.000022239 -0.000014483 +9341 9.21452 -0.000022225 -0.000014472 +9342 9.21548 -0.000022211 -0.000014461 +9343 9.21645 -0.000022197 -0.000014451 +9344 9.21741 -0.000022183 -0.000014440 +9345 9.21838 -0.000022169 -0.000014430 +9346 9.21934 -0.000022156 -0.000014419 +9347 9.22031 -0.000022142 -0.000014408 +9348 9.22127 -0.000022128 -0.000014398 +9349 9.22224 -0.000022114 -0.000014387 +9350 9.22320 -0.000022100 -0.000014377 +9351 9.22417 -0.000022086 -0.000014366 +9352 9.22513 -0.000022072 -0.000014356 +9353 9.22610 -0.000022058 -0.000014345 +9354 9.22706 -0.000022045 -0.000014335 +9355 9.22803 -0.000022031 -0.000014324 +9356 9.22899 -0.000022017 -0.000014314 +9357 9.22996 -0.000022003 -0.000014303 +9358 9.23092 -0.000021989 -0.000014293 +9359 9.23189 -0.000021976 -0.000014282 +9360 9.23285 -0.000021962 -0.000014272 +9361 9.23382 -0.000021948 -0.000014261 +9362 9.23478 -0.000021934 -0.000014251 +9363 9.23575 -0.000021920 -0.000014241 +9364 9.23671 -0.000021907 -0.000014230 +9365 9.23768 -0.000021893 -0.000014220 +9366 9.23864 -0.000021879 -0.000014209 +9367 9.23961 -0.000021866 -0.000014199 +9368 9.24057 -0.000021852 -0.000014189 +9369 9.24154 -0.000021838 -0.000014178 +9370 9.24250 -0.000021825 -0.000014168 +9371 9.24347 -0.000021811 -0.000014158 +9372 9.24443 -0.000021797 -0.000014147 +9373 9.24540 -0.000021784 -0.000014137 +9374 9.24636 -0.000021770 -0.000014127 +9375 9.24733 -0.000021756 -0.000014116 +9376 9.24829 -0.000021743 -0.000014106 +9377 9.24926 -0.000021729 -0.000014096 +9378 9.25022 -0.000021715 -0.000014085 +9379 9.25119 -0.000021702 -0.000014075 +9380 9.25215 -0.000021688 -0.000014065 +9381 9.25312 -0.000021675 -0.000014055 +9382 9.25408 -0.000021661 -0.000014044 +9383 9.25505 -0.000021648 -0.000014034 +9384 9.25601 -0.000021634 -0.000014024 +9385 9.25698 -0.000021621 -0.000014014 +9386 9.25794 -0.000021607 -0.000014003 +9387 9.25891 -0.000021594 -0.000013993 +9388 9.25987 -0.000021580 -0.000013983 +9389 9.26084 -0.000021567 -0.000013973 +9390 9.26180 -0.000021553 -0.000013963 +9391 9.26277 -0.000021540 -0.000013952 +9392 9.26373 -0.000021526 -0.000013942 +9393 9.26470 -0.000021513 -0.000013932 +9394 9.26566 -0.000021499 -0.000013922 +9395 9.26663 -0.000021486 -0.000013912 +9396 9.26759 -0.000021472 -0.000013902 +9397 9.26856 -0.000021459 -0.000013891 +9398 9.26952 -0.000021446 -0.000013881 +9399 9.27049 -0.000021432 -0.000013871 +9400 9.27145 -0.000021419 -0.000013861 +9401 9.27242 -0.000021405 -0.000013851 +9402 9.27338 -0.000021392 -0.000013841 +9403 9.27435 -0.000021379 -0.000013831 +9404 9.27531 -0.000021365 -0.000013821 +9405 9.27628 -0.000021352 -0.000013811 +9406 9.27724 -0.000021339 -0.000013801 +9407 9.27821 -0.000021325 -0.000013791 +9408 9.27917 -0.000021312 -0.000013781 +9409 9.28014 -0.000021299 -0.000013771 +9410 9.28110 -0.000021286 -0.000013761 +9411 9.28207 -0.000021272 -0.000013751 +9412 9.28303 -0.000021259 -0.000013741 +9413 9.28400 -0.000021246 -0.000013731 +9414 9.28496 -0.000021232 -0.000013721 +9415 9.28593 -0.000021219 -0.000013711 +9416 9.28689 -0.000021206 -0.000013701 +9417 9.28786 -0.000021193 -0.000013691 +9418 9.28882 -0.000021180 -0.000013681 +9419 9.28979 -0.000021166 -0.000013671 +9420 9.29075 -0.000021153 -0.000013661 +9421 9.29172 -0.000021140 -0.000013651 +9422 9.29268 -0.000021127 -0.000013641 +9423 9.29365 -0.000021114 -0.000013631 +9424 9.29461 -0.000021101 -0.000013621 +9425 9.29558 -0.000021087 -0.000013611 +9426 9.29654 -0.000021074 -0.000013601 +9427 9.29751 -0.000021061 -0.000013591 +9428 9.29847 -0.000021048 -0.000013582 +9429 9.29944 -0.000021035 -0.000013572 +9430 9.30041 -0.000021022 -0.000013562 +9431 9.30137 -0.000021009 -0.000013552 +9432 9.30234 -0.000020996 -0.000013542 +9433 9.30330 -0.000020983 -0.000013532 +9434 9.30427 -0.000020970 -0.000013523 +9435 9.30523 -0.000020957 -0.000013513 +9436 9.30620 -0.000020944 -0.000013503 +9437 9.30716 -0.000020930 -0.000013493 +9438 9.30813 -0.000020917 -0.000013483 +9439 9.30909 -0.000020904 -0.000013474 +9440 9.31006 -0.000020891 -0.000013464 +9441 9.31102 -0.000020878 -0.000013454 +9442 9.31199 -0.000020865 -0.000013444 +9443 9.31295 -0.000020853 -0.000013435 +9444 9.31392 -0.000020840 -0.000013425 +9445 9.31488 -0.000020827 -0.000013415 +9446 9.31585 -0.000020814 -0.000013405 +9447 9.31681 -0.000020801 -0.000013396 +9448 9.31778 -0.000020788 -0.000013386 +9449 9.31874 -0.000020775 -0.000013376 +9450 9.31971 -0.000020762 -0.000013367 +9451 9.32067 -0.000020749 -0.000013357 +9452 9.32164 -0.000020736 -0.000013347 +9453 9.32260 -0.000020723 -0.000013337 +9454 9.32357 -0.000020710 -0.000013328 +9455 9.32453 -0.000020698 -0.000013318 +9456 9.32550 -0.000020685 -0.000013309 +9457 9.32646 -0.000020672 -0.000013299 +9458 9.32743 -0.000020659 -0.000013289 +9459 9.32839 -0.000020646 -0.000013280 +9460 9.32936 -0.000020633 -0.000013270 +9461 9.33032 -0.000020621 -0.000013260 +9462 9.33129 -0.000020608 -0.000013251 +9463 9.33225 -0.000020595 -0.000013241 +9464 9.33322 -0.000020582 -0.000013232 +9465 9.33418 -0.000020570 -0.000013222 +9466 9.33515 -0.000020557 -0.000013213 +9467 9.33611 -0.000020544 -0.000013203 +9468 9.33708 -0.000020531 -0.000013193 +9469 9.33804 -0.000020519 -0.000013184 +9470 9.33901 -0.000020506 -0.000013174 +9471 9.33997 -0.000020493 -0.000013165 +9472 9.34094 -0.000020480 -0.000013155 +9473 9.34190 -0.000020468 -0.000013146 +9474 9.34287 -0.000020455 -0.000013136 +9475 9.34383 -0.000020442 -0.000013127 +9476 9.34480 -0.000020430 -0.000013117 +9477 9.34576 -0.000020417 -0.000013108 +9478 9.34673 -0.000020404 -0.000013098 +9479 9.34769 -0.000020392 -0.000013089 +9480 9.34866 -0.000020379 -0.000013079 +9481 9.34962 -0.000020367 -0.000013070 +9482 9.35059 -0.000020354 -0.000013061 +9483 9.35155 -0.000020341 -0.000013051 +9484 9.35252 -0.000020329 -0.000013042 +9485 9.35348 -0.000020316 -0.000013032 +9486 9.35445 -0.000020304 -0.000013023 +9487 9.35541 -0.000020291 -0.000013013 +9488 9.35638 -0.000020278 -0.000013004 +9489 9.35734 -0.000020266 -0.000012995 +9490 9.35831 -0.000020253 -0.000012985 +9491 9.35927 -0.000020241 -0.000012976 +9492 9.36024 -0.000020228 -0.000012967 +9493 9.36120 -0.000020216 -0.000012957 +9494 9.36217 -0.000020203 -0.000012948 +9495 9.36313 -0.000020191 -0.000012939 +9496 9.36410 -0.000020178 -0.000012929 +9497 9.36506 -0.000020166 -0.000012920 +9498 9.36603 -0.000020153 -0.000012911 +9499 9.36699 -0.000020141 -0.000012901 +9500 9.36796 -0.000020129 -0.000012892 +9501 9.36892 -0.000020116 -0.000012883 +9502 9.36989 -0.000020104 -0.000012873 +9503 9.37085 -0.000020091 -0.000012864 +9504 9.37182 -0.000020079 -0.000012855 +9505 9.37278 -0.000020066 -0.000012846 +9506 9.37375 -0.000020054 -0.000012836 +9507 9.37471 -0.000020042 -0.000012827 +9508 9.37568 -0.000020029 -0.000012818 +9509 9.37664 -0.000020017 -0.000012809 +9510 9.37761 -0.000020005 -0.000012799 +9511 9.37857 -0.000019992 -0.000012790 +9512 9.37954 -0.000019980 -0.000012781 +9513 9.38050 -0.000019968 -0.000012772 +9514 9.38147 -0.000019955 -0.000012763 +9515 9.38243 -0.000019943 -0.000012753 +9516 9.38340 -0.000019931 -0.000012744 +9517 9.38436 -0.000019918 -0.000012735 +9518 9.38533 -0.000019906 -0.000012726 +9519 9.38629 -0.000019894 -0.000012717 +9520 9.38726 -0.000019882 -0.000012708 +9521 9.38822 -0.000019869 -0.000012698 +9522 9.38919 -0.000019857 -0.000012689 +9523 9.39015 -0.000019845 -0.000012680 +9524 9.39112 -0.000019833 -0.000012671 +9525 9.39208 -0.000019820 -0.000012662 +9526 9.39305 -0.000019808 -0.000012653 +9527 9.39401 -0.000019796 -0.000012644 +9528 9.39498 -0.000019784 -0.000012635 +9529 9.39594 -0.000019771 -0.000012626 +9530 9.39691 -0.000019759 -0.000012616 +9531 9.39787 -0.000019747 -0.000012607 +9532 9.39884 -0.000019735 -0.000012598 +9533 9.39980 -0.000019723 -0.000012589 +9534 9.40077 -0.000019711 -0.000012580 +9535 9.40174 -0.000019699 -0.000012571 +9536 9.40270 -0.000019686 -0.000012562 +9537 9.40367 -0.000019674 -0.000012553 +9538 9.40463 -0.000019662 -0.000012544 +9539 9.40560 -0.000019650 -0.000012535 +9540 9.40656 -0.000019638 -0.000012526 +9541 9.40753 -0.000019626 -0.000012517 +9542 9.40849 -0.000019614 -0.000012508 +9543 9.40946 -0.000019602 -0.000012499 +9544 9.41042 -0.000019590 -0.000012490 +9545 9.41139 -0.000019578 -0.000012481 +9546 9.41235 -0.000019566 -0.000012472 +9547 9.41332 -0.000019554 -0.000012463 +9548 9.41428 -0.000019542 -0.000012454 +9549 9.41525 -0.000019530 -0.000012445 +9550 9.41621 -0.000019518 -0.000012437 +9551 9.41718 -0.000019506 -0.000012428 +9552 9.41814 -0.000019494 -0.000012419 +9553 9.41911 -0.000019482 -0.000012410 +9554 9.42007 -0.000019470 -0.000012401 +9555 9.42104 -0.000019458 -0.000012392 +9556 9.42200 -0.000019446 -0.000012383 +9557 9.42297 -0.000019434 -0.000012374 +9558 9.42393 -0.000019422 -0.000012365 +9559 9.42490 -0.000019410 -0.000012357 +9560 9.42586 -0.000019398 -0.000012348 +9561 9.42683 -0.000019386 -0.000012339 +9562 9.42779 -0.000019374 -0.000012330 +9563 9.42876 -0.000019362 -0.000012321 +9564 9.42972 -0.000019350 -0.000012312 +9565 9.43069 -0.000019338 -0.000012304 +9566 9.43165 -0.000019327 -0.000012295 +9567 9.43262 -0.000019315 -0.000012286 +9568 9.43358 -0.000019303 -0.000012277 +9569 9.43455 -0.000019291 -0.000012268 +9570 9.43551 -0.000019279 -0.000012260 +9571 9.43648 -0.000019267 -0.000012251 +9572 9.43744 -0.000019256 -0.000012242 +9573 9.43841 -0.000019244 -0.000012233 +9574 9.43937 -0.000019232 -0.000012225 +9575 9.44034 -0.000019220 -0.000012216 +9576 9.44130 -0.000019208 -0.000012207 +9577 9.44227 -0.000019197 -0.000012198 +9578 9.44323 -0.000019185 -0.000012190 +9579 9.44420 -0.000019173 -0.000012181 +9580 9.44516 -0.000019161 -0.000012172 +9581 9.44613 -0.000019150 -0.000012163 +9582 9.44709 -0.000019138 -0.000012155 +9583 9.44806 -0.000019126 -0.000012146 +9584 9.44902 -0.000019114 -0.000012137 +9585 9.44999 -0.000019103 -0.000012129 +9586 9.45095 -0.000019091 -0.000012120 +9587 9.45192 -0.000019079 -0.000012111 +9588 9.45288 -0.000019068 -0.000012103 +9589 9.45385 -0.000019056 -0.000012094 +9590 9.45481 -0.000019044 -0.000012085 +9591 9.45578 -0.000019033 -0.000012077 +9592 9.45674 -0.000019021 -0.000012068 +9593 9.45771 -0.000019009 -0.000012060 +9594 9.45867 -0.000018998 -0.000012051 +9595 9.45964 -0.000018986 -0.000012042 +9596 9.46060 -0.000018974 -0.000012034 +9597 9.46157 -0.000018963 -0.000012025 +9598 9.46253 -0.000018951 -0.000012017 +9599 9.46350 -0.000018940 -0.000012008 +9600 9.46446 -0.000018928 -0.000011999 +9601 9.46543 -0.000018916 -0.000011991 +9602 9.46639 -0.000018905 -0.000011982 +9603 9.46736 -0.000018893 -0.000011974 +9604 9.46832 -0.000018882 -0.000011965 +9605 9.46929 -0.000018870 -0.000011957 +9606 9.47025 -0.000018859 -0.000011948 +9607 9.47122 -0.000018847 -0.000011940 +9608 9.47218 -0.000018836 -0.000011931 +9609 9.47315 -0.000018824 -0.000011923 +9610 9.47411 -0.000018813 -0.000011914 +9611 9.47508 -0.000018801 -0.000011906 +9612 9.47604 -0.000018790 -0.000011897 +9613 9.47701 -0.000018778 -0.000011889 +9614 9.47797 -0.000018767 -0.000011880 +9615 9.47894 -0.000018755 -0.000011872 +9616 9.47990 -0.000018744 -0.000011863 +9617 9.48087 -0.000018732 -0.000011855 +9618 9.48183 -0.000018721 -0.000011846 +9619 9.48280 -0.000018710 -0.000011838 +9620 9.48376 -0.000018698 -0.000011830 +9621 9.48473 -0.000018687 -0.000011821 +9622 9.48569 -0.000018675 -0.000011813 +9623 9.48666 -0.000018664 -0.000011804 +9624 9.48762 -0.000018653 -0.000011796 +9625 9.48859 -0.000018641 -0.000011788 +9626 9.48955 -0.000018630 -0.000011779 +9627 9.49052 -0.000018618 -0.000011771 +9628 9.49148 -0.000018607 -0.000011762 +9629 9.49245 -0.000018596 -0.000011754 +9630 9.49341 -0.000018584 -0.000011746 +9631 9.49438 -0.000018573 -0.000011737 +9632 9.49534 -0.000018562 -0.000011729 +9633 9.49631 -0.000018550 -0.000011721 +9634 9.49727 -0.000018539 -0.000011712 +9635 9.49824 -0.000018528 -0.000011704 +9636 9.49920 -0.000018516 -0.000011696 +9637 9.50017 -0.000018505 -0.000011687 +9638 9.50114 -0.000018494 -0.000011679 +9639 9.50210 -0.000018483 -0.000011671 +9640 9.50307 -0.000018471 -0.000011662 +9641 9.50403 -0.000018460 -0.000011654 +9642 9.50500 -0.000018449 -0.000011646 +9643 9.50596 -0.000018438 -0.000011638 +9644 9.50693 -0.000018426 -0.000011629 +9645 9.50789 -0.000018415 -0.000011621 +9646 9.50886 -0.000018404 -0.000011613 +9647 9.50982 -0.000018393 -0.000011605 +9648 9.51079 -0.000018382 -0.000011596 +9649 9.51175 -0.000018370 -0.000011588 +9650 9.51272 -0.000018359 -0.000011580 +9651 9.51368 -0.000018348 -0.000011572 +9652 9.51465 -0.000018337 -0.000011563 +9653 9.51561 -0.000018326 -0.000011555 +9654 9.51658 -0.000018315 -0.000011547 +9655 9.51754 -0.000018303 -0.000011539 +9656 9.51851 -0.000018292 -0.000011531 +9657 9.51947 -0.000018281 -0.000011522 +9658 9.52044 -0.000018270 -0.000011514 +9659 9.52140 -0.000018259 -0.000011506 +9660 9.52237 -0.000018248 -0.000011498 +9661 9.52333 -0.000018237 -0.000011490 +9662 9.52430 -0.000018226 -0.000011482 +9663 9.52526 -0.000018215 -0.000011473 +9664 9.52623 -0.000018204 -0.000011465 +9665 9.52719 -0.000018193 -0.000011457 +9666 9.52816 -0.000018181 -0.000011449 +9667 9.52912 -0.000018170 -0.000011441 +9668 9.53009 -0.000018159 -0.000011433 +9669 9.53105 -0.000018148 -0.000011425 +9670 9.53202 -0.000018137 -0.000011417 +9671 9.53298 -0.000018126 -0.000011409 +9672 9.53395 -0.000018115 -0.000011401 +9673 9.53491 -0.000018104 -0.000011392 +9674 9.53588 -0.000018093 -0.000011384 +9675 9.53684 -0.000018082 -0.000011376 +9676 9.53781 -0.000018071 -0.000011368 +9677 9.53877 -0.000018060 -0.000011360 +9678 9.53974 -0.000018049 -0.000011352 +9679 9.54070 -0.000018038 -0.000011344 +9680 9.54167 -0.000018028 -0.000011336 +9681 9.54263 -0.000018017 -0.000011328 +9682 9.54360 -0.000018006 -0.000011320 +9683 9.54456 -0.000017995 -0.000011312 +9684 9.54553 -0.000017984 -0.000011304 +9685 9.54649 -0.000017973 -0.000011296 +9686 9.54746 -0.000017962 -0.000011288 +9687 9.54842 -0.000017951 -0.000011280 +9688 9.54939 -0.000017940 -0.000011272 +9689 9.55035 -0.000017929 -0.000011264 +9690 9.55132 -0.000017919 -0.000011256 +9691 9.55228 -0.000017908 -0.000011248 +9692 9.55325 -0.000017897 -0.000011240 +9693 9.55421 -0.000017886 -0.000011232 +9694 9.55518 -0.000017875 -0.000011224 +9695 9.55614 -0.000017864 -0.000011216 +9696 9.55711 -0.000017854 -0.000011209 +9697 9.55807 -0.000017843 -0.000011201 +9698 9.55904 -0.000017832 -0.000011193 +9699 9.56000 -0.000017821 -0.000011185 +9700 9.56097 -0.000017810 -0.000011177 +9701 9.56193 -0.000017800 -0.000011169 +9702 9.56290 -0.000017789 -0.000011161 +9703 9.56386 -0.000017778 -0.000011153 +9704 9.56483 -0.000017767 -0.000011145 +9705 9.56579 -0.000017756 -0.000011137 +9706 9.56676 -0.000017746 -0.000011130 +9707 9.56772 -0.000017735 -0.000011122 +9708 9.56869 -0.000017724 -0.000011114 +9709 9.56965 -0.000017714 -0.000011106 +9710 9.57062 -0.000017703 -0.000011098 +9711 9.57158 -0.000017692 -0.000011090 +9712 9.57255 -0.000017681 -0.000011083 +9713 9.57351 -0.000017671 -0.000011075 +9714 9.57448 -0.000017660 -0.000011067 +9715 9.57544 -0.000017649 -0.000011059 +9716 9.57641 -0.000017639 -0.000011051 +9717 9.57737 -0.000017628 -0.000011044 +9718 9.57834 -0.000017617 -0.000011036 +9719 9.57930 -0.000017607 -0.000011028 +9720 9.58027 -0.000017596 -0.000011020 +9721 9.58123 -0.000017585 -0.000011012 +9722 9.58220 -0.000017575 -0.000011005 +9723 9.58316 -0.000017564 -0.000010997 +9724 9.58413 -0.000017554 -0.000010989 +9725 9.58509 -0.000017543 -0.000010981 +9726 9.58606 -0.000017532 -0.000010974 +9727 9.58702 -0.000017522 -0.000010966 +9728 9.58799 -0.000017511 -0.000010958 +9729 9.58895 -0.000017501 -0.000010951 +9730 9.58992 -0.000017490 -0.000010943 +9731 9.59088 -0.000017480 -0.000010935 +9732 9.59185 -0.000017469 -0.000010927 +9733 9.59281 -0.000017458 -0.000010920 +9734 9.59378 -0.000017448 -0.000010912 +9735 9.59474 -0.000017437 -0.000010904 +9736 9.59571 -0.000017427 -0.000010897 +9737 9.59667 -0.000017416 -0.000010889 +9738 9.59764 -0.000017406 -0.000010881 +9739 9.59860 -0.000017395 -0.000010874 +9740 9.59957 -0.000017385 -0.000010866 +9741 9.60054 -0.000017374 -0.000010858 +9742 9.60150 -0.000017364 -0.000010851 +9743 9.60247 -0.000017353 -0.000010843 +9744 9.60343 -0.000017343 -0.000010835 +9745 9.60440 -0.000017333 -0.000010828 +9746 9.60536 -0.000017322 -0.000010820 +9747 9.60633 -0.000017312 -0.000010813 +9748 9.60729 -0.000017301 -0.000010805 +9749 9.60826 -0.000017291 -0.000010797 +9750 9.60922 -0.000017280 -0.000010790 +9751 9.61019 -0.000017270 -0.000010782 +9752 9.61115 -0.000017260 -0.000010775 +9753 9.61212 -0.000017249 -0.000010767 +9754 9.61308 -0.000017239 -0.000010760 +9755 9.61405 -0.000017228 -0.000010752 +9756 9.61501 -0.000017218 -0.000010744 +9757 9.61598 -0.000017208 -0.000010737 +9758 9.61694 -0.000017197 -0.000010729 +9759 9.61791 -0.000017187 -0.000010722 +9760 9.61887 -0.000017177 -0.000010714 +9761 9.61984 -0.000017166 -0.000010707 +9762 9.62080 -0.000017156 -0.000010699 +9763 9.62177 -0.000017146 -0.000010692 +9764 9.62273 -0.000017135 -0.000010684 +9765 9.62370 -0.000017125 -0.000010677 +9766 9.62466 -0.000017115 -0.000010669 +9767 9.62563 -0.000017104 -0.000010662 +9768 9.62659 -0.000017094 -0.000010654 +9769 9.62756 -0.000017084 -0.000010647 +9770 9.62852 -0.000017074 -0.000010639 +9771 9.62949 -0.000017063 -0.000010632 +9772 9.63045 -0.000017053 -0.000010624 +9773 9.63142 -0.000017043 -0.000010617 +9774 9.63238 -0.000017033 -0.000010610 +9775 9.63335 -0.000017022 -0.000010602 +9776 9.63431 -0.000017012 -0.000010595 +9777 9.63528 -0.000017002 -0.000010587 +9778 9.63624 -0.000016992 -0.000010580 +9779 9.63721 -0.000016981 -0.000010572 +9780 9.63817 -0.000016971 -0.000010565 +9781 9.63914 -0.000016961 -0.000010558 +9782 9.64010 -0.000016951 -0.000010550 +9783 9.64107 -0.000016941 -0.000010543 +9784 9.64203 -0.000016931 -0.000010535 +9785 9.64300 -0.000016920 -0.000010528 +9786 9.64396 -0.000016910 -0.000010521 +9787 9.64493 -0.000016900 -0.000010513 +9788 9.64589 -0.000016890 -0.000010506 +9789 9.64686 -0.000016880 -0.000010499 +9790 9.64782 -0.000016870 -0.000010491 +9791 9.64879 -0.000016860 -0.000010484 +9792 9.64975 -0.000016849 -0.000010477 +9793 9.65072 -0.000016839 -0.000010469 +9794 9.65168 -0.000016829 -0.000010462 +9795 9.65265 -0.000016819 -0.000010455 +9796 9.65361 -0.000016809 -0.000010447 +9797 9.65458 -0.000016799 -0.000010440 +9798 9.65554 -0.000016789 -0.000010433 +9799 9.65651 -0.000016779 -0.000010425 +9800 9.65747 -0.000016769 -0.000010418 +9801 9.65844 -0.000016759 -0.000010411 +9802 9.65940 -0.000016749 -0.000010404 +9803 9.66037 -0.000016739 -0.000010396 +9804 9.66133 -0.000016729 -0.000010389 +9805 9.66230 -0.000016719 -0.000010382 +9806 9.66326 -0.000016709 -0.000010374 +9807 9.66423 -0.000016699 -0.000010367 +9808 9.66519 -0.000016689 -0.000010360 +9809 9.66616 -0.000016679 -0.000010353 +9810 9.66712 -0.000016669 -0.000010346 +9811 9.66809 -0.000016659 -0.000010338 +9812 9.66905 -0.000016649 -0.000010331 +9813 9.67002 -0.000016639 -0.000010324 +9814 9.67098 -0.000016629 -0.000010317 +9815 9.67195 -0.000016619 -0.000010309 +9816 9.67291 -0.000016609 -0.000010302 +9817 9.67388 -0.000016599 -0.000010295 +9818 9.67484 -0.000016589 -0.000010288 +9819 9.67581 -0.000016579 -0.000010281 +9820 9.67677 -0.000016569 -0.000010274 +9821 9.67774 -0.000016559 -0.000010266 +9822 9.67870 -0.000016549 -0.000010259 +9823 9.67967 -0.000016539 -0.000010252 +9824 9.68063 -0.000016529 -0.000010245 +9825 9.68160 -0.000016520 -0.000010238 +9826 9.68256 -0.000016510 -0.000010231 +9827 9.68353 -0.000016500 -0.000010223 +9828 9.68449 -0.000016490 -0.000010216 +9829 9.68546 -0.000016480 -0.000010209 +9830 9.68642 -0.000016470 -0.000010202 +9831 9.68739 -0.000016460 -0.000010195 +9832 9.68835 -0.000016451 -0.000010188 +9833 9.68932 -0.000016441 -0.000010181 +9834 9.69028 -0.000016431 -0.000010174 +9835 9.69125 -0.000016421 -0.000010167 +9836 9.69221 -0.000016411 -0.000010160 +9837 9.69318 -0.000016402 -0.000010152 +9838 9.69414 -0.000016392 -0.000010145 +9839 9.69511 -0.000016382 -0.000010138 +9840 9.69607 -0.000016372 -0.000010131 +9841 9.69704 -0.000016362 -0.000010124 +9842 9.69800 -0.000016353 -0.000010117 +9843 9.69897 -0.000016343 -0.000010110 +9844 9.69993 -0.000016333 -0.000010103 +9845 9.70090 -0.000016323 -0.000010096 +9846 9.70187 -0.000016314 -0.000010089 +9847 9.70283 -0.000016304 -0.000010082 +9848 9.70380 -0.000016294 -0.000010075 +9849 9.70476 -0.000016284 -0.000010068 +9850 9.70573 -0.000016275 -0.000010061 +9851 9.70669 -0.000016265 -0.000010054 +9852 9.70766 -0.000016255 -0.000010047 +9853 9.70862 -0.000016246 -0.000010040 +9854 9.70959 -0.000016236 -0.000010033 +9855 9.71055 -0.000016226 -0.000010026 +9856 9.71152 -0.000016217 -0.000010019 +9857 9.71248 -0.000016207 -0.000010012 +9858 9.71345 -0.000016197 -0.000010005 +9859 9.71441 -0.000016188 -0.000009998 +9860 9.71538 -0.000016178 -0.000009991 +9861 9.71634 -0.000016168 -0.000009984 +9862 9.71731 -0.000016159 -0.000009977 +9863 9.71827 -0.000016149 -0.000009970 +9864 9.71924 -0.000016139 -0.000009963 +9865 9.72020 -0.000016130 -0.000009957 +9866 9.72117 -0.000016120 -0.000009950 +9867 9.72213 -0.000016111 -0.000009943 +9868 9.72310 -0.000016101 -0.000009936 +9869 9.72406 -0.000016091 -0.000009929 +9870 9.72503 -0.000016082 -0.000009922 +9871 9.72599 -0.000016072 -0.000009915 +9872 9.72696 -0.000016063 -0.000009908 +9873 9.72792 -0.000016053 -0.000009901 +9874 9.72889 -0.000016044 -0.000009894 +9875 9.72985 -0.000016034 -0.000009888 +9876 9.73082 -0.000016025 -0.000009881 +9877 9.73178 -0.000016015 -0.000009874 +9878 9.73275 -0.000016006 -0.000009867 +9879 9.73371 -0.000015996 -0.000009860 +9880 9.73468 -0.000015986 -0.000009853 +9881 9.73564 -0.000015977 -0.000009846 +9882 9.73661 -0.000015967 -0.000009840 +9883 9.73757 -0.000015958 -0.000009833 +9884 9.73854 -0.000015949 -0.000009826 +9885 9.73950 -0.000015939 -0.000009819 +9886 9.74047 -0.000015930 -0.000009812 +9887 9.74143 -0.000015920 -0.000009806 +9888 9.74240 -0.000015911 -0.000009799 +9889 9.74336 -0.000015901 -0.000009792 +9890 9.74433 -0.000015892 -0.000009785 +9891 9.74529 -0.000015882 -0.000009778 +9892 9.74626 -0.000015873 -0.000009772 +9893 9.74722 -0.000015863 -0.000009765 +9894 9.74819 -0.000015854 -0.000009758 +9895 9.74915 -0.000015845 -0.000009751 +9896 9.75012 -0.000015835 -0.000009745 +9897 9.75108 -0.000015826 -0.000009738 +9898 9.75205 -0.000015816 -0.000009731 +9899 9.75301 -0.000015807 -0.000009724 +9900 9.75398 -0.000015798 -0.000009718 +9901 9.75494 -0.000015788 -0.000009711 +9902 9.75591 -0.000015779 -0.000009704 +9903 9.75687 -0.000015770 -0.000009697 +9904 9.75784 -0.000015760 -0.000009691 +9905 9.75880 -0.000015751 -0.000009684 +9906 9.75977 -0.000015741 -0.000009677 +9907 9.76073 -0.000015732 -0.000009671 +9908 9.76170 -0.000015723 -0.000009664 +9909 9.76266 -0.000015713 -0.000009657 +9910 9.76363 -0.000015704 -0.000009651 +9911 9.76459 -0.000015695 -0.000009644 +9912 9.76556 -0.000015686 -0.000009637 +9913 9.76652 -0.000015676 -0.000009631 +9914 9.76749 -0.000015667 -0.000009624 +9915 9.76845 -0.000015658 -0.000009617 +9916 9.76942 -0.000015648 -0.000009611 +9917 9.77038 -0.000015639 -0.000009604 +9918 9.77135 -0.000015630 -0.000009597 +9919 9.77231 -0.000015621 -0.000009591 +9920 9.77328 -0.000015611 -0.000009584 +9921 9.77424 -0.000015602 -0.000009577 +9922 9.77521 -0.000015593 -0.000009571 +9923 9.77617 -0.000015584 -0.000009564 +9924 9.77714 -0.000015574 -0.000009558 +9925 9.77810 -0.000015565 -0.000009551 +9926 9.77907 -0.000015556 -0.000009544 +9927 9.78003 -0.000015547 -0.000009538 +9928 9.78100 -0.000015538 -0.000009531 +9929 9.78196 -0.000015528 -0.000009525 +9930 9.78293 -0.000015519 -0.000009518 +9931 9.78389 -0.000015510 -0.000009512 +9932 9.78486 -0.000015501 -0.000009505 +9933 9.78582 -0.000015492 -0.000009498 +9934 9.78679 -0.000015482 -0.000009492 +9935 9.78775 -0.000015473 -0.000009485 +9936 9.78872 -0.000015464 -0.000009479 +9937 9.78968 -0.000015455 -0.000009472 +9938 9.79065 -0.000015446 -0.000009466 +9939 9.79161 -0.000015437 -0.000009459 +9940 9.79258 -0.000015428 -0.000009453 +9941 9.79354 -0.000015419 -0.000009446 +9942 9.79451 -0.000015409 -0.000009440 +9943 9.79547 -0.000015400 -0.000009433 +9944 9.79644 -0.000015391 -0.000009427 +9945 9.79740 -0.000015382 -0.000009420 +9946 9.79837 -0.000015373 -0.000009414 +9947 9.79933 -0.000015364 -0.000009407 +9948 9.80030 -0.000015355 -0.000009401 +9949 9.80127 -0.000015346 -0.000009394 +9950 9.80223 -0.000015337 -0.000009388 +9951 9.80320 -0.000015328 -0.000009381 +9952 9.80416 -0.000015319 -0.000009375 +9953 9.80513 -0.000015310 -0.000009368 +9954 9.80609 -0.000015301 -0.000009362 +9955 9.80706 -0.000015292 -0.000009355 +9956 9.80802 -0.000015282 -0.000009349 +9957 9.80899 -0.000015273 -0.000009343 +9958 9.80995 -0.000015264 -0.000009336 +9959 9.81092 -0.000015255 -0.000009330 +9960 9.81188 -0.000015246 -0.000009323 +9961 9.81285 -0.000015237 -0.000009317 +9962 9.81381 -0.000015228 -0.000009310 +9963 9.81478 -0.000015219 -0.000009304 +9964 9.81574 -0.000015211 -0.000009298 +9965 9.81671 -0.000015202 -0.000009291 +9966 9.81767 -0.000015193 -0.000009285 +9967 9.81864 -0.000015184 -0.000009278 +9968 9.81960 -0.000015175 -0.000009272 +9969 9.82057 -0.000015166 -0.000009266 +9970 9.82153 -0.000015157 -0.000009259 +9971 9.82250 -0.000015148 -0.000009253 +9972 9.82346 -0.000015139 -0.000009247 +9973 9.82443 -0.000015130 -0.000009240 +9974 9.82539 -0.000015121 -0.000009234 +9975 9.82636 -0.000015112 -0.000009228 +9976 9.82732 -0.000015103 -0.000009221 +9977 9.82829 -0.000015094 -0.000009215 +9978 9.82925 -0.000015085 -0.000009209 +9979 9.83022 -0.000015077 -0.000009202 +9980 9.83118 -0.000015068 -0.000009196 +9981 9.83215 -0.000015059 -0.000009190 +9982 9.83311 -0.000015050 -0.000009183 +9983 9.83408 -0.000015041 -0.000009177 +9984 9.83504 -0.000015032 -0.000009171 +9985 9.83601 -0.000015023 -0.000009164 +9986 9.83697 -0.000015015 -0.000009158 +9987 9.83794 -0.000015006 -0.000009152 +9988 9.83890 -0.000014997 -0.000009145 +9989 9.83987 -0.000014988 -0.000009139 +9990 9.84083 -0.000014979 -0.000009133 +9991 9.84180 -0.000014970 -0.000009127 +9992 9.84276 -0.000014962 -0.000009120 +9993 9.84373 -0.000014953 -0.000009114 +9994 9.84469 -0.000014944 -0.000009108 +9995 9.84566 -0.000014935 -0.000009102 +9996 9.84662 -0.000014927 -0.000009095 +9997 9.84759 -0.000014918 -0.000009089 +9998 9.84855 -0.000014909 -0.000009083 +9999 9.84952 -0.000014900 -0.000009077 +10000 9.85048 -0.000014891 -0.000009070 +10001 9.85145 -0.000014883 -0.000009064 +10002 9.85241 -0.000014874 -0.000009058 +10003 9.85338 -0.000014865 -0.000009052 +10004 9.85434 -0.000014856 -0.000009046 +10005 9.85531 -0.000014848 -0.000009039 +10006 9.85627 -0.000014839 -0.000009033 +10007 9.85724 -0.000014830 -0.000009027 +10008 9.85820 -0.000014822 -0.000009021 +10009 9.85917 -0.000014813 -0.000009015 +10010 9.86013 -0.000014804 -0.000009009 +10011 9.86110 -0.000014796 -0.000009002 +10012 9.86206 -0.000014787 -0.000008996 +10013 9.86303 -0.000014778 -0.000008990 +10014 9.86399 -0.000014769 -0.000008984 +10015 9.86496 -0.000014761 -0.000008978 +10016 9.86592 -0.000014752 -0.000008972 +10017 9.86689 -0.000014744 -0.000008965 +10018 9.86785 -0.000014735 -0.000008959 +10019 9.86882 -0.000014726 -0.000008953 +10020 9.86978 -0.000014718 -0.000008947 +10021 9.87075 -0.000014709 -0.000008941 +10022 9.87171 -0.000014700 -0.000008935 +10023 9.87268 -0.000014692 -0.000008929 +10024 9.87364 -0.000014683 -0.000008923 +10025 9.87461 -0.000014674 -0.000008916 +10026 9.87557 -0.000014666 -0.000008910 +10027 9.87654 -0.000014657 -0.000008904 +10028 9.87750 -0.000014649 -0.000008898 +10029 9.87847 -0.000014640 -0.000008892 +10030 9.87943 -0.000014632 -0.000008886 +10031 9.88040 -0.000014623 -0.000008880 +10032 9.88136 -0.000014614 -0.000008874 +10033 9.88233 -0.000014606 -0.000008868 +10034 9.88329 -0.000014597 -0.000008862 +10035 9.88426 -0.000014589 -0.000008856 +10036 9.88522 -0.000014580 -0.000008850 +10037 9.88619 -0.000014572 -0.000008844 +10038 9.88715 -0.000014563 -0.000008838 +10039 9.88812 -0.000014555 -0.000008832 +10040 9.88908 -0.000014546 -0.000008826 +10041 9.89005 -0.000014538 -0.000008820 +10042 9.89101 -0.000014529 -0.000008813 +10043 9.89198 -0.000014521 -0.000008807 +10044 9.89294 -0.000014512 -0.000008801 +10045 9.89391 -0.000014504 -0.000008795 +10046 9.89487 -0.000014495 -0.000008789 +10047 9.89584 -0.000014487 -0.000008783 +10048 9.89680 -0.000014478 -0.000008777 +10049 9.89777 -0.000014470 -0.000008771 +10050 9.89873 -0.000014461 -0.000008765 +10051 9.89970 -0.000014453 -0.000008760 +10052 9.90067 -0.000014444 -0.000008754 +10053 9.90163 -0.000014436 -0.000008748 +10054 9.90260 -0.000014427 -0.000008742 +10055 9.90356 -0.000014419 -0.000008736 +10056 9.90453 -0.000014411 -0.000008730 +10057 9.90549 -0.000014402 -0.000008724 +10058 9.90646 -0.000014394 -0.000008718 +10059 9.90742 -0.000014385 -0.000008712 +10060 9.90839 -0.000014377 -0.000008706 +10061 9.90935 -0.000014368 -0.000008700 +10062 9.91032 -0.000014360 -0.000008694 +10063 9.91128 -0.000014352 -0.000008688 +10064 9.91225 -0.000014343 -0.000008682 +10065 9.91321 -0.000014335 -0.000008676 +10066 9.91418 -0.000014327 -0.000008670 +10067 9.91514 -0.000014318 -0.000008664 +10068 9.91611 -0.000014310 -0.000008659 +10069 9.91707 -0.000014302 -0.000008653 +10070 9.91804 -0.000014293 -0.000008647 +10071 9.91900 -0.000014285 -0.000008641 +10072 9.91997 -0.000014276 -0.000008635 +10073 9.92093 -0.000014268 -0.000008629 +10074 9.92190 -0.000014260 -0.000008623 +10075 9.92286 -0.000014252 -0.000008617 +10076 9.92383 -0.000014243 -0.000008612 +10077 9.92479 -0.000014235 -0.000008606 +10078 9.92576 -0.000014227 -0.000008600 +10079 9.92672 -0.000014218 -0.000008594 +10080 9.92769 -0.000014210 -0.000008588 +10081 9.92865 -0.000014202 -0.000008582 +10082 9.92962 -0.000014193 -0.000008576 +10083 9.93058 -0.000014185 -0.000008571 +10084 9.93155 -0.000014177 -0.000008565 +10085 9.93251 -0.000014169 -0.000008559 +10086 9.93348 -0.000014160 -0.000008553 +10087 9.93444 -0.000014152 -0.000008547 +10088 9.93541 -0.000014144 -0.000008541 +10089 9.93637 -0.000014136 -0.000008536 +10090 9.93734 -0.000014127 -0.000008530 +10091 9.93830 -0.000014119 -0.000008524 +10092 9.93927 -0.000014111 -0.000008518 +10093 9.94023 -0.000014103 -0.000008513 +10094 9.94120 -0.000014095 -0.000008507 +10095 9.94216 -0.000014086 -0.000008501 +10096 9.94313 -0.000014078 -0.000008495 +10097 9.94409 -0.000014070 -0.000008489 +10098 9.94506 -0.000014062 -0.000008484 +10099 9.94602 -0.000014054 -0.000008478 +10100 9.94699 -0.000014045 -0.000008472 +10101 9.94795 -0.000014037 -0.000008466 +10102 9.94892 -0.000014029 -0.000008461 +10103 9.94988 -0.000014021 -0.000008455 +10104 9.95085 -0.000014013 -0.000008449 +10105 9.95181 -0.000014005 -0.000008443 +10106 9.95278 -0.000013996 -0.000008438 +10107 9.95374 -0.000013988 -0.000008432 +10108 9.95471 -0.000013980 -0.000008426 +10109 9.95567 -0.000013972 -0.000008421 +10110 9.95664 -0.000013964 -0.000008415 +10111 9.95760 -0.000013956 -0.000008409 +10112 9.95857 -0.000013948 -0.000008403 +10113 9.95953 -0.000013940 -0.000008398 +10114 9.96050 -0.000013931 -0.000008392 +10115 9.96146 -0.000013923 -0.000008386 +10116 9.96243 -0.000013915 -0.000008381 +10117 9.96339 -0.000013907 -0.000008375 +10118 9.96436 -0.000013899 -0.000008369 +10119 9.96532 -0.000013891 -0.000008364 +10120 9.96629 -0.000013883 -0.000008358 +10121 9.96725 -0.000013875 -0.000008352 +10122 9.96822 -0.000013867 -0.000008347 +10123 9.96918 -0.000013859 -0.000008341 +10124 9.97015 -0.000013851 -0.000008335 +10125 9.97111 -0.000013843 -0.000008330 +10126 9.97208 -0.000013835 -0.000008324 +10127 9.97304 -0.000013827 -0.000008318 +10128 9.97401 -0.000013819 -0.000008313 +10129 9.97497 -0.000013811 -0.000008307 +10130 9.97594 -0.000013803 -0.000008302 +10131 9.97690 -0.000013795 -0.000008296 +10132 9.97787 -0.000013787 -0.000008290 +10133 9.97883 -0.000013779 -0.000008285 +10134 9.97980 -0.000013771 -0.000008279 +10135 9.98076 -0.000013763 -0.000008273 +10136 9.98173 -0.000013755 -0.000008268 +10137 9.98269 -0.000013747 -0.000008262 +10138 9.98366 -0.000013739 -0.000008257 +10139 9.98462 -0.000013731 -0.000008251 +10140 9.98559 -0.000013723 -0.000008246 +10141 9.98655 -0.000013715 -0.000008240 +10142 9.98752 -0.000013707 -0.000008234 +10143 9.98848 -0.000013699 -0.000008229 +10144 9.98945 -0.000013691 -0.000008223 +10145 9.99041 -0.000013683 -0.000008218 +10146 9.99138 -0.000013675 -0.000008212 +10147 9.99234 -0.000013667 -0.000008207 +10148 9.99331 -0.000013659 -0.000008201 +10149 9.99427 -0.000013651 -0.000008195 +10150 9.99524 -0.000013643 -0.000008190 +10151 9.99620 -0.000013636 -0.000008184 +10152 9.99717 -0.000013628 -0.000008179 +10153 9.99813 -0.000013620 -0.000008173 +10154 9.99910 -0.000013612 -0.000008168 +10155 10.00007 -0.000013604 -0.000008162 +10156 10.00103 -0.000013596 -0.000008157 +10157 10.00200 -0.000013588 -0.000008151 +10158 10.00296 -0.000013580 -0.000008146 +10159 10.00393 -0.000013572 -0.000008140 +10160 10.00489 -0.000013565 -0.000008135 +10161 10.00586 -0.000013557 -0.000008129 +10162 10.00682 -0.000013549 -0.000008124 +10163 10.00779 -0.000013541 -0.000008118 +10164 10.00875 -0.000013533 -0.000008113 +10165 10.00972 -0.000013525 -0.000008107 +10166 10.01068 -0.000013518 -0.000008102 +10167 10.01165 -0.000013510 -0.000008096 +10168 10.01261 -0.000013502 -0.000008091 +10169 10.01358 -0.000013494 -0.000008086 +10170 10.01454 -0.000013486 -0.000008080 +10171 10.01551 -0.000013479 -0.000008075 +10172 10.01647 -0.000013471 -0.000008069 +10173 10.01744 -0.000013463 -0.000008064 +10174 10.01840 -0.000013455 -0.000008058 +10175 10.01937 -0.000013447 -0.000008053 +10176 10.02033 -0.000013440 -0.000008047 +10177 10.02130 -0.000013432 -0.000008042 +10178 10.02226 -0.000013424 -0.000008037 +10179 10.02323 -0.000013416 -0.000008031 +10180 10.02419 -0.000013409 -0.000008026 +10181 10.02516 -0.000013401 -0.000008020 +10182 10.02612 -0.000013393 -0.000008015 +10183 10.02709 -0.000013385 -0.000008010 +10184 10.02805 -0.000013378 -0.000008004 +10185 10.02902 -0.000013370 -0.000007999 +10186 10.02998 -0.000013362 -0.000007993 +10187 10.03095 -0.000013355 -0.000007988 +10188 10.03191 -0.000013347 -0.000007983 +10189 10.03288 -0.000013339 -0.000007977 +10190 10.03384 -0.000013331 -0.000007972 +10191 10.03481 -0.000013324 -0.000007967 +10192 10.03577 -0.000013316 -0.000007961 +10193 10.03674 -0.000013308 -0.000007956 +10194 10.03770 -0.000013301 -0.000007950 +10195 10.03867 -0.000013293 -0.000007945 +10196 10.03963 -0.000013285 -0.000007940 +10197 10.04060 -0.000013278 -0.000007934 +10198 10.04156 -0.000013270 -0.000007929 +10199 10.04253 -0.000013262 -0.000007924 +10200 10.04349 -0.000013255 -0.000007918 +10201 10.04446 -0.000013247 -0.000007913 +10202 10.04542 -0.000013240 -0.000007908 +10203 10.04639 -0.000013232 -0.000007902 +10204 10.04735 -0.000013224 -0.000007897 +10205 10.04832 -0.000013217 -0.000007892 +10206 10.04928 -0.000013209 -0.000007887 +10207 10.05025 -0.000013201 -0.000007881 +10208 10.05121 -0.000013194 -0.000007876 +10209 10.05218 -0.000013186 -0.000007871 +10210 10.05314 -0.000013179 -0.000007865 +10211 10.05411 -0.000013171 -0.000007860 +10212 10.05507 -0.000013163 -0.000007855 +10213 10.05604 -0.000013156 -0.000007850 +10214 10.05700 -0.000013148 -0.000007844 +10215 10.05797 -0.000013141 -0.000007839 +10216 10.05893 -0.000013133 -0.000007834 +10217 10.05990 -0.000013126 -0.000007828 +10218 10.06086 -0.000013118 -0.000007823 +10219 10.06183 -0.000013111 -0.000007818 +10220 10.06279 -0.000013103 -0.000007813 +10221 10.06376 -0.000013095 -0.000007808 +10222 10.06472 -0.000013088 -0.000007802 +10223 10.06569 -0.000013080 -0.000007797 +10224 10.06665 -0.000013073 -0.000007792 +10225 10.06762 -0.000013065 -0.000007787 +10226 10.06858 -0.000013058 -0.000007781 +10227 10.06955 -0.000013050 -0.000007776 +10228 10.07051 -0.000013043 -0.000007771 +10229 10.07148 -0.000013035 -0.000007766 +10230 10.07244 -0.000013028 -0.000007760 +10231 10.07341 -0.000013020 -0.000007755 +10232 10.07437 -0.000013013 -0.000007750 +10233 10.07534 -0.000013005 -0.000007745 +10234 10.07630 -0.000012998 -0.000007740 +10235 10.07727 -0.000012990 -0.000007735 +10236 10.07823 -0.000012983 -0.000007729 +10237 10.07920 -0.000012976 -0.000007724 +10238 10.08016 -0.000012968 -0.000007719 +10239 10.08113 -0.000012961 -0.000007714 +10240 10.08209 -0.000012953 -0.000007709 +10241 10.08306 -0.000012946 -0.000007703 +10242 10.08402 -0.000012938 -0.000007698 +10243 10.08499 -0.000012931 -0.000007693 +10244 10.08595 -0.000012924 -0.000007688 +10245 10.08692 -0.000012916 -0.000007683 +10246 10.08788 -0.000012909 -0.000007678 +10247 10.08885 -0.000012901 -0.000007673 +10248 10.08981 -0.000012894 -0.000007667 +10249 10.09078 -0.000012886 -0.000007662 +10250 10.09174 -0.000012879 -0.000007657 +10251 10.09271 -0.000012872 -0.000007652 +10252 10.09367 -0.000012864 -0.000007647 +10253 10.09464 -0.000012857 -0.000007642 +10254 10.09560 -0.000012850 -0.000007637 +10255 10.09657 -0.000012842 -0.000007632 +10256 10.09753 -0.000012835 -0.000007627 +10257 10.09850 -0.000012827 -0.000007621 +10258 10.09946 -0.000012820 -0.000007616 +10259 10.10043 -0.000012813 -0.000007611 +10260 10.10140 -0.000012805 -0.000007606 +10261 10.10236 -0.000012798 -0.000007601 +10262 10.10333 -0.000012791 -0.000007596 +10263 10.10429 -0.000012783 -0.000007591 +10264 10.10526 -0.000012776 -0.000007586 +10265 10.10622 -0.000012769 -0.000007581 +10266 10.10719 -0.000012761 -0.000007576 +10267 10.10815 -0.000012754 -0.000007571 +10268 10.10912 -0.000012747 -0.000007566 +10269 10.11008 -0.000012740 -0.000007561 +10270 10.11105 -0.000012732 -0.000007555 +10271 10.11201 -0.000012725 -0.000007550 +10272 10.11298 -0.000012718 -0.000007545 +10273 10.11394 -0.000012710 -0.000007540 +10274 10.11491 -0.000012703 -0.000007535 +10275 10.11587 -0.000012696 -0.000007530 +10276 10.11684 -0.000012689 -0.000007525 +10277 10.11780 -0.000012681 -0.000007520 +10278 10.11877 -0.000012674 -0.000007515 +10279 10.11973 -0.000012667 -0.000007510 +10280 10.12070 -0.000012660 -0.000007505 +10281 10.12166 -0.000012652 -0.000007500 +10282 10.12263 -0.000012645 -0.000007495 +10283 10.12359 -0.000012638 -0.000007490 +10284 10.12456 -0.000012631 -0.000007485 +10285 10.12552 -0.000012623 -0.000007480 +10286 10.12649 -0.000012616 -0.000007475 +10287 10.12745 -0.000012609 -0.000007470 +10288 10.12842 -0.000012602 -0.000007465 +10289 10.12938 -0.000012595 -0.000007460 +10290 10.13035 -0.000012587 -0.000007455 +10291 10.13131 -0.000012580 -0.000007450 +10292 10.13228 -0.000012573 -0.000007445 +10293 10.13324 -0.000012566 -0.000007440 +10294 10.13421 -0.000012559 -0.000007435 +10295 10.13517 -0.000012552 -0.000007430 +10296 10.13614 -0.000012544 -0.000007426 +10297 10.13710 -0.000012537 -0.000007421 +10298 10.13807 -0.000012530 -0.000007416 +10299 10.13903 -0.000012523 -0.000007411 +10300 10.14000 -0.000012516 -0.000007406 +10301 10.14096 -0.000012509 -0.000007401 +10302 10.14193 -0.000012501 -0.000007396 +10303 10.14289 -0.000012494 -0.000007391 +10304 10.14386 -0.000012487 -0.000007386 +10305 10.14482 -0.000012480 -0.000007381 +10306 10.14579 -0.000012473 -0.000007376 +10307 10.14675 -0.000012466 -0.000007371 +10308 10.14772 -0.000012459 -0.000007366 +10309 10.14868 -0.000012452 -0.000007361 +10310 10.14965 -0.000012444 -0.000007357 +10311 10.15061 -0.000012437 -0.000007352 +10312 10.15158 -0.000012430 -0.000007347 +10313 10.15254 -0.000012423 -0.000007342 +10314 10.15351 -0.000012416 -0.000007337 +10315 10.15447 -0.000012409 -0.000007332 +10316 10.15544 -0.000012402 -0.000007327 +10317 10.15640 -0.000012395 -0.000007322 +10318 10.15737 -0.000012388 -0.000007318 +10319 10.15833 -0.000012381 -0.000007313 +10320 10.15930 -0.000012374 -0.000007308 +10321 10.16026 -0.000012367 -0.000007303 +10322 10.16123 -0.000012360 -0.000007298 +10323 10.16219 -0.000012353 -0.000007293 +10324 10.16316 -0.000012346 -0.000007288 +10325 10.16412 -0.000012339 -0.000007284 +10326 10.16509 -0.000012331 -0.000007279 +10327 10.16605 -0.000012324 -0.000007274 +10328 10.16702 -0.000012317 -0.000007269 +10329 10.16798 -0.000012310 -0.000007264 +10330 10.16895 -0.000012303 -0.000007259 +10331 10.16991 -0.000012296 -0.000007255 +10332 10.17088 -0.000012289 -0.000007250 +10333 10.17184 -0.000012282 -0.000007245 +10334 10.17281 -0.000012275 -0.000007240 +10335 10.17377 -0.000012268 -0.000007235 +10336 10.17474 -0.000012261 -0.000007231 +10337 10.17570 -0.000012254 -0.000007226 +10338 10.17667 -0.000012248 -0.000007221 +10339 10.17763 -0.000012241 -0.000007216 +10340 10.17860 -0.000012234 -0.000007211 +10341 10.17956 -0.000012227 -0.000007207 +10342 10.18053 -0.000012220 -0.000007202 +10343 10.18149 -0.000012213 -0.000007197 +10344 10.18246 -0.000012206 -0.000007192 +10345 10.18342 -0.000012199 -0.000007187 +10346 10.18439 -0.000012192 -0.000007183 +10347 10.18535 -0.000012185 -0.000007178 +10348 10.18632 -0.000012178 -0.000007173 +10349 10.18728 -0.000012171 -0.000007168 +10350 10.18825 -0.000012164 -0.000007164 +10351 10.18921 -0.000012157 -0.000007159 +10352 10.19018 -0.000012150 -0.000007154 +10353 10.19114 -0.000012144 -0.000007149 +10354 10.19211 -0.000012137 -0.000007145 +10355 10.19307 -0.000012130 -0.000007140 +10356 10.19404 -0.000012123 -0.000007135 +10357 10.19500 -0.000012116 -0.000007131 +10358 10.19597 -0.000012109 -0.000007126 +10359 10.19693 -0.000012102 -0.000007121 +10360 10.19790 -0.000012095 -0.000007116 +10361 10.19886 -0.000012088 -0.000007112 +10362 10.19983 -0.000012082 -0.000007107 +10363 10.20080 -0.000012075 -0.000007102 +10364 10.20176 -0.000012068 -0.000007098 +10365 10.20273 -0.000012061 -0.000007093 +10366 10.20369 -0.000012054 -0.000007088 +10367 10.20466 -0.000012047 -0.000007083 +10368 10.20562 -0.000012041 -0.000007079 +10369 10.20659 -0.000012034 -0.000007074 +10370 10.20755 -0.000012027 -0.000007069 +10371 10.20852 -0.000012020 -0.000007065 +10372 10.20948 -0.000012013 -0.000007060 +10373 10.21045 -0.000012006 -0.000007055 +10374 10.21141 -0.000012000 -0.000007051 +10375 10.21238 -0.000011993 -0.000007046 +10376 10.21334 -0.000011986 -0.000007041 +10377 10.21431 -0.000011979 -0.000007037 +10378 10.21527 -0.000011972 -0.000007032 +10379 10.21624 -0.000011966 -0.000007027 +10380 10.21720 -0.000011959 -0.000007023 +10381 10.21817 -0.000011952 -0.000007018 +10382 10.21913 -0.000011945 -0.000007014 +10383 10.22010 -0.000011939 -0.000007009 +10384 10.22106 -0.000011932 -0.000007004 +10385 10.22203 -0.000011925 -0.000007000 +10386 10.22299 -0.000011918 -0.000006995 +10387 10.22396 -0.000011912 -0.000006990 +10388 10.22492 -0.000011905 -0.000006986 +10389 10.22589 -0.000011898 -0.000006981 +10390 10.22685 -0.000011891 -0.000006977 +10391 10.22782 -0.000011885 -0.000006972 +10392 10.22878 -0.000011878 -0.000006967 +10393 10.22975 -0.000011871 -0.000006963 +10394 10.23071 -0.000011864 -0.000006958 +10395 10.23168 -0.000011858 -0.000006954 +10396 10.23264 -0.000011851 -0.000006949 +10397 10.23361 -0.000011844 -0.000006944 +10398 10.23457 -0.000011838 -0.000006940 +10399 10.23554 -0.000011831 -0.000006935 +10400 10.23650 -0.000011824 -0.000006931 +10401 10.23747 -0.000011818 -0.000006926 +10402 10.23843 -0.000011811 -0.000006921 +10403 10.23940 -0.000011804 -0.000006917 +10404 10.24036 -0.000011798 -0.000006912 +10405 10.24133 -0.000011791 -0.000006908 +10406 10.24229 -0.000011784 -0.000006903 +10407 10.24326 -0.000011778 -0.000006899 +10408 10.24422 -0.000011771 -0.000006894 +10409 10.24519 -0.000011764 -0.000006890 +10410 10.24615 -0.000011758 -0.000006885 +10411 10.24712 -0.000011751 -0.000006881 +10412 10.24808 -0.000011744 -0.000006876 +10413 10.24905 -0.000011738 -0.000006871 +10414 10.25001 -0.000011731 -0.000006867 +10415 10.25098 -0.000011724 -0.000006862 +10416 10.25194 -0.000011718 -0.000006858 +10417 10.25291 -0.000011711 -0.000006853 +10418 10.25387 -0.000011705 -0.000006849 +10419 10.25484 -0.000011698 -0.000006844 +10420 10.25580 -0.000011691 -0.000006840 +10421 10.25677 -0.000011685 -0.000006835 +10422 10.25773 -0.000011678 -0.000006831 +10423 10.25870 -0.000011672 -0.000006826 +10424 10.25966 -0.000011665 -0.000006822 +10425 10.26063 -0.000011658 -0.000006817 +10426 10.26159 -0.000011652 -0.000006813 +10427 10.26256 -0.000011645 -0.000006808 +10428 10.26352 -0.000011639 -0.000006804 +10429 10.26449 -0.000011632 -0.000006799 +10430 10.26545 -0.000011626 -0.000006795 +10431 10.26642 -0.000011619 -0.000006790 +10432 10.26738 -0.000011612 -0.000006786 +10433 10.26835 -0.000011606 -0.000006782 +10434 10.26931 -0.000011599 -0.000006777 +10435 10.27028 -0.000011593 -0.000006773 +10436 10.27124 -0.000011586 -0.000006768 +10437 10.27221 -0.000011580 -0.000006764 +10438 10.27317 -0.000011573 -0.000006759 +10439 10.27414 -0.000011567 -0.000006755 +10440 10.27510 -0.000011560 -0.000006750 +10441 10.27607 -0.000011554 -0.000006746 +10442 10.27703 -0.000011547 -0.000006742 +10443 10.27800 -0.000011541 -0.000006737 +10444 10.27896 -0.000011534 -0.000006733 +10445 10.27993 -0.000011528 -0.000006728 +10446 10.28089 -0.000011521 -0.000006724 +10447 10.28186 -0.000011515 -0.000006719 +10448 10.28282 -0.000011508 -0.000006715 +10449 10.28379 -0.000011502 -0.000006711 +10450 10.28475 -0.000011495 -0.000006706 +10451 10.28572 -0.000011489 -0.000006702 +10452 10.28668 -0.000011482 -0.000006697 +10453 10.28765 -0.000011476 -0.000006693 +10454 10.28861 -0.000011469 -0.000006689 +10455 10.28958 -0.000011463 -0.000006684 +10456 10.29054 -0.000011457 -0.000006680 +10457 10.29151 -0.000011450 -0.000006675 +10458 10.29247 -0.000011444 -0.000006671 +10459 10.29344 -0.000011437 -0.000006667 +10460 10.29440 -0.000011431 -0.000006662 +10461 10.29537 -0.000011424 -0.000006658 +10462 10.29633 -0.000011418 -0.000006654 +10463 10.29730 -0.000011411 -0.000006649 +10464 10.29826 -0.000011405 -0.000006645 +10465 10.29923 -0.000011399 -0.000006640 +10466 10.30020 -0.000011392 -0.000006636 +10467 10.30116 -0.000011386 -0.000006632 +10468 10.30213 -0.000011379 -0.000006627 +10469 10.30309 -0.000011373 -0.000006623 +10470 10.30406 -0.000011367 -0.000006619 +10471 10.30502 -0.000011360 -0.000006614 +10472 10.30599 -0.000011354 -0.000006610 +10473 10.30695 -0.000011348 -0.000006606 +10474 10.30792 -0.000011341 -0.000006601 +10475 10.30888 -0.000011335 -0.000006597 +10476 10.30985 -0.000011328 -0.000006593 +10477 10.31081 -0.000011322 -0.000006588 +10478 10.31178 -0.000011316 -0.000006584 +10479 10.31274 -0.000011309 -0.000006580 +10480 10.31371 -0.000011303 -0.000006576 +10481 10.31467 -0.000011297 -0.000006571 +10482 10.31564 -0.000011290 -0.000006567 +10483 10.31660 -0.000011284 -0.000006563 +10484 10.31757 -0.000011278 -0.000006558 +10485 10.31853 -0.000011271 -0.000006554 +10486 10.31950 -0.000011265 -0.000006550 +10487 10.32046 -0.000011259 -0.000006545 +10488 10.32143 -0.000011252 -0.000006541 +10489 10.32239 -0.000011246 -0.000006537 +10490 10.32336 -0.000011240 -0.000006533 +10491 10.32432 -0.000011233 -0.000006528 +10492 10.32529 -0.000011227 -0.000006524 +10493 10.32625 -0.000011221 -0.000006520 +10494 10.32722 -0.000011215 -0.000006516 +10495 10.32818 -0.000011208 -0.000006511 +10496 10.32915 -0.000011202 -0.000006507 +10497 10.33011 -0.000011196 -0.000006503 +10498 10.33108 -0.000011189 -0.000006499 +10499 10.33204 -0.000011183 -0.000006494 +10500 10.33301 -0.000011177 -0.000006490 +10501 10.33397 -0.000011171 -0.000006486 +10502 10.33494 -0.000011164 -0.000006482 +10503 10.33590 -0.000011158 -0.000006477 +10504 10.33687 -0.000011152 -0.000006473 +10505 10.33783 -0.000011146 -0.000006469 +10506 10.33880 -0.000011139 -0.000006465 +10507 10.33976 -0.000011133 -0.000006460 +10508 10.34073 -0.000011127 -0.000006456 +10509 10.34169 -0.000011121 -0.000006452 +10510 10.34266 -0.000011114 -0.000006448 +10511 10.34362 -0.000011108 -0.000006444 +10512 10.34459 -0.000011102 -0.000006439 +10513 10.34555 -0.000011096 -0.000006435 +10514 10.34652 -0.000011090 -0.000006431 +10515 10.34748 -0.000011083 -0.000006427 +10516 10.34845 -0.000011077 -0.000006423 +10517 10.34941 -0.000011071 -0.000006418 +10518 10.35038 -0.000011065 -0.000006414 +10519 10.35134 -0.000011059 -0.000006410 +10520 10.35231 -0.000011052 -0.000006406 +10521 10.35327 -0.000011046 -0.000006402 +10522 10.35424 -0.000011040 -0.000006397 +10523 10.35520 -0.000011034 -0.000006393 +10524 10.35617 -0.000011028 -0.000006389 +10525 10.35713 -0.000011022 -0.000006385 +10526 10.35810 -0.000011015 -0.000006381 +10527 10.35906 -0.000011009 -0.000006377 +10528 10.36003 -0.000011003 -0.000006372 +10529 10.36099 -0.000010997 -0.000006368 +10530 10.36196 -0.000010991 -0.000006364 +10531 10.36292 -0.000010985 -0.000006360 +10532 10.36389 -0.000010979 -0.000006356 +10533 10.36485 -0.000010972 -0.000006352 +10534 10.36582 -0.000010966 -0.000006348 +10535 10.36678 -0.000010960 -0.000006343 +10536 10.36775 -0.000010954 -0.000006339 +10537 10.36871 -0.000010948 -0.000006335 +10538 10.36968 -0.000010942 -0.000006331 +10539 10.37064 -0.000010936 -0.000006327 +10540 10.37161 -0.000010930 -0.000006323 +10541 10.37257 -0.000010924 -0.000006319 +10542 10.37354 -0.000010917 -0.000006315 +10543 10.37450 -0.000010911 -0.000006310 +10544 10.37547 -0.000010905 -0.000006306 +10545 10.37643 -0.000010899 -0.000006302 +10546 10.37740 -0.000010893 -0.000006298 +10547 10.37836 -0.000010887 -0.000006294 +10548 10.37933 -0.000010881 -0.000006290 +10549 10.38029 -0.000010875 -0.000006286 +10550 10.38126 -0.000010869 -0.000006282 +10551 10.38222 -0.000010863 -0.000006278 +10552 10.38319 -0.000010857 -0.000006274 +10553 10.38415 -0.000010851 -0.000006270 +10554 10.38512 -0.000010845 -0.000006265 +10555 10.38608 -0.000010839 -0.000006261 +10556 10.38705 -0.000010833 -0.000006257 +10557 10.38801 -0.000010826 -0.000006253 +10558 10.38898 -0.000010820 -0.000006249 +10559 10.38994 -0.000010814 -0.000006245 +10560 10.39091 -0.000010808 -0.000006241 +10561 10.39187 -0.000010802 -0.000006237 +10562 10.39284 -0.000010796 -0.000006233 +10563 10.39380 -0.000010790 -0.000006229 +10564 10.39477 -0.000010784 -0.000006225 +10565 10.39573 -0.000010778 -0.000006221 +10566 10.39670 -0.000010772 -0.000006217 +10567 10.39766 -0.000010766 -0.000006213 +10568 10.39863 -0.000010760 -0.000006209 +10569 10.39959 -0.000010754 -0.000006205 +10570 10.40056 -0.000010748 -0.000006201 +10571 10.40153 -0.000010742 -0.000006197 +10572 10.40249 -0.000010736 -0.000006193 +10573 10.40346 -0.000010730 -0.000006189 +10574 10.40442 -0.000010724 -0.000006185 +10575 10.40539 -0.000010718 -0.000006181 +10576 10.40635 -0.000010713 -0.000006177 +10577 10.40732 -0.000010707 -0.000006173 +10578 10.40828 -0.000010701 -0.000006169 +10579 10.40925 -0.000010695 -0.000006165 +10580 10.41021 -0.000010689 -0.000006161 +10581 10.41118 -0.000010683 -0.000006157 +10582 10.41214 -0.000010677 -0.000006153 +10583 10.41311 -0.000010671 -0.000006149 +10584 10.41407 -0.000010665 -0.000006145 +10585 10.41504 -0.000010659 -0.000006141 +10586 10.41600 -0.000010653 -0.000006137 +10587 10.41697 -0.000010647 -0.000006133 +10588 10.41793 -0.000010641 -0.000006129 +10589 10.41890 -0.000010635 -0.000006125 +10590 10.41986 -0.000010629 -0.000006121 +10591 10.42083 -0.000010624 -0.000006117 +10592 10.42179 -0.000010618 -0.000006113 +10593 10.42276 -0.000010612 -0.000006109 +10594 10.42372 -0.000010606 -0.000006105 +10595 10.42469 -0.000010600 -0.000006101 +10596 10.42565 -0.000010594 -0.000006097 +10597 10.42662 -0.000010588 -0.000006093 +10598 10.42758 -0.000010582 -0.000006089 +10599 10.42855 -0.000010576 -0.000006085 +10600 10.42951 -0.000010571 -0.000006081 +10601 10.43048 -0.000010565 -0.000006077 +10602 10.43144 -0.000010559 -0.000006073 +10603 10.43241 -0.000010553 -0.000006069 +10604 10.43337 -0.000010547 -0.000006065 +10605 10.43434 -0.000010541 -0.000006061 +10606 10.43530 -0.000010535 -0.000006058 +10607 10.43627 -0.000010530 -0.000006054 +10608 10.43723 -0.000010524 -0.000006050 +10609 10.43820 -0.000010518 -0.000006046 +10610 10.43916 -0.000010512 -0.000006042 +10611 10.44013 -0.000010506 -0.000006038 +10612 10.44109 -0.000010500 -0.000006034 +10613 10.44206 -0.000010495 -0.000006030 +10614 10.44302 -0.000010489 -0.000006026 +10615 10.44399 -0.000010483 -0.000006022 +10616 10.44495 -0.000010477 -0.000006019 +10617 10.44592 -0.000010471 -0.000006015 +10618 10.44688 -0.000010466 -0.000006011 +10619 10.44785 -0.000010460 -0.000006007 +10620 10.44881 -0.000010454 -0.000006003 +10621 10.44978 -0.000010448 -0.000005999 +10622 10.45074 -0.000010442 -0.000005995 +10623 10.45171 -0.000010437 -0.000005991 +10624 10.45267 -0.000010431 -0.000005987 +10625 10.45364 -0.000010425 -0.000005984 +10626 10.45460 -0.000010419 -0.000005980 +10627 10.45557 -0.000010413 -0.000005976 +10628 10.45653 -0.000010408 -0.000005972 +10629 10.45750 -0.000010402 -0.000005968 +10630 10.45846 -0.000010396 -0.000005964 +10631 10.45943 -0.000010390 -0.000005960 +10632 10.46039 -0.000010385 -0.000005957 +10633 10.46136 -0.000010379 -0.000005953 +10634 10.46232 -0.000010373 -0.000005949 +10635 10.46329 -0.000010367 -0.000005945 +10636 10.46425 -0.000010362 -0.000005941 +10637 10.46522 -0.000010356 -0.000005937 +10638 10.46618 -0.000010350 -0.000005934 +10639 10.46715 -0.000010345 -0.000005930 +10640 10.46811 -0.000010339 -0.000005926 +10641 10.46908 -0.000010333 -0.000005922 +10642 10.47004 -0.000010327 -0.000005918 +10643 10.47101 -0.000010322 -0.000005914 +10644 10.47197 -0.000010316 -0.000005911 +10645 10.47294 -0.000010310 -0.000005907 +10646 10.47390 -0.000010305 -0.000005903 +10647 10.47487 -0.000010299 -0.000005899 +10648 10.47583 -0.000010293 -0.000005895 +10649 10.47680 -0.000010288 -0.000005892 +10650 10.47776 -0.000010282 -0.000005888 +10651 10.47873 -0.000010276 -0.000005884 +10652 10.47969 -0.000010270 -0.000005880 +10653 10.48066 -0.000010265 -0.000005876 +10654 10.48162 -0.000010259 -0.000005873 +10655 10.48259 -0.000010253 -0.000005869 +10656 10.48355 -0.000010248 -0.000005865 +10657 10.48452 -0.000010242 -0.000005861 +10658 10.48548 -0.000010236 -0.000005858 +10659 10.48645 -0.000010231 -0.000005854 +10660 10.48741 -0.000010225 -0.000005850 +10661 10.48838 -0.000010220 -0.000005846 +10662 10.48934 -0.000010214 -0.000005842 +10663 10.49031 -0.000010208 -0.000005839 +10664 10.49127 -0.000010203 -0.000005835 +10665 10.49224 -0.000010197 -0.000005831 +10666 10.49320 -0.000010191 -0.000005827 +10667 10.49417 -0.000010186 -0.000005824 +10668 10.49513 -0.000010180 -0.000005820 +10669 10.49610 -0.000010175 -0.000005816 +10670 10.49706 -0.000010169 -0.000005812 +10671 10.49803 -0.000010163 -0.000005809 +10672 10.49899 -0.000010158 -0.000005805 +10673 10.49996 -0.000010152 -0.000005801 +10674 10.50093 -0.000010147 -0.000005797 +10675 10.50189 -0.000010141 -0.000005794 +10676 10.50286 -0.000010135 -0.000005790 +10677 10.50382 -0.000010130 -0.000005786 +10678 10.50479 -0.000010124 -0.000005783 +10679 10.50575 -0.000010119 -0.000005779 +10680 10.50672 -0.000010113 -0.000005775 +10681 10.50768 -0.000010107 -0.000005771 +10682 10.50865 -0.000010102 -0.000005768 +10683 10.50961 -0.000010096 -0.000005764 +10684 10.51058 -0.000010091 -0.000005760 +10685 10.51154 -0.000010085 -0.000005757 +10686 10.51251 -0.000010080 -0.000005753 +10687 10.51347 -0.000010074 -0.000005749 +10688 10.51444 -0.000010069 -0.000005746 +10689 10.51540 -0.000010063 -0.000005742 +10690 10.51637 -0.000010057 -0.000005738 +10691 10.51733 -0.000010052 -0.000005734 +10692 10.51830 -0.000010046 -0.000005731 +10693 10.51926 -0.000010041 -0.000005727 +10694 10.52023 -0.000010035 -0.000005723 +10695 10.52119 -0.000010030 -0.000005720 +10696 10.52216 -0.000010024 -0.000005716 +10697 10.52312 -0.000010019 -0.000005712 +10698 10.52409 -0.000010013 -0.000005709 +10699 10.52505 -0.000010008 -0.000005705 +10700 10.52602 -0.000010002 -0.000005701 +10701 10.52698 -0.000009997 -0.000005698 +10702 10.52795 -0.000009991 -0.000005694 +10703 10.52891 -0.000009986 -0.000005690 +10704 10.52988 -0.000009980 -0.000005687 +10705 10.53084 -0.000009975 -0.000005683 +10706 10.53181 -0.000009969 -0.000005680 +10707 10.53277 -0.000009964 -0.000005676 +10708 10.53374 -0.000009958 -0.000005672 +10709 10.53470 -0.000009953 -0.000005669 +10710 10.53567 -0.000009947 -0.000005665 +10711 10.53663 -0.000009942 -0.000005661 +10712 10.53760 -0.000009936 -0.000005658 +10713 10.53856 -0.000009931 -0.000005654 +10714 10.53953 -0.000009926 -0.000005650 +10715 10.54049 -0.000009920 -0.000005647 +10716 10.54146 -0.000009915 -0.000005643 +10717 10.54242 -0.000009909 -0.000005640 +10718 10.54339 -0.000009904 -0.000005636 +10719 10.54435 -0.000009898 -0.000005632 +10720 10.54532 -0.000009893 -0.000005629 +10721 10.54628 -0.000009887 -0.000005625 +10722 10.54725 -0.000009882 -0.000005622 +10723 10.54821 -0.000009877 -0.000005618 +10724 10.54918 -0.000009871 -0.000005614 +10725 10.55014 -0.000009866 -0.000005611 +10726 10.55111 -0.000009860 -0.000005607 +10727 10.55207 -0.000009855 -0.000005604 +10728 10.55304 -0.000009850 -0.000005600 +10729 10.55400 -0.000009844 -0.000005596 +10730 10.55497 -0.000009839 -0.000005593 +10731 10.55593 -0.000009833 -0.000005589 +10732 10.55690 -0.000009828 -0.000005586 +10733 10.55786 -0.000009823 -0.000005582 +10734 10.55883 -0.000009817 -0.000005579 +10735 10.55979 -0.000009812 -0.000005575 +10736 10.56076 -0.000009806 -0.000005571 +10737 10.56172 -0.000009801 -0.000005568 +10738 10.56269 -0.000009796 -0.000005564 +10739 10.56365 -0.000009790 -0.000005561 +10740 10.56462 -0.000009785 -0.000005557 +10741 10.56558 -0.000009780 -0.000005554 +10742 10.56655 -0.000009774 -0.000005550 +10743 10.56751 -0.000009769 -0.000005547 +10744 10.56848 -0.000009764 -0.000005543 +10745 10.56944 -0.000009758 -0.000005539 +10746 10.57041 -0.000009753 -0.000005536 +10747 10.57137 -0.000009748 -0.000005532 +10748 10.57234 -0.000009742 -0.000005529 +10749 10.57330 -0.000009737 -0.000005525 +10750 10.57427 -0.000009732 -0.000005522 +10751 10.57523 -0.000009726 -0.000005518 +10752 10.57620 -0.000009721 -0.000005515 +10753 10.57716 -0.000009716 -0.000005511 +10754 10.57813 -0.000009710 -0.000005508 +10755 10.57909 -0.000009705 -0.000005504 +10756 10.58006 -0.000009700 -0.000005501 +10757 10.58102 -0.000009694 -0.000005497 +10758 10.58199 -0.000009689 -0.000005494 +10759 10.58295 -0.000009684 -0.000005490 +10760 10.58392 -0.000009678 -0.000005487 +10761 10.58488 -0.000009673 -0.000005483 +10762 10.58585 -0.000009668 -0.000005480 +10763 10.58681 -0.000009663 -0.000005476 +10764 10.58778 -0.000009657 -0.000005473 +10765 10.58874 -0.000009652 -0.000005469 +10766 10.58971 -0.000009647 -0.000005466 +10767 10.59067 -0.000009641 -0.000005462 +10768 10.59164 -0.000009636 -0.000005459 +10769 10.59260 -0.000009631 -0.000005455 +10770 10.59357 -0.000009626 -0.000005452 +10771 10.59453 -0.000009620 -0.000005448 +10772 10.59550 -0.000009615 -0.000005445 +10773 10.59646 -0.000009610 -0.000005441 +10774 10.59743 -0.000009605 -0.000005438 +10775 10.59839 -0.000009599 -0.000005434 +10776 10.59936 -0.000009594 -0.000005431 +10777 10.60033 -0.000009589 -0.000005427 +10778 10.60129 -0.000009584 -0.000005424 +10779 10.60226 -0.000009578 -0.000005421 +10780 10.60322 -0.000009573 -0.000005417 +10781 10.60419 -0.000009568 -0.000005414 +10782 10.60515 -0.000009563 -0.000005410 +10783 10.60612 -0.000009557 -0.000005407 +10784 10.60708 -0.000009552 -0.000005403 +10785 10.60805 -0.000009547 -0.000005400 +10786 10.60901 -0.000009542 -0.000005396 +10787 10.60998 -0.000009537 -0.000005393 +10788 10.61094 -0.000009531 -0.000005390 +10789 10.61191 -0.000009526 -0.000005386 +10790 10.61287 -0.000009521 -0.000005383 +10791 10.61384 -0.000009516 -0.000005379 +10792 10.61480 -0.000009511 -0.000005376 +10793 10.61577 -0.000009505 -0.000005372 +10794 10.61673 -0.000009500 -0.000005369 +10795 10.61770 -0.000009495 -0.000005366 +10796 10.61866 -0.000009490 -0.000005362 +10797 10.61963 -0.000009485 -0.000005359 +10798 10.62059 -0.000009480 -0.000005355 +10799 10.62156 -0.000009474 -0.000005352 +10800 10.62252 -0.000009469 -0.000005349 +10801 10.62349 -0.000009464 -0.000005345 +10802 10.62445 -0.000009459 -0.000005342 +10803 10.62542 -0.000009454 -0.000005338 +10804 10.62638 -0.000009449 -0.000005335 +10805 10.62735 -0.000009444 -0.000005332 +10806 10.62831 -0.000009438 -0.000005328 +10807 10.62928 -0.000009433 -0.000005325 +10808 10.63024 -0.000009428 -0.000005321 +10809 10.63121 -0.000009423 -0.000005318 +10810 10.63217 -0.000009418 -0.000005315 +10811 10.63314 -0.000009413 -0.000005311 +10812 10.63410 -0.000009408 -0.000005308 +10813 10.63507 -0.000009402 -0.000005305 +10814 10.63603 -0.000009397 -0.000005301 +10815 10.63700 -0.000009392 -0.000005298 +10816 10.63796 -0.000009387 -0.000005294 +10817 10.63893 -0.000009382 -0.000005291 +10818 10.63989 -0.000009377 -0.000005288 +10819 10.64086 -0.000009372 -0.000005284 +10820 10.64182 -0.000009367 -0.000005281 +10821 10.64279 -0.000009362 -0.000005278 +10822 10.64375 -0.000009357 -0.000005274 +10823 10.64472 -0.000009351 -0.000005271 +10824 10.64568 -0.000009346 -0.000005268 +10825 10.64665 -0.000009341 -0.000005264 +10826 10.64761 -0.000009336 -0.000005261 +10827 10.64858 -0.000009331 -0.000005258 +10828 10.64954 -0.000009326 -0.000005254 +10829 10.65051 -0.000009321 -0.000005251 +10830 10.65147 -0.000009316 -0.000005248 +10831 10.65244 -0.000009311 -0.000005244 +10832 10.65340 -0.000009306 -0.000005241 +10833 10.65437 -0.000009301 -0.000005238 +10834 10.65533 -0.000009296 -0.000005234 +10835 10.65630 -0.000009291 -0.000005231 +10836 10.65726 -0.000009286 -0.000005228 +10837 10.65823 -0.000009281 -0.000005224 +10838 10.65919 -0.000009275 -0.000005221 +10839 10.66016 -0.000009270 -0.000005218 +10840 10.66112 -0.000009265 -0.000005215 +10841 10.66209 -0.000009260 -0.000005211 +10842 10.66305 -0.000009255 -0.000005208 +10843 10.66402 -0.000009250 -0.000005205 +10844 10.66498 -0.000009245 -0.000005201 +10845 10.66595 -0.000009240 -0.000005198 +10846 10.66691 -0.000009235 -0.000005195 +10847 10.66788 -0.000009230 -0.000005191 +10848 10.66884 -0.000009225 -0.000005188 +10849 10.66981 -0.000009220 -0.000005185 +10850 10.67077 -0.000009215 -0.000005182 +10851 10.67174 -0.000009210 -0.000005178 +10852 10.67270 -0.000009205 -0.000005175 +10853 10.67367 -0.000009200 -0.000005172 +10854 10.67463 -0.000009195 -0.000005168 +10855 10.67560 -0.000009190 -0.000005165 +10856 10.67656 -0.000009185 -0.000005162 +10857 10.67753 -0.000009180 -0.000005159 +10858 10.67849 -0.000009175 -0.000005155 +10859 10.67946 -0.000009170 -0.000005152 +10860 10.68042 -0.000009165 -0.000005149 +10861 10.68139 -0.000009160 -0.000005146 +10862 10.68235 -0.000009155 -0.000005142 +10863 10.68332 -0.000009151 -0.000005139 +10864 10.68428 -0.000009146 -0.000005136 +10865 10.68525 -0.000009141 -0.000005133 +10866 10.68621 -0.000009136 -0.000005129 +10867 10.68718 -0.000009131 -0.000005126 +10868 10.68814 -0.000009126 -0.000005123 +10869 10.68911 -0.000009121 -0.000005120 +10870 10.69007 -0.000009116 -0.000005116 +10871 10.69104 -0.000009111 -0.000005113 +10872 10.69200 -0.000009106 -0.000005110 +10873 10.69297 -0.000009101 -0.000005107 +10874 10.69393 -0.000009096 -0.000005104 +10875 10.69490 -0.000009091 -0.000005100 +10876 10.69586 -0.000009086 -0.000005097 +10877 10.69683 -0.000009081 -0.000005094 +10878 10.69779 -0.000009076 -0.000005091 +10879 10.69876 -0.000009072 -0.000005087 +10880 10.69972 -0.000009067 -0.000005084 +10881 10.70069 -0.000009062 -0.000005081 +10882 10.70166 -0.000009057 -0.000005078 +10883 10.70262 -0.000009052 -0.000005075 +10884 10.70359 -0.000009047 -0.000005071 +10885 10.70455 -0.000009042 -0.000005068 +10886 10.70552 -0.000009037 -0.000005065 +10887 10.70648 -0.000009032 -0.000005062 +10888 10.70745 -0.000009027 -0.000005059 +10889 10.70841 -0.000009023 -0.000005055 +10890 10.70938 -0.000009018 -0.000005052 +10891 10.71034 -0.000009013 -0.000005049 +10892 10.71131 -0.000009008 -0.000005046 +10893 10.71227 -0.000009003 -0.000005043 +10894 10.71324 -0.000008998 -0.000005040 +10895 10.71420 -0.000008993 -0.000005036 +10896 10.71517 -0.000008989 -0.000005033 +10897 10.71613 -0.000008984 -0.000005030 +10898 10.71710 -0.000008979 -0.000005027 +10899 10.71806 -0.000008974 -0.000005024 +10900 10.71903 -0.000008969 -0.000005020 +10901 10.71999 -0.000008964 -0.000005017 +10902 10.72096 -0.000008959 -0.000005014 +10903 10.72192 -0.000008955 -0.000005011 +10904 10.72289 -0.000008950 -0.000005008 +10905 10.72385 -0.000008945 -0.000005005 +10906 10.72482 -0.000008940 -0.000005002 +10907 10.72578 -0.000008935 -0.000004998 +10908 10.72675 -0.000008930 -0.000004995 +10909 10.72771 -0.000008926 -0.000004992 +10910 10.72868 -0.000008921 -0.000004989 +10911 10.72964 -0.000008916 -0.000004986 +10912 10.73061 -0.000008911 -0.000004983 +10913 10.73157 -0.000008906 -0.000004980 +10914 10.73254 -0.000008902 -0.000004976 +10915 10.73350 -0.000008897 -0.000004973 +10916 10.73447 -0.000008892 -0.000004970 +10917 10.73543 -0.000008887 -0.000004967 +10918 10.73640 -0.000008882 -0.000004964 +10919 10.73736 -0.000008878 -0.000004961 +10920 10.73833 -0.000008873 -0.000004958 +10921 10.73929 -0.000008868 -0.000004955 +10922 10.74026 -0.000008863 -0.000004951 +10923 10.74122 -0.000008858 -0.000004948 +10924 10.74219 -0.000008854 -0.000004945 +10925 10.74315 -0.000008849 -0.000004942 +10926 10.74412 -0.000008844 -0.000004939 +10927 10.74508 -0.000008839 -0.000004936 +10928 10.74605 -0.000008835 -0.000004933 +10929 10.74701 -0.000008830 -0.000004930 +10930 10.74798 -0.000008825 -0.000004927 +10931 10.74894 -0.000008820 -0.000004923 +10932 10.74991 -0.000008816 -0.000004920 +10933 10.75087 -0.000008811 -0.000004917 +10934 10.75184 -0.000008806 -0.000004914 +10935 10.75280 -0.000008801 -0.000004911 +10936 10.75377 -0.000008797 -0.000004908 +10937 10.75473 -0.000008792 -0.000004905 +10938 10.75570 -0.000008787 -0.000004902 +10939 10.75666 -0.000008782 -0.000004899 +10940 10.75763 -0.000008778 -0.000004896 +10941 10.75859 -0.000008773 -0.000004893 +10942 10.75956 -0.000008768 -0.000004890 +10943 10.76052 -0.000008764 -0.000004887 +10944 10.76149 -0.000008759 -0.000004883 +10945 10.76245 -0.000008754 -0.000004880 +10946 10.76342 -0.000008749 -0.000004877 +10947 10.76438 -0.000008745 -0.000004874 +10948 10.76535 -0.000008740 -0.000004871 +10949 10.76631 -0.000008735 -0.000004868 +10950 10.76728 -0.000008731 -0.000004865 +10951 10.76824 -0.000008726 -0.000004862 +10952 10.76921 -0.000008721 -0.000004859 +10953 10.77017 -0.000008717 -0.000004856 +10954 10.77114 -0.000008712 -0.000004853 +10955 10.77210 -0.000008707 -0.000004850 +10956 10.77307 -0.000008703 -0.000004847 +10957 10.77403 -0.000008698 -0.000004844 +10958 10.77500 -0.000008693 -0.000004841 +10959 10.77596 -0.000008689 -0.000004838 +10960 10.77693 -0.000008684 -0.000004835 +10961 10.77789 -0.000008679 -0.000004832 +10962 10.77886 -0.000008675 -0.000004829 +10963 10.77982 -0.000008670 -0.000004826 +10964 10.78079 -0.000008665 -0.000004823 +10965 10.78175 -0.000008661 -0.000004820 +10966 10.78272 -0.000008656 -0.000004817 +10967 10.78368 -0.000008651 -0.000004814 +10968 10.78465 -0.000008647 -0.000004811 +10969 10.78561 -0.000008642 -0.000004808 +10970 10.78658 -0.000008637 -0.000004804 +10971 10.78754 -0.000008633 -0.000004801 +10972 10.78851 -0.000008628 -0.000004798 +10973 10.78947 -0.000008623 -0.000004795 +10974 10.79044 -0.000008619 -0.000004792 +10975 10.79140 -0.000008614 -0.000004789 +10976 10.79237 -0.000008610 -0.000004786 +10977 10.79333 -0.000008605 -0.000004783 +10978 10.79430 -0.000008600 -0.000004780 +10979 10.79526 -0.000008596 -0.000004777 +10980 10.79623 -0.000008591 -0.000004775 +10981 10.79719 -0.000008587 -0.000004772 +10982 10.79816 -0.000008582 -0.000004769 +10983 10.79912 -0.000008577 -0.000004766 +10984 10.80009 -0.000008573 -0.000004763 +10985 10.80106 -0.000008568 -0.000004760 +10986 10.80202 -0.000008564 -0.000004757 +10987 10.80299 -0.000008559 -0.000004754 +10988 10.80395 -0.000008554 -0.000004751 +10989 10.80492 -0.000008550 -0.000004748 +10990 10.80588 -0.000008545 -0.000004745 +10991 10.80685 -0.000008541 -0.000004742 +10992 10.80781 -0.000008536 -0.000004739 +10993 10.80878 -0.000008531 -0.000004736 +10994 10.80974 -0.000008527 -0.000004733 +10995 10.81071 -0.000008522 -0.000004730 +10996 10.81167 -0.000008518 -0.000004727 +10997 10.81264 -0.000008513 -0.000004724 +10998 10.81360 -0.000008509 -0.000004721 +10999 10.81457 -0.000008504 -0.000004718 +11000 10.81553 -0.000008500 -0.000004715 +11001 10.81650 -0.000008495 -0.000004712 +11002 10.81746 -0.000008490 -0.000004709 +11003 10.81843 -0.000008486 -0.000004706 +11004 10.81939 -0.000008481 -0.000004703 +11005 10.82036 -0.000008477 -0.000004700 +11006 10.82132 -0.000008472 -0.000004698 +11007 10.82229 -0.000008468 -0.000004695 +11008 10.82325 -0.000008463 -0.000004692 +11009 10.82422 -0.000008459 -0.000004689 +11010 10.82518 -0.000008454 -0.000004686 +11011 10.82615 -0.000008450 -0.000004683 +11012 10.82711 -0.000008445 -0.000004680 +11013 10.82808 -0.000008441 -0.000004677 +11014 10.82904 -0.000008436 -0.000004674 +11015 10.83001 -0.000008432 -0.000004671 +11016 10.83097 -0.000008427 -0.000004668 +11017 10.83194 -0.000008423 -0.000004665 +11018 10.83290 -0.000008418 -0.000004663 +11019 10.83387 -0.000008414 -0.000004660 +11020 10.83483 -0.000008409 -0.000004657 +11021 10.83580 -0.000008405 -0.000004654 +11022 10.83676 -0.000008400 -0.000004651 +11023 10.83773 -0.000008396 -0.000004648 +11024 10.83869 -0.000008391 -0.000004645 +11025 10.83966 -0.000008387 -0.000004642 +11026 10.84062 -0.000008382 -0.000004639 +11027 10.84159 -0.000008378 -0.000004636 +11028 10.84255 -0.000008373 -0.000004634 +11029 10.84352 -0.000008369 -0.000004631 +11030 10.84448 -0.000008364 -0.000004628 +11031 10.84545 -0.000008360 -0.000004625 +11032 10.84641 -0.000008355 -0.000004622 +11033 10.84738 -0.000008351 -0.000004619 +11034 10.84834 -0.000008346 -0.000004616 +11035 10.84931 -0.000008342 -0.000004613 +11036 10.85027 -0.000008338 -0.000004611 +11037 10.85124 -0.000008333 -0.000004608 +11038 10.85220 -0.000008329 -0.000004605 +11039 10.85317 -0.000008324 -0.000004602 +11040 10.85413 -0.000008320 -0.000004599 +11041 10.85510 -0.000008315 -0.000004596 +11042 10.85606 -0.000008311 -0.000004593 +11043 10.85703 -0.000008306 -0.000004590 +11044 10.85799 -0.000008302 -0.000004588 +11045 10.85896 -0.000008298 -0.000004585 +11046 10.85992 -0.000008293 -0.000004582 +11047 10.86089 -0.000008289 -0.000004579 +11048 10.86185 -0.000008284 -0.000004576 +11049 10.86282 -0.000008280 -0.000004573 +11050 10.86378 -0.000008276 -0.000004571 +11051 10.86475 -0.000008271 -0.000004568 +11052 10.86571 -0.000008267 -0.000004565 +11053 10.86668 -0.000008262 -0.000004562 +11054 10.86764 -0.000008258 -0.000004559 +11055 10.86861 -0.000008254 -0.000004556 +11056 10.86957 -0.000008249 -0.000004554 +11057 10.87054 -0.000008245 -0.000004551 +11058 10.87150 -0.000008240 -0.000004548 +11059 10.87247 -0.000008236 -0.000004545 +11060 10.87343 -0.000008232 -0.000004542 +11061 10.87440 -0.000008227 -0.000004539 +11062 10.87536 -0.000008223 -0.000004537 +11063 10.87633 -0.000008218 -0.000004534 +11064 10.87729 -0.000008214 -0.000004531 +11065 10.87826 -0.000008210 -0.000004528 +11066 10.87922 -0.000008205 -0.000004525 +11067 10.88019 -0.000008201 -0.000004522 +11068 10.88115 -0.000008197 -0.000004520 +11069 10.88212 -0.000008192 -0.000004517 +11070 10.88308 -0.000008188 -0.000004514 +11071 10.88405 -0.000008184 -0.000004511 +11072 10.88501 -0.000008179 -0.000004508 +11073 10.88598 -0.000008175 -0.000004506 +11074 10.88694 -0.000008170 -0.000004503 +11075 10.88791 -0.000008166 -0.000004500 +11076 10.88887 -0.000008162 -0.000004497 +11077 10.88984 -0.000008157 -0.000004495 +11078 10.89080 -0.000008153 -0.000004492 +11079 10.89177 -0.000008149 -0.000004489 +11080 10.89273 -0.000008144 -0.000004486 +11081 10.89370 -0.000008140 -0.000004483 +11082 10.89466 -0.000008136 -0.000004481 +11083 10.89563 -0.000008131 -0.000004478 +11084 10.89659 -0.000008127 -0.000004475 +11085 10.89756 -0.000008123 -0.000004472 +11086 10.89852 -0.000008119 -0.000004470 +11087 10.89949 -0.000008114 -0.000004467 +11088 10.90046 -0.000008110 -0.000004464 +11089 10.90142 -0.000008106 -0.000004461 +11090 10.90239 -0.000008101 -0.000004458 +11091 10.90335 -0.000008097 -0.000004456 +11092 10.90432 -0.000008093 -0.000004453 +11093 10.90528 -0.000008088 -0.000004450 +11094 10.90625 -0.000008084 -0.000004447 +11095 10.90721 -0.000008080 -0.000004445 +11096 10.90818 -0.000008076 -0.000004442 +11097 10.90914 -0.000008071 -0.000004439 +11098 10.91011 -0.000008067 -0.000004436 +11099 10.91107 -0.000008063 -0.000004434 +11100 10.91204 -0.000008058 -0.000004431 +11101 10.91300 -0.000008054 -0.000004428 +11102 10.91397 -0.000008050 -0.000004425 +11103 10.91493 -0.000008046 -0.000004423 +11104 10.91590 -0.000008041 -0.000004420 +11105 10.91686 -0.000008037 -0.000004417 +11106 10.91783 -0.000008033 -0.000004414 +11107 10.91879 -0.000008029 -0.000004412 +11108 10.91976 -0.000008024 -0.000004409 +11109 10.92072 -0.000008020 -0.000004406 +11110 10.92169 -0.000008016 -0.000004404 +11111 10.92265 -0.000008012 -0.000004401 +11112 10.92362 -0.000008007 -0.000004398 +11113 10.92458 -0.000008003 -0.000004395 +11114 10.92555 -0.000007999 -0.000004393 +11115 10.92651 -0.000007995 -0.000004390 +11116 10.92748 -0.000007990 -0.000004387 +11117 10.92844 -0.000007986 -0.000004385 +11118 10.92941 -0.000007982 -0.000004382 +11119 10.93037 -0.000007978 -0.000004379 +11120 10.93134 -0.000007973 -0.000004376 +11121 10.93230 -0.000007969 -0.000004374 +11122 10.93327 -0.000007965 -0.000004371 +11123 10.93423 -0.000007961 -0.000004368 +11124 10.93520 -0.000007957 -0.000004366 +11125 10.93616 -0.000007952 -0.000004363 +11126 10.93713 -0.000007948 -0.000004360 +11127 10.93809 -0.000007944 -0.000004358 +11128 10.93906 -0.000007940 -0.000004355 +11129 10.94002 -0.000007935 -0.000004352 +11130 10.94099 -0.000007931 -0.000004349 +11131 10.94195 -0.000007927 -0.000004347 +11132 10.94292 -0.000007923 -0.000004344 +11133 10.94388 -0.000007919 -0.000004341 +11134 10.94485 -0.000007915 -0.000004339 +11135 10.94581 -0.000007910 -0.000004336 +11136 10.94678 -0.000007906 -0.000004333 +11137 10.94774 -0.000007902 -0.000004331 +11138 10.94871 -0.000007898 -0.000004328 +11139 10.94967 -0.000007894 -0.000004325 +11140 10.95064 -0.000007889 -0.000004323 +11141 10.95160 -0.000007885 -0.000004320 +11142 10.95257 -0.000007881 -0.000004317 +11143 10.95353 -0.000007877 -0.000004315 +11144 10.95450 -0.000007873 -0.000004312 +11145 10.95546 -0.000007869 -0.000004309 +11146 10.95643 -0.000007864 -0.000004307 +11147 10.95739 -0.000007860 -0.000004304 +11148 10.95836 -0.000007856 -0.000004301 +11149 10.95932 -0.000007852 -0.000004299 +11150 10.96029 -0.000007848 -0.000004296 +11151 10.96125 -0.000007844 -0.000004294 +11152 10.96222 -0.000007840 -0.000004291 +11153 10.96318 -0.000007835 -0.000004288 +11154 10.96415 -0.000007831 -0.000004286 +11155 10.96511 -0.000007827 -0.000004283 +11156 10.96608 -0.000007823 -0.000004280 +11157 10.96704 -0.000007819 -0.000004278 +11158 10.96801 -0.000007815 -0.000004275 +11159 10.96897 -0.000007811 -0.000004272 +11160 10.96994 -0.000007807 -0.000004270 +11161 10.97090 -0.000007802 -0.000004267 +11162 10.97187 -0.000007798 -0.000004265 +11163 10.97283 -0.000007794 -0.000004262 +11164 10.97380 -0.000007790 -0.000004259 +11165 10.97476 -0.000007786 -0.000004257 +11166 10.97573 -0.000007782 -0.000004254 +11167 10.97669 -0.000007778 -0.000004251 +11168 10.97766 -0.000007774 -0.000004249 +11169 10.97862 -0.000007770 -0.000004246 +11170 10.97959 -0.000007765 -0.000004244 +11171 10.98055 -0.000007761 -0.000004241 +11172 10.98152 -0.000007757 -0.000004238 +11173 10.98248 -0.000007753 -0.000004236 +11174 10.98345 -0.000007749 -0.000004233 +11175 10.98441 -0.000007745 -0.000004231 +11176 10.98538 -0.000007741 -0.000004228 +11177 10.98634 -0.000007737 -0.000004225 +11178 10.98731 -0.000007733 -0.000004223 +11179 10.98827 -0.000007729 -0.000004220 +11180 10.98924 -0.000007725 -0.000004218 +11181 10.99020 -0.000007721 -0.000004215 +11182 10.99117 -0.000007716 -0.000004212 +11183 10.99213 -0.000007712 -0.000004210 +11184 10.99310 -0.000007708 -0.000004207 +11185 10.99406 -0.000007704 -0.000004205 +11186 10.99503 -0.000007700 -0.000004202 +11187 10.99599 -0.000007696 -0.000004199 +11188 10.99696 -0.000007692 -0.000004197 +11189 10.99792 -0.000007688 -0.000004194 +11190 10.99889 -0.000007684 -0.000004192 +11191 10.99985 -0.000007680 -0.000004189 +11192 11.00082 -0.000007676 -0.000004187 +11193 11.00179 -0.000007672 -0.000004184 +11194 11.00275 -0.000007668 -0.000004181 +11195 11.00372 -0.000007664 -0.000004179 +11196 11.00468 -0.000007660 -0.000004176 +11197 11.00565 -0.000007656 -0.000004174 +11198 11.00661 -0.000007652 -0.000004171 +11199 11.00758 -0.000007648 -0.000004169 +11200 11.00854 -0.000007644 -0.000004166 +11201 11.00951 -0.000007640 -0.000004164 +11202 11.01047 -0.000007636 -0.000004161 +11203 11.01144 -0.000007632 -0.000004158 +11204 11.01240 -0.000007628 -0.000004156 +11205 11.01337 -0.000007624 -0.000004153 +11206 11.01433 -0.000007620 -0.000004151 +11207 11.01530 -0.000007616 -0.000004148 +11208 11.01626 -0.000007612 -0.000004146 +11209 11.01723 -0.000007608 -0.000004143 +11210 11.01819 -0.000007604 -0.000004141 +11211 11.01916 -0.000007600 -0.000004138 +11212 11.02012 -0.000007596 -0.000004136 +11213 11.02109 -0.000007592 -0.000004133 +11214 11.02205 -0.000007588 -0.000004130 +11215 11.02302 -0.000007584 -0.000004128 +11216 11.02398 -0.000007580 -0.000004125 +11217 11.02495 -0.000007576 -0.000004123 +11218 11.02591 -0.000007572 -0.000004120 +11219 11.02688 -0.000007568 -0.000004118 +11220 11.02784 -0.000007564 -0.000004115 +11221 11.02881 -0.000007560 -0.000004113 +11222 11.02977 -0.000007556 -0.000004110 +11223 11.03074 -0.000007552 -0.000004108 +11224 11.03170 -0.000007548 -0.000004105 +11225 11.03267 -0.000007544 -0.000004103 +11226 11.03363 -0.000007540 -0.000004100 +11227 11.03460 -0.000007536 -0.000004098 +11228 11.03556 -0.000007532 -0.000004095 +11229 11.03653 -0.000007528 -0.000004093 +11230 11.03749 -0.000007524 -0.000004090 +11231 11.03846 -0.000007520 -0.000004088 +11232 11.03942 -0.000007516 -0.000004085 +11233 11.04039 -0.000007512 -0.000004083 +11234 11.04135 -0.000007508 -0.000004080 +11235 11.04232 -0.000007504 -0.000004078 +11236 11.04328 -0.000007501 -0.000004075 +11237 11.04425 -0.000007497 -0.000004073 +11238 11.04521 -0.000007493 -0.000004070 +11239 11.04618 -0.000007489 -0.000004068 +11240 11.04714 -0.000007485 -0.000004065 +11241 11.04811 -0.000007481 -0.000004063 +11242 11.04907 -0.000007477 -0.000004060 +11243 11.05004 -0.000007473 -0.000004058 +11244 11.05100 -0.000007469 -0.000004055 +11245 11.05197 -0.000007465 -0.000004053 +11246 11.05293 -0.000007461 -0.000004050 +11247 11.05390 -0.000007457 -0.000004048 +11248 11.05486 -0.000007454 -0.000004045 +11249 11.05583 -0.000007450 -0.000004043 +11250 11.05679 -0.000007446 -0.000004040 +11251 11.05776 -0.000007442 -0.000004038 +11252 11.05872 -0.000007438 -0.000004036 +11253 11.05969 -0.000007434 -0.000004033 +11254 11.06065 -0.000007430 -0.000004031 +11255 11.06162 -0.000007426 -0.000004028 +11256 11.06258 -0.000007422 -0.000004026 +11257 11.06355 -0.000007418 -0.000004023 +11258 11.06451 -0.000007415 -0.000004021 +11259 11.06548 -0.000007411 -0.000004018 +11260 11.06644 -0.000007407 -0.000004016 +11261 11.06741 -0.000007403 -0.000004013 +11262 11.06837 -0.000007399 -0.000004011 +11263 11.06934 -0.000007395 -0.000004009 +11264 11.07030 -0.000007391 -0.000004006 +11265 11.07127 -0.000007388 -0.000004004 +11266 11.07223 -0.000007384 -0.000004001 +11267 11.07320 -0.000007380 -0.000003999 +11268 11.07416 -0.000007376 -0.000003996 +11269 11.07513 -0.000007372 -0.000003994 +11270 11.07609 -0.000007368 -0.000003991 +11271 11.07706 -0.000007364 -0.000003989 +11272 11.07802 -0.000007361 -0.000003987 +11273 11.07899 -0.000007357 -0.000003984 +11274 11.07995 -0.000007353 -0.000003982 +11275 11.08092 -0.000007349 -0.000003979 +11276 11.08188 -0.000007345 -0.000003977 +11277 11.08285 -0.000007341 -0.000003974 +11278 11.08381 -0.000007337 -0.000003972 +11279 11.08478 -0.000007334 -0.000003970 +11280 11.08574 -0.000007330 -0.000003967 +11281 11.08671 -0.000007326 -0.000003965 +11282 11.08767 -0.000007322 -0.000003962 +11283 11.08864 -0.000007318 -0.000003960 +11284 11.08960 -0.000007315 -0.000003958 +11285 11.09057 -0.000007311 -0.000003955 +11286 11.09153 -0.000007307 -0.000003953 +11287 11.09250 -0.000007303 -0.000003950 +11288 11.09346 -0.000007299 -0.000003948 +11289 11.09443 -0.000007295 -0.000003945 +11290 11.09539 -0.000007292 -0.000003943 +11291 11.09636 -0.000007288 -0.000003941 +11292 11.09732 -0.000007284 -0.000003938 +11293 11.09829 -0.000007280 -0.000003936 +11294 11.09925 -0.000007276 -0.000003933 +11295 11.10022 -0.000007273 -0.000003931 +11296 11.10119 -0.000007269 -0.000003929 +11297 11.10215 -0.000007265 -0.000003926 +11298 11.10312 -0.000007261 -0.000003924 +11299 11.10408 -0.000007257 -0.000003922 +11300 11.10505 -0.000007254 -0.000003919 +11301 11.10601 -0.000007250 -0.000003917 +11302 11.10698 -0.000007246 -0.000003914 +11303 11.10794 -0.000007242 -0.000003912 +11304 11.10891 -0.000007239 -0.000003910 +11305 11.10987 -0.000007235 -0.000003907 +11306 11.11084 -0.000007231 -0.000003905 +11307 11.11180 -0.000007227 -0.000003902 +11308 11.11277 -0.000007224 -0.000003900 +11309 11.11373 -0.000007220 -0.000003898 +11310 11.11470 -0.000007216 -0.000003895 +11311 11.11566 -0.000007212 -0.000003893 +11312 11.11663 -0.000007208 -0.000003891 +11313 11.11759 -0.000007205 -0.000003888 +11314 11.11856 -0.000007201 -0.000003886 +11315 11.11952 -0.000007197 -0.000003884 +11316 11.12049 -0.000007193 -0.000003881 +11317 11.12145 -0.000007190 -0.000003879 +11318 11.12242 -0.000007186 -0.000003876 +11319 11.12338 -0.000007182 -0.000003874 +11320 11.12435 -0.000007179 -0.000003872 +11321 11.12531 -0.000007175 -0.000003869 +11322 11.12628 -0.000007171 -0.000003867 +11323 11.12724 -0.000007167 -0.000003865 +11324 11.12821 -0.000007164 -0.000003862 +11325 11.12917 -0.000007160 -0.000003860 +11326 11.13014 -0.000007156 -0.000003858 +11327 11.13110 -0.000007152 -0.000003855 +11328 11.13207 -0.000007149 -0.000003853 +11329 11.13303 -0.000007145 -0.000003851 +11330 11.13400 -0.000007141 -0.000003848 +11331 11.13496 -0.000007138 -0.000003846 +11332 11.13593 -0.000007134 -0.000003844 +11333 11.13689 -0.000007130 -0.000003841 +11334 11.13786 -0.000007126 -0.000003839 +11335 11.13882 -0.000007123 -0.000003837 +11336 11.13979 -0.000007119 -0.000003834 +11337 11.14075 -0.000007115 -0.000003832 +11338 11.14172 -0.000007112 -0.000003830 +11339 11.14268 -0.000007108 -0.000003827 +11340 11.14365 -0.000007104 -0.000003825 +11341 11.14461 -0.000007101 -0.000003823 +11342 11.14558 -0.000007097 -0.000003820 +11343 11.14654 -0.000007093 -0.000003818 +11344 11.14751 -0.000007090 -0.000003816 +11345 11.14847 -0.000007086 -0.000003814 +11346 11.14944 -0.000007082 -0.000003811 +11347 11.15040 -0.000007078 -0.000003809 +11348 11.15137 -0.000007075 -0.000003807 +11349 11.15233 -0.000007071 -0.000003804 +11350 11.15330 -0.000007067 -0.000003802 +11351 11.15426 -0.000007064 -0.000003800 +11352 11.15523 -0.000007060 -0.000003797 +11353 11.15619 -0.000007056 -0.000003795 +11354 11.15716 -0.000007053 -0.000003793 +11355 11.15812 -0.000007049 -0.000003790 +11356 11.15909 -0.000007045 -0.000003788 +11357 11.16005 -0.000007042 -0.000003786 +11358 11.16102 -0.000007038 -0.000003784 +11359 11.16198 -0.000007035 -0.000003781 +11360 11.16295 -0.000007031 -0.000003779 +11361 11.16391 -0.000007027 -0.000003777 +11362 11.16488 -0.000007024 -0.000003774 +11363 11.16584 -0.000007020 -0.000003772 +11364 11.16681 -0.000007016 -0.000003770 +11365 11.16777 -0.000007013 -0.000003768 +11366 11.16874 -0.000007009 -0.000003765 +11367 11.16970 -0.000007005 -0.000003763 +11368 11.17067 -0.000007002 -0.000003761 +11369 11.17163 -0.000006998 -0.000003759 +11370 11.17260 -0.000006995 -0.000003756 +11371 11.17356 -0.000006991 -0.000003754 +11372 11.17453 -0.000006987 -0.000003752 +11373 11.17549 -0.000006984 -0.000003749 +11374 11.17646 -0.000006980 -0.000003747 +11375 11.17742 -0.000006976 -0.000003745 +11376 11.17839 -0.000006973 -0.000003743 +11377 11.17935 -0.000006969 -0.000003740 +11378 11.18032 -0.000006966 -0.000003738 +11379 11.18128 -0.000006962 -0.000003736 +11380 11.18225 -0.000006958 -0.000003734 +11381 11.18321 -0.000006955 -0.000003731 +11382 11.18418 -0.000006951 -0.000003729 +11383 11.18514 -0.000006948 -0.000003727 +11384 11.18611 -0.000006944 -0.000003725 +11385 11.18707 -0.000006940 -0.000003722 +11386 11.18804 -0.000006937 -0.000003720 +11387 11.18900 -0.000006933 -0.000003718 +11388 11.18997 -0.000006930 -0.000003716 +11389 11.19093 -0.000006926 -0.000003713 +11390 11.19190 -0.000006922 -0.000003711 +11391 11.19286 -0.000006919 -0.000003709 +11392 11.19383 -0.000006915 -0.000003707 +11393 11.19479 -0.000006912 -0.000003704 +11394 11.19576 -0.000006908 -0.000003702 +11395 11.19672 -0.000006905 -0.000003700 +11396 11.19769 -0.000006901 -0.000003698 +11397 11.19865 -0.000006897 -0.000003695 +11398 11.19962 -0.000006894 -0.000003693 +11399 11.20059 -0.000006890 -0.000003691 +11400 11.20155 -0.000006887 -0.000003689 +11401 11.20252 -0.000006883 -0.000003687 +11402 11.20348 -0.000006880 -0.000003684 +11403 11.20445 -0.000006876 -0.000003682 +11404 11.20541 -0.000006873 -0.000003680 +11405 11.20638 -0.000006869 -0.000003678 +11406 11.20734 -0.000006865 -0.000003675 +11407 11.20831 -0.000006862 -0.000003673 +11408 11.20927 -0.000006858 -0.000003671 +11409 11.21024 -0.000006855 -0.000003669 +11410 11.21120 -0.000006851 -0.000003667 +11411 11.21217 -0.000006848 -0.000003664 +11412 11.21313 -0.000006844 -0.000003662 +11413 11.21410 -0.000006841 -0.000003660 +11414 11.21506 -0.000006837 -0.000003658 +11415 11.21603 -0.000006834 -0.000003656 +11416 11.21699 -0.000006830 -0.000003653 +11417 11.21796 -0.000006827 -0.000003651 +11418 11.21892 -0.000006823 -0.000003649 +11419 11.21989 -0.000006819 -0.000003647 +11420 11.22085 -0.000006816 -0.000003645 +11421 11.22182 -0.000006812 -0.000003642 +11422 11.22278 -0.000006809 -0.000003640 +11423 11.22375 -0.000006805 -0.000003638 +11424 11.22471 -0.000006802 -0.000003636 +11425 11.22568 -0.000006798 -0.000003634 +11426 11.22664 -0.000006795 -0.000003631 +11427 11.22761 -0.000006791 -0.000003629 +11428 11.22857 -0.000006788 -0.000003627 +11429 11.22954 -0.000006784 -0.000003625 +11430 11.23050 -0.000006781 -0.000003623 +11431 11.23147 -0.000006777 -0.000003621 +11432 11.23243 -0.000006774 -0.000003618 +11433 11.23340 -0.000006770 -0.000003616 +11434 11.23436 -0.000006767 -0.000003614 +11435 11.23533 -0.000006763 -0.000003612 +11436 11.23629 -0.000006760 -0.000003610 +11437 11.23726 -0.000006756 -0.000003608 +11438 11.23822 -0.000006753 -0.000003605 +11439 11.23919 -0.000006750 -0.000003603 +11440 11.24015 -0.000006746 -0.000003601 +11441 11.24112 -0.000006743 -0.000003599 +11442 11.24208 -0.000006739 -0.000003597 +11443 11.24305 -0.000006736 -0.000003595 +11444 11.24401 -0.000006732 -0.000003592 +11445 11.24498 -0.000006729 -0.000003590 +11446 11.24594 -0.000006725 -0.000003588 +11447 11.24691 -0.000006722 -0.000003586 +11448 11.24787 -0.000006718 -0.000003584 +11449 11.24884 -0.000006715 -0.000003582 +11450 11.24980 -0.000006711 -0.000003579 +11451 11.25077 -0.000006708 -0.000003577 +11452 11.25173 -0.000006704 -0.000003575 +11453 11.25270 -0.000006701 -0.000003573 +11454 11.25366 -0.000006698 -0.000003571 +11455 11.25463 -0.000006694 -0.000003569 +11456 11.25559 -0.000006691 -0.000003567 +11457 11.25656 -0.000006687 -0.000003564 +11458 11.25752 -0.000006684 -0.000003562 +11459 11.25849 -0.000006680 -0.000003560 +11460 11.25945 -0.000006677 -0.000003558 +11461 11.26042 -0.000006674 -0.000003556 +11462 11.26138 -0.000006670 -0.000003554 +11463 11.26235 -0.000006667 -0.000003552 +11464 11.26331 -0.000006663 -0.000003550 +11465 11.26428 -0.000006660 -0.000003547 +11466 11.26524 -0.000006656 -0.000003545 +11467 11.26621 -0.000006653 -0.000003543 +11468 11.26717 -0.000006650 -0.000003541 +11469 11.26814 -0.000006646 -0.000003539 +11470 11.26910 -0.000006643 -0.000003537 +11471 11.27007 -0.000006639 -0.000003535 +11472 11.27103 -0.000006636 -0.000003533 +11473 11.27200 -0.000006632 -0.000003530 +11474 11.27296 -0.000006629 -0.000003528 +11475 11.27393 -0.000006626 -0.000003526 +11476 11.27489 -0.000006622 -0.000003524 +11477 11.27586 -0.000006619 -0.000003522 +11478 11.27682 -0.000006615 -0.000003520 +11479 11.27779 -0.000006612 -0.000003518 +11480 11.27875 -0.000006609 -0.000003516 +11481 11.27972 -0.000006605 -0.000003514 +11482 11.28068 -0.000006602 -0.000003511 +11483 11.28165 -0.000006599 -0.000003509 +11484 11.28261 -0.000006595 -0.000003507 +11485 11.28358 -0.000006592 -0.000003505 +11486 11.28454 -0.000006588 -0.000003503 +11487 11.28551 -0.000006585 -0.000003501 +11488 11.28647 -0.000006582 -0.000003499 +11489 11.28744 -0.000006578 -0.000003497 +11490 11.28840 -0.000006575 -0.000003495 +11491 11.28937 -0.000006571 -0.000003493 +11492 11.29033 -0.000006568 -0.000003490 +11493 11.29130 -0.000006565 -0.000003488 +11494 11.29226 -0.000006561 -0.000003486 +11495 11.29323 -0.000006558 -0.000003484 +11496 11.29419 -0.000006555 -0.000003482 +11497 11.29516 -0.000006551 -0.000003480 +11498 11.29612 -0.000006548 -0.000003478 +11499 11.29709 -0.000006545 -0.000003476 +11500 11.29805 -0.000006541 -0.000003474 +11501 11.29902 -0.000006538 -0.000003472 +11502 11.29998 -0.000006535 -0.000003470 +11503 11.30095 -0.000006531 -0.000003468 +11504 11.30192 -0.000006528 -0.000003466 +11505 11.30288 -0.000006525 -0.000003463 +11506 11.30385 -0.000006521 -0.000003461 +11507 11.30481 -0.000006518 -0.000003459 +11508 11.30578 -0.000006514 -0.000003457 +11509 11.30674 -0.000006511 -0.000003455 +11510 11.30771 -0.000006508 -0.000003453 +11511 11.30867 -0.000006504 -0.000003451 +11512 11.30964 -0.000006501 -0.000003449 +11513 11.31060 -0.000006498 -0.000003447 +11514 11.31157 -0.000006495 -0.000003445 +11515 11.31253 -0.000006491 -0.000003443 +11516 11.31350 -0.000006488 -0.000003441 +11517 11.31446 -0.000006485 -0.000003439 +11518 11.31543 -0.000006481 -0.000003437 +11519 11.31639 -0.000006478 -0.000003435 +11520 11.31736 -0.000006475 -0.000003433 +11521 11.31832 -0.000006471 -0.000003431 +11522 11.31929 -0.000006468 -0.000003428 +11523 11.32025 -0.000006465 -0.000003426 +11524 11.32122 -0.000006461 -0.000003424 +11525 11.32218 -0.000006458 -0.000003422 +11526 11.32315 -0.000006455 -0.000003420 +11527 11.32411 -0.000006451 -0.000003418 +11528 11.32508 -0.000006448 -0.000003416 +11529 11.32604 -0.000006445 -0.000003414 +11530 11.32701 -0.000006442 -0.000003412 +11531 11.32797 -0.000006438 -0.000003410 +11532 11.32894 -0.000006435 -0.000003408 +11533 11.32990 -0.000006432 -0.000003406 +11534 11.33087 -0.000006428 -0.000003404 +11535 11.33183 -0.000006425 -0.000003402 +11536 11.33280 -0.000006422 -0.000003400 +11537 11.33376 -0.000006419 -0.000003398 +11538 11.33473 -0.000006415 -0.000003396 +11539 11.33569 -0.000006412 -0.000003394 +11540 11.33666 -0.000006409 -0.000003392 +11541 11.33762 -0.000006405 -0.000003390 +11542 11.33859 -0.000006402 -0.000003388 +11543 11.33955 -0.000006399 -0.000003386 +11544 11.34052 -0.000006396 -0.000003384 +11545 11.34148 -0.000006392 -0.000003382 +11546 11.34245 -0.000006389 -0.000003380 +11547 11.34341 -0.000006386 -0.000003378 +11548 11.34438 -0.000006383 -0.000003376 +11549 11.34534 -0.000006379 -0.000003374 +11550 11.34631 -0.000006376 -0.000003372 +11551 11.34727 -0.000006373 -0.000003370 +11552 11.34824 -0.000006370 -0.000003368 +11553 11.34920 -0.000006366 -0.000003366 +11554 11.35017 -0.000006363 -0.000003364 +11555 11.35113 -0.000006360 -0.000003362 +11556 11.35210 -0.000006357 -0.000003360 +11557 11.35306 -0.000006353 -0.000003358 +11558 11.35403 -0.000006350 -0.000003356 +11559 11.35499 -0.000006347 -0.000003354 +11560 11.35596 -0.000006344 -0.000003352 +11561 11.35692 -0.000006340 -0.000003350 +11562 11.35789 -0.000006337 -0.000003348 +11563 11.35885 -0.000006334 -0.000003346 +11564 11.35982 -0.000006331 -0.000003344 +11565 11.36078 -0.000006328 -0.000003342 +11566 11.36175 -0.000006324 -0.000003340 +11567 11.36271 -0.000006321 -0.000003338 +11568 11.36368 -0.000006318 -0.000003336 +11569 11.36464 -0.000006315 -0.000003334 +11570 11.36561 -0.000006311 -0.000003332 +11571 11.36657 -0.000006308 -0.000003330 +11572 11.36754 -0.000006305 -0.000003328 +11573 11.36850 -0.000006302 -0.000003326 +11574 11.36947 -0.000006299 -0.000003324 +11575 11.37043 -0.000006295 -0.000003322 +11576 11.37140 -0.000006292 -0.000003320 +11577 11.37236 -0.000006289 -0.000003318 +11578 11.37333 -0.000006286 -0.000003316 +11579 11.37429 -0.000006283 -0.000003314 +11580 11.37526 -0.000006279 -0.000003312 +11581 11.37622 -0.000006276 -0.000003310 +11582 11.37719 -0.000006273 -0.000003308 +11583 11.37815 -0.000006270 -0.000003306 +11584 11.37912 -0.000006267 -0.000003304 +11585 11.38008 -0.000006263 -0.000003302 +11586 11.38105 -0.000006260 -0.000003300 +11587 11.38201 -0.000006257 -0.000003298 +11588 11.38298 -0.000006254 -0.000003296 +11589 11.38394 -0.000006251 -0.000003294 +11590 11.38491 -0.000006247 -0.000003293 +11591 11.38587 -0.000006244 -0.000003291 +11592 11.38684 -0.000006241 -0.000003289 +11593 11.38780 -0.000006238 -0.000003287 +11594 11.38877 -0.000006235 -0.000003285 +11595 11.38973 -0.000006232 -0.000003283 +11596 11.39070 -0.000006228 -0.000003281 +11597 11.39166 -0.000006225 -0.000003279 +11598 11.39263 -0.000006222 -0.000003277 +11599 11.39359 -0.000006219 -0.000003275 +11600 11.39456 -0.000006216 -0.000003273 +11601 11.39552 -0.000006213 -0.000003271 +11602 11.39649 -0.000006209 -0.000003269 +11603 11.39745 -0.000006206 -0.000003267 +11604 11.39842 -0.000006203 -0.000003265 +11605 11.39938 -0.000006200 -0.000003263 +11606 11.40035 -0.000006197 -0.000003261 +11607 11.40132 -0.000006194 -0.000003259 +11608 11.40228 -0.000006191 -0.000003258 +11609 11.40325 -0.000006187 -0.000003256 +11610 11.40421 -0.000006184 -0.000003254 +11611 11.40518 -0.000006181 -0.000003252 +11612 11.40614 -0.000006178 -0.000003250 +11613 11.40711 -0.000006175 -0.000003248 +11614 11.40807 -0.000006172 -0.000003246 +11615 11.40904 -0.000006169 -0.000003244 +11616 11.41000 -0.000006166 -0.000003242 +11617 11.41097 -0.000006162 -0.000003240 +11618 11.41193 -0.000006159 -0.000003238 +11619 11.41290 -0.000006156 -0.000003236 +11620 11.41386 -0.000006153 -0.000003234 +11621 11.41483 -0.000006150 -0.000003233 +11622 11.41579 -0.000006147 -0.000003231 +11623 11.41676 -0.000006144 -0.000003229 +11624 11.41772 -0.000006141 -0.000003227 +11625 11.41869 -0.000006137 -0.000003225 +11626 11.41965 -0.000006134 -0.000003223 +11627 11.42062 -0.000006131 -0.000003221 +11628 11.42158 -0.000006128 -0.000003219 +11629 11.42255 -0.000006125 -0.000003217 +11630 11.42351 -0.000006122 -0.000003215 +11631 11.42448 -0.000006119 -0.000003214 +11632 11.42544 -0.000006116 -0.000003212 +11633 11.42641 -0.000006113 -0.000003210 +11634 11.42737 -0.000006109 -0.000003208 +11635 11.42834 -0.000006106 -0.000003206 +11636 11.42930 -0.000006103 -0.000003204 +11637 11.43027 -0.000006100 -0.000003202 +11638 11.43123 -0.000006097 -0.000003200 +11639 11.43220 -0.000006094 -0.000003198 +11640 11.43316 -0.000006091 -0.000003196 +11641 11.43413 -0.000006088 -0.000003195 +11642 11.43509 -0.000006085 -0.000003193 +11643 11.43606 -0.000006082 -0.000003191 +11644 11.43702 -0.000006079 -0.000003189 +11645 11.43799 -0.000006076 -0.000003187 +11646 11.43895 -0.000006072 -0.000003185 +11647 11.43992 -0.000006069 -0.000003183 +11648 11.44088 -0.000006066 -0.000003181 +11649 11.44185 -0.000006063 -0.000003180 +11650 11.44281 -0.000006060 -0.000003178 +11651 11.44378 -0.000006057 -0.000003176 +11652 11.44474 -0.000006054 -0.000003174 +11653 11.44571 -0.000006051 -0.000003172 +11654 11.44667 -0.000006048 -0.000003170 +11655 11.44764 -0.000006045 -0.000003168 +11656 11.44860 -0.000006042 -0.000003166 +11657 11.44957 -0.000006039 -0.000003165 +11658 11.45053 -0.000006036 -0.000003163 +11659 11.45150 -0.000006033 -0.000003161 +11660 11.45246 -0.000006030 -0.000003159 +11661 11.45343 -0.000006027 -0.000003157 +11662 11.45439 -0.000006024 -0.000003155 +11663 11.45536 -0.000006020 -0.000003153 +11664 11.45632 -0.000006017 -0.000003151 +11665 11.45729 -0.000006014 -0.000003150 +11666 11.45825 -0.000006011 -0.000003148 +11667 11.45922 -0.000006008 -0.000003146 +11668 11.46018 -0.000006005 -0.000003144 +11669 11.46115 -0.000006002 -0.000003142 +11670 11.46211 -0.000005999 -0.000003140 +11671 11.46308 -0.000005996 -0.000003139 +11672 11.46404 -0.000005993 -0.000003137 +11673 11.46501 -0.000005990 -0.000003135 +11674 11.46597 -0.000005987 -0.000003133 +11675 11.46694 -0.000005984 -0.000003131 +11676 11.46790 -0.000005981 -0.000003129 +11677 11.46887 -0.000005978 -0.000003127 +11678 11.46983 -0.000005975 -0.000003126 +11679 11.47080 -0.000005972 -0.000003124 +11680 11.47176 -0.000005969 -0.000003122 +11681 11.47273 -0.000005966 -0.000003120 +11682 11.47369 -0.000005963 -0.000003118 +11683 11.47466 -0.000005960 -0.000003116 +11684 11.47562 -0.000005957 -0.000003115 +11685 11.47659 -0.000005954 -0.000003113 +11686 11.47755 -0.000005951 -0.000003111 +11687 11.47852 -0.000005948 -0.000003109 +11688 11.47948 -0.000005945 -0.000003107 +11689 11.48045 -0.000005942 -0.000003105 +11690 11.48141 -0.000005939 -0.000003104 +11691 11.48238 -0.000005936 -0.000003102 +11692 11.48334 -0.000005933 -0.000003100 +11693 11.48431 -0.000005930 -0.000003098 +11694 11.48527 -0.000005927 -0.000003096 +11695 11.48624 -0.000005924 -0.000003094 +11696 11.48720 -0.000005921 -0.000003093 +11697 11.48817 -0.000005918 -0.000003091 +11698 11.48913 -0.000005915 -0.000003089 +11699 11.49010 -0.000005912 -0.000003087 +11700 11.49106 -0.000005909 -0.000003085 +11701 11.49203 -0.000005906 -0.000003084 +11702 11.49299 -0.000005903 -0.000003082 +11703 11.49396 -0.000005900 -0.000003080 +11704 11.49492 -0.000005897 -0.000003078 +11705 11.49589 -0.000005894 -0.000003076 +11706 11.49685 -0.000005891 -0.000003075 +11707 11.49782 -0.000005888 -0.000003073 +11708 11.49878 -0.000005885 -0.000003071 +11709 11.49975 -0.000005882 -0.000003069 +11710 11.50072 -0.000005879 -0.000003067 +11711 11.50168 -0.000005876 -0.000003066 +11712 11.50265 -0.000005873 -0.000003064 +11713 11.50361 -0.000005871 -0.000003062 +11714 11.50458 -0.000005868 -0.000003060 +11715 11.50554 -0.000005865 -0.000003058 +11716 11.50651 -0.000005862 -0.000003057 +11717 11.50747 -0.000005859 -0.000003055 +11718 11.50844 -0.000005856 -0.000003053 +11719 11.50940 -0.000005853 -0.000003051 +11720 11.51037 -0.000005850 -0.000003049 +11721 11.51133 -0.000005847 -0.000003048 +11722 11.51230 -0.000005844 -0.000003046 +11723 11.51326 -0.000005841 -0.000003044 +11724 11.51423 -0.000005838 -0.000003042 +11725 11.51519 -0.000005835 -0.000003040 +11726 11.51616 -0.000005832 -0.000003039 +11727 11.51712 -0.000005829 -0.000003037 +11728 11.51809 -0.000005826 -0.000003035 +11729 11.51905 -0.000005823 -0.000003033 +11730 11.52002 -0.000005821 -0.000003032 +11731 11.52098 -0.000005818 -0.000003030 +11732 11.52195 -0.000005815 -0.000003028 +11733 11.52291 -0.000005812 -0.000003026 +11734 11.52388 -0.000005809 -0.000003024 +11735 11.52484 -0.000005806 -0.000003023 +11736 11.52581 -0.000005803 -0.000003021 +11737 11.52677 -0.000005800 -0.000003019 +11738 11.52774 -0.000005797 -0.000003017 +11739 11.52870 -0.000005794 -0.000003016 +11740 11.52967 -0.000005791 -0.000003014 +11741 11.53063 -0.000005788 -0.000003012 +11742 11.53160 -0.000005786 -0.000003010 +11743 11.53256 -0.000005783 -0.000003009 +11744 11.53353 -0.000005780 -0.000003007 +11745 11.53449 -0.000005777 -0.000003005 +11746 11.53546 -0.000005774 -0.000003003 +11747 11.53642 -0.000005771 -0.000003001 +11748 11.53739 -0.000005768 -0.000003000 +11749 11.53835 -0.000005765 -0.000002998 +11750 11.53932 -0.000005762 -0.000002996 +11751 11.54028 -0.000005759 -0.000002994 +11752 11.54125 -0.000005757 -0.000002993 +11753 11.54221 -0.000005754 -0.000002991 +11754 11.54318 -0.000005751 -0.000002989 +11755 11.54414 -0.000005748 -0.000002987 +11756 11.54511 -0.000005745 -0.000002986 +11757 11.54607 -0.000005742 -0.000002984 +11758 11.54704 -0.000005739 -0.000002982 +11759 11.54800 -0.000005736 -0.000002980 +11760 11.54897 -0.000005734 -0.000002979 +11761 11.54993 -0.000005731 -0.000002977 +11762 11.55090 -0.000005728 -0.000002975 +11763 11.55186 -0.000005725 -0.000002974 +11764 11.55283 -0.000005722 -0.000002972 +11765 11.55379 -0.000005719 -0.000002970 +11766 11.55476 -0.000005716 -0.000002968 +11767 11.55572 -0.000005713 -0.000002967 +11768 11.55669 -0.000005711 -0.000002965 +11769 11.55765 -0.000005708 -0.000002963 +11770 11.55862 -0.000005705 -0.000002961 +11771 11.55958 -0.000005702 -0.000002960 +11772 11.56055 -0.000005699 -0.000002958 +11773 11.56151 -0.000005696 -0.000002956 +11774 11.56248 -0.000005693 -0.000002954 +11775 11.56344 -0.000005691 -0.000002953 +11776 11.56441 -0.000005688 -0.000002951 +11777 11.56537 -0.000005685 -0.000002949 +11778 11.56634 -0.000005682 -0.000002948 +11779 11.56730 -0.000005679 -0.000002946 +11780 11.56827 -0.000005676 -0.000002944 +11781 11.56923 -0.000005674 -0.000002942 +11782 11.57020 -0.000005671 -0.000002941 +11783 11.57116 -0.000005668 -0.000002939 +11784 11.57213 -0.000005665 -0.000002937 +11785 11.57309 -0.000005662 -0.000002936 +11786 11.57406 -0.000005659 -0.000002934 +11787 11.57502 -0.000005657 -0.000002932 +11788 11.57599 -0.000005654 -0.000002930 +11789 11.57695 -0.000005651 -0.000002929 +11790 11.57792 -0.000005648 -0.000002927 +11791 11.57888 -0.000005645 -0.000002925 +11792 11.57985 -0.000005642 -0.000002924 +11793 11.58081 -0.000005640 -0.000002922 +11794 11.58178 -0.000005637 -0.000002920 +11795 11.58274 -0.000005634 -0.000002918 +11796 11.58371 -0.000005631 -0.000002917 +11797 11.58467 -0.000005628 -0.000002915 +11798 11.58564 -0.000005626 -0.000002913 +11799 11.58660 -0.000005623 -0.000002912 +11800 11.58757 -0.000005620 -0.000002910 +11801 11.58853 -0.000005617 -0.000002908 +11802 11.58950 -0.000005614 -0.000002907 +11803 11.59046 -0.000005611 -0.000002905 +11804 11.59143 -0.000005609 -0.000002903 +11805 11.59239 -0.000005606 -0.000002901 +11806 11.59336 -0.000005603 -0.000002900 +11807 11.59432 -0.000005600 -0.000002898 +11808 11.59529 -0.000005597 -0.000002896 +11809 11.59625 -0.000005595 -0.000002895 +11810 11.59722 -0.000005592 -0.000002893 +11811 11.59818 -0.000005589 -0.000002891 +11812 11.59915 -0.000005586 -0.000002890 +11813 11.60012 -0.000005584 -0.000002888 +11814 11.60108 -0.000005581 -0.000002886 +11815 11.60205 -0.000005578 -0.000002885 +11816 11.60301 -0.000005575 -0.000002883 +11817 11.60398 -0.000005572 -0.000002881 +11818 11.60494 -0.000005570 -0.000002880 +11819 11.60591 -0.000005567 -0.000002878 +11820 11.60687 -0.000005564 -0.000002876 +11821 11.60784 -0.000005561 -0.000002875 +11822 11.60880 -0.000005559 -0.000002873 +11823 11.60977 -0.000005556 -0.000002871 +11824 11.61073 -0.000005553 -0.000002870 +11825 11.61170 -0.000005550 -0.000002868 +11826 11.61266 -0.000005547 -0.000002866 +11827 11.61363 -0.000005545 -0.000002865 +11828 11.61459 -0.000005542 -0.000002863 +11829 11.61556 -0.000005539 -0.000002861 +11830 11.61652 -0.000005536 -0.000002860 +11831 11.61749 -0.000005534 -0.000002858 +11832 11.61845 -0.000005531 -0.000002856 +11833 11.61942 -0.000005528 -0.000002855 +11834 11.62038 -0.000005525 -0.000002853 +11835 11.62135 -0.000005523 -0.000002851 +11836 11.62231 -0.000005520 -0.000002850 +11837 11.62328 -0.000005517 -0.000002848 +11838 11.62424 -0.000005514 -0.000002846 +11839 11.62521 -0.000005512 -0.000002845 +11840 11.62617 -0.000005509 -0.000002843 +11841 11.62714 -0.000005506 -0.000002841 +11842 11.62810 -0.000005503 -0.000002840 +11843 11.62907 -0.000005501 -0.000002838 +11844 11.63003 -0.000005498 -0.000002836 +11845 11.63100 -0.000005495 -0.000002835 +11846 11.63196 -0.000005492 -0.000002833 +11847 11.63293 -0.000005490 -0.000002831 +11848 11.63389 -0.000005487 -0.000002830 +11849 11.63486 -0.000005484 -0.000002828 +11850 11.63582 -0.000005482 -0.000002827 +11851 11.63679 -0.000005479 -0.000002825 +11852 11.63775 -0.000005476 -0.000002823 +11853 11.63872 -0.000005473 -0.000002822 +11854 11.63968 -0.000005471 -0.000002820 +11855 11.64065 -0.000005468 -0.000002818 +11856 11.64161 -0.000005465 -0.000002817 +11857 11.64258 -0.000005462 -0.000002815 +11858 11.64354 -0.000005460 -0.000002813 +11859 11.64451 -0.000005457 -0.000002812 +11860 11.64547 -0.000005454 -0.000002810 +11861 11.64644 -0.000005452 -0.000002809 +11862 11.64740 -0.000005449 -0.000002807 +11863 11.64837 -0.000005446 -0.000002805 +11864 11.64933 -0.000005443 -0.000002804 +11865 11.65030 -0.000005441 -0.000002802 +11866 11.65126 -0.000005438 -0.000002800 +11867 11.65223 -0.000005435 -0.000002799 +11868 11.65319 -0.000005433 -0.000002797 +11869 11.65416 -0.000005430 -0.000002796 +11870 11.65512 -0.000005427 -0.000002794 +11871 11.65609 -0.000005425 -0.000002792 +11872 11.65705 -0.000005422 -0.000002791 +11873 11.65802 -0.000005419 -0.000002789 +11874 11.65898 -0.000005416 -0.000002787 +11875 11.65995 -0.000005414 -0.000002786 +11876 11.66091 -0.000005411 -0.000002784 +11877 11.66188 -0.000005408 -0.000002783 +11878 11.66284 -0.000005406 -0.000002781 +11879 11.66381 -0.000005403 -0.000002779 +11880 11.66477 -0.000005400 -0.000002778 +11881 11.66574 -0.000005398 -0.000002776 +11882 11.66670 -0.000005395 -0.000002775 +11883 11.66767 -0.000005392 -0.000002773 +11884 11.66863 -0.000005390 -0.000002771 +11885 11.66960 -0.000005387 -0.000002770 +11886 11.67056 -0.000005384 -0.000002768 +11887 11.67153 -0.000005382 -0.000002767 +11888 11.67249 -0.000005379 -0.000002765 +11889 11.67346 -0.000005376 -0.000002763 +11890 11.67442 -0.000005374 -0.000002762 +11891 11.67539 -0.000005371 -0.000002760 +11892 11.67635 -0.000005368 -0.000002759 +11893 11.67732 -0.000005366 -0.000002757 +11894 11.67828 -0.000005363 -0.000002755 +11895 11.67925 -0.000005360 -0.000002754 +11896 11.68021 -0.000005358 -0.000002752 +11897 11.68118 -0.000005355 -0.000002751 +11898 11.68214 -0.000005352 -0.000002749 +11899 11.68311 -0.000005350 -0.000002747 +11900 11.68407 -0.000005347 -0.000002746 +11901 11.68504 -0.000005344 -0.000002744 +11902 11.68600 -0.000005342 -0.000002743 +11903 11.68697 -0.000005339 -0.000002741 +11904 11.68793 -0.000005336 -0.000002739 +11905 11.68890 -0.000005334 -0.000002738 +11906 11.68986 -0.000005331 -0.000002736 +11907 11.69083 -0.000005329 -0.000002735 +11908 11.69179 -0.000005326 -0.000002733 +11909 11.69276 -0.000005323 -0.000002732 +11910 11.69372 -0.000005321 -0.000002730 +11911 11.69469 -0.000005318 -0.000002728 +11912 11.69565 -0.000005315 -0.000002727 +11913 11.69662 -0.000005313 -0.000002725 +11914 11.69758 -0.000005310 -0.000002724 +11915 11.69855 -0.000005308 -0.000002722 +11916 11.69951 -0.000005305 -0.000002721 +11917 11.70048 -0.000005302 -0.000002719 +11918 11.70145 -0.000005300 -0.000002717 +11919 11.70241 -0.000005297 -0.000002716 +11920 11.70338 -0.000005294 -0.000002714 +11921 11.70434 -0.000005292 -0.000002713 +11922 11.70531 -0.000005289 -0.000002711 +11923 11.70627 -0.000005287 -0.000002710 +11924 11.70724 -0.000005284 -0.000002708 +11925 11.70820 -0.000005281 -0.000002706 +11926 11.70917 -0.000005279 -0.000002705 +11927 11.71013 -0.000005276 -0.000002703 +11928 11.71110 -0.000005273 -0.000002702 +11929 11.71206 -0.000005271 -0.000002700 +11930 11.71303 -0.000005268 -0.000002699 +11931 11.71399 -0.000005266 -0.000002697 +11932 11.71496 -0.000005263 -0.000002696 +11933 11.71592 -0.000005260 -0.000002694 +11934 11.71689 -0.000005258 -0.000002692 +11935 11.71785 -0.000005255 -0.000002691 +11936 11.71882 -0.000005253 -0.000002689 +11937 11.71978 -0.000005250 -0.000002688 +11938 11.72075 -0.000005247 -0.000002686 +11939 11.72171 -0.000005245 -0.000002685 +11940 11.72268 -0.000005242 -0.000002683 +11941 11.72364 -0.000005240 -0.000002682 +11942 11.72461 -0.000005237 -0.000002680 +11943 11.72557 -0.000005235 -0.000002679 +11944 11.72654 -0.000005232 -0.000002677 +11945 11.72750 -0.000005229 -0.000002675 +11946 11.72847 -0.000005227 -0.000002674 +11947 11.72943 -0.000005224 -0.000002672 +11948 11.73040 -0.000005222 -0.000002671 +11949 11.73136 -0.000005219 -0.000002669 +11950 11.73233 -0.000005216 -0.000002668 +11951 11.73329 -0.000005214 -0.000002666 +11952 11.73426 -0.000005211 -0.000002665 +11953 11.73522 -0.000005209 -0.000002663 +11954 11.73619 -0.000005206 -0.000002662 +11955 11.73715 -0.000005204 -0.000002660 +11956 11.73812 -0.000005201 -0.000002659 +11957 11.73908 -0.000005198 -0.000002657 +11958 11.74005 -0.000005196 -0.000002655 +11959 11.74101 -0.000005193 -0.000002654 +11960 11.74198 -0.000005191 -0.000002652 +11961 11.74294 -0.000005188 -0.000002651 +11962 11.74391 -0.000005186 -0.000002649 +11963 11.74487 -0.000005183 -0.000002648 +11964 11.74584 -0.000005181 -0.000002646 +11965 11.74680 -0.000005178 -0.000002645 +11966 11.74777 -0.000005175 -0.000002643 +11967 11.74873 -0.000005173 -0.000002642 +11968 11.74970 -0.000005170 -0.000002640 +11969 11.75066 -0.000005168 -0.000002639 +11970 11.75163 -0.000005165 -0.000002637 +11971 11.75259 -0.000005163 -0.000002636 +11972 11.75356 -0.000005160 -0.000002634 +11973 11.75452 -0.000005158 -0.000002633 +11974 11.75549 -0.000005155 -0.000002631 +11975 11.75645 -0.000005153 -0.000002630 +11976 11.75742 -0.000005150 -0.000002628 +11977 11.75838 -0.000005148 -0.000002627 +11978 11.75935 -0.000005145 -0.000002625 +11979 11.76031 -0.000005142 -0.000002624 +11980 11.76128 -0.000005140 -0.000002622 +11981 11.76224 -0.000005137 -0.000002621 +11982 11.76321 -0.000005135 -0.000002619 +11983 11.76417 -0.000005132 -0.000002618 +11984 11.76514 -0.000005130 -0.000002616 +11985 11.76610 -0.000005127 -0.000002615 +11986 11.76707 -0.000005125 -0.000002613 +11987 11.76803 -0.000005122 -0.000002612 +11988 11.76900 -0.000005120 -0.000002610 +11989 11.76996 -0.000005117 -0.000002609 +11990 11.77093 -0.000005115 -0.000002607 +11991 11.77189 -0.000005112 -0.000002606 +11992 11.77286 -0.000005110 -0.000002604 +11993 11.77382 -0.000005107 -0.000002603 +11994 11.77479 -0.000005105 -0.000002601 +11995 11.77575 -0.000005102 -0.000002600 +11996 11.77672 -0.000005100 -0.000002598 +11997 11.77768 -0.000005097 -0.000002597 +11998 11.77865 -0.000005095 -0.000002595 +11999 11.77961 -0.000005092 -0.000002594 +12000 11.78058 -0.000005090 -0.000002592 +12001 11.78154 -0.000005087 -0.000002591 +12002 11.78251 -0.000005085 -0.000002589 +12003 11.78347 -0.000005082 -0.000002588 +12004 11.78444 -0.000005080 -0.000002586 +12005 11.78540 -0.000005077 -0.000002585 +12006 11.78637 -0.000005075 -0.000002583 +12007 11.78733 -0.000005072 -0.000002582 +12008 11.78830 -0.000005070 -0.000002580 +12009 11.78926 -0.000005067 -0.000002579 +12010 11.79023 -0.000005065 -0.000002577 +12011 11.79119 -0.000005062 -0.000002576 +12012 11.79216 -0.000005060 -0.000002574 +12013 11.79312 -0.000005057 -0.000002573 +12014 11.79409 -0.000005055 -0.000002571 +12015 11.79505 -0.000005052 -0.000002570 +12016 11.79602 -0.000005050 -0.000002569 +12017 11.79698 -0.000005047 -0.000002567 +12018 11.79795 -0.000005045 -0.000002566 +12019 11.79891 -0.000005042 -0.000002564 +12020 11.79988 -0.000005040 -0.000002563 +12021 11.80085 -0.000005037 -0.000002561 +12022 11.80181 -0.000005035 -0.000002560 +12023 11.80278 -0.000005032 -0.000002558 +12024 11.80374 -0.000005030 -0.000002557 +12025 11.80471 -0.000005027 -0.000002555 +12026 11.80567 -0.000005025 -0.000002554 +12027 11.80664 -0.000005023 -0.000002552 +12028 11.80760 -0.000005020 -0.000002551 +12029 11.80857 -0.000005018 -0.000002549 +12030 11.80953 -0.000005015 -0.000002548 +12031 11.81050 -0.000005013 -0.000002547 +12032 11.81146 -0.000005010 -0.000002545 +12033 11.81243 -0.000005008 -0.000002544 +12034 11.81339 -0.000005005 -0.000002542 +12035 11.81436 -0.000005003 -0.000002541 +12036 11.81532 -0.000005000 -0.000002539 +12037 11.81629 -0.000004998 -0.000002538 +12038 11.81725 -0.000004996 -0.000002536 +12039 11.81822 -0.000004993 -0.000002535 +12040 11.81918 -0.000004991 -0.000002534 +12041 11.82015 -0.000004988 -0.000002532 +12042 11.82111 -0.000004986 -0.000002531 +12043 11.82208 -0.000004983 -0.000002529 +12044 11.82304 -0.000004981 -0.000002528 +12045 11.82401 -0.000004978 -0.000002526 +12046 11.82497 -0.000004976 -0.000002525 +12047 11.82594 -0.000004974 -0.000002523 +12048 11.82690 -0.000004971 -0.000002522 +12049 11.82787 -0.000004969 -0.000002521 +12050 11.82883 -0.000004966 -0.000002519 +12051 11.82980 -0.000004964 -0.000002518 +12052 11.83076 -0.000004961 -0.000002516 +12053 11.83173 -0.000004959 -0.000002515 +12054 11.83269 -0.000004957 -0.000002513 +12055 11.83366 -0.000004954 -0.000002512 +12056 11.83462 -0.000004952 -0.000002510 +12057 11.83559 -0.000004949 -0.000002509 +12058 11.83655 -0.000004947 -0.000002508 +12059 11.83752 -0.000004944 -0.000002506 +12060 11.83848 -0.000004942 -0.000002505 +12061 11.83945 -0.000004940 -0.000002503 +12062 11.84041 -0.000004937 -0.000002502 +12063 11.84138 -0.000004935 -0.000002500 +12064 11.84234 -0.000004932 -0.000002499 +12065 11.84331 -0.000004930 -0.000002498 +12066 11.84427 -0.000004928 -0.000002496 +12067 11.84524 -0.000004925 -0.000002495 +12068 11.84620 -0.000004923 -0.000002493 +12069 11.84717 -0.000004920 -0.000002492 +12070 11.84813 -0.000004918 -0.000002490 +12071 11.84910 -0.000004916 -0.000002489 +12072 11.85006 -0.000004913 -0.000002488 +12073 11.85103 -0.000004911 -0.000002486 +12074 11.85199 -0.000004908 -0.000002485 +12075 11.85296 -0.000004906 -0.000002483 +12076 11.85392 -0.000004904 -0.000002482 +12077 11.85489 -0.000004901 -0.000002481 +12078 11.85585 -0.000004899 -0.000002479 +12079 11.85682 -0.000004896 -0.000002478 +12080 11.85778 -0.000004894 -0.000002476 +12081 11.85875 -0.000004892 -0.000002475 +12082 11.85971 -0.000004889 -0.000002474 +12083 11.86068 -0.000004887 -0.000002472 +12084 11.86164 -0.000004884 -0.000002471 +12085 11.86261 -0.000004882 -0.000002469 +12086 11.86357 -0.000004880 -0.000002468 +12087 11.86454 -0.000004877 -0.000002466 +12088 11.86550 -0.000004875 -0.000002465 +12089 11.86647 -0.000004873 -0.000002464 +12090 11.86743 -0.000004870 -0.000002462 +12091 11.86840 -0.000004868 -0.000002461 +12092 11.86936 -0.000004865 -0.000002459 +12093 11.87033 -0.000004863 -0.000002458 +12094 11.87129 -0.000004861 -0.000002457 +12095 11.87226 -0.000004858 -0.000002455 +12096 11.87322 -0.000004856 -0.000002454 +12097 11.87419 -0.000004854 -0.000002452 +12098 11.87515 -0.000004851 -0.000002451 +12099 11.87612 -0.000004849 -0.000002450 +12100 11.87708 -0.000004846 -0.000002448 +12101 11.87805 -0.000004844 -0.000002447 +12102 11.87901 -0.000004842 -0.000002446 +12103 11.87998 -0.000004839 -0.000002444 +12104 11.88094 -0.000004837 -0.000002443 +12105 11.88191 -0.000004835 -0.000002441 +12106 11.88287 -0.000004832 -0.000002440 +12107 11.88384 -0.000004830 -0.000002439 +12108 11.88480 -0.000004828 -0.000002437 +12109 11.88577 -0.000004825 -0.000002436 +12110 11.88673 -0.000004823 -0.000002434 +12111 11.88770 -0.000004821 -0.000002433 +12112 11.88866 -0.000004818 -0.000002432 +12113 11.88963 -0.000004816 -0.000002430 +12114 11.89059 -0.000004813 -0.000002429 +12115 11.89156 -0.000004811 -0.000002428 +12116 11.89252 -0.000004809 -0.000002426 +12117 11.89349 -0.000004806 -0.000002425 +12118 11.89445 -0.000004804 -0.000002423 +12119 11.89542 -0.000004802 -0.000002422 +12120 11.89638 -0.000004799 -0.000002421 +12121 11.89735 -0.000004797 -0.000002419 +12122 11.89831 -0.000004795 -0.000002418 +12123 11.89928 -0.000004792 -0.000002417 +12124 11.90025 -0.000004790 -0.000002415 +12125 11.90121 -0.000004788 -0.000002414 +12126 11.90218 -0.000004785 -0.000002412 +12127 11.90314 -0.000004783 -0.000002411 +12128 11.90411 -0.000004781 -0.000002410 +12129 11.90507 -0.000004778 -0.000002408 +12130 11.90604 -0.000004776 -0.000002407 +12131 11.90700 -0.000004774 -0.000002406 +12132 11.90797 -0.000004772 -0.000002404 +12133 11.90893 -0.000004769 -0.000002403 +12134 11.90990 -0.000004767 -0.000002401 +12135 11.91086 -0.000004765 -0.000002400 +12136 11.91183 -0.000004762 -0.000002399 +12137 11.91279 -0.000004760 -0.000002397 +12138 11.91376 -0.000004758 -0.000002396 +12139 11.91472 -0.000004755 -0.000002395 +12140 11.91569 -0.000004753 -0.000002393 +12141 11.91665 -0.000004751 -0.000002392 +12142 11.91762 -0.000004748 -0.000002391 +12143 11.91858 -0.000004746 -0.000002389 +12144 11.91955 -0.000004744 -0.000002388 +12145 11.92051 -0.000004741 -0.000002387 +12146 11.92148 -0.000004739 -0.000002385 +12147 11.92244 -0.000004737 -0.000002384 +12148 11.92341 -0.000004735 -0.000002382 +12149 11.92437 -0.000004732 -0.000002381 +12150 11.92534 -0.000004730 -0.000002380 +12151 11.92630 -0.000004728 -0.000002378 +12152 11.92727 -0.000004725 -0.000002377 +12153 11.92823 -0.000004723 -0.000002376 +12154 11.92920 -0.000004721 -0.000002374 +12155 11.93016 -0.000004719 -0.000002373 +12156 11.93113 -0.000004716 -0.000002372 +12157 11.93209 -0.000004714 -0.000002370 +12158 11.93306 -0.000004712 -0.000002369 +12159 11.93402 -0.000004709 -0.000002368 +12160 11.93499 -0.000004707 -0.000002366 +12161 11.93595 -0.000004705 -0.000002365 +12162 11.93692 -0.000004703 -0.000002364 +12163 11.93788 -0.000004700 -0.000002362 +12164 11.93885 -0.000004698 -0.000002361 +12165 11.93981 -0.000004696 -0.000002360 +12166 11.94078 -0.000004693 -0.000002358 +12167 11.94174 -0.000004691 -0.000002357 +12168 11.94271 -0.000004689 -0.000002356 +12169 11.94367 -0.000004687 -0.000002354 +12170 11.94464 -0.000004684 -0.000002353 +12171 11.94560 -0.000004682 -0.000002352 +12172 11.94657 -0.000004680 -0.000002350 +12173 11.94753 -0.000004677 -0.000002349 +12174 11.94850 -0.000004675 -0.000002348 +12175 11.94946 -0.000004673 -0.000002346 +12176 11.95043 -0.000004671 -0.000002345 +12177 11.95139 -0.000004668 -0.000002344 +12178 11.95236 -0.000004666 -0.000002342 +12179 11.95332 -0.000004664 -0.000002341 +12180 11.95429 -0.000004662 -0.000002340 +12181 11.95525 -0.000004659 -0.000002338 +12182 11.95622 -0.000004657 -0.000002337 +12183 11.95718 -0.000004655 -0.000002336 +12184 11.95815 -0.000004653 -0.000002334 +12185 11.95911 -0.000004650 -0.000002333 +12186 11.96008 -0.000004648 -0.000002332 +12187 11.96104 -0.000004646 -0.000002331 +12188 11.96201 -0.000004644 -0.000002329 +12189 11.96297 -0.000004641 -0.000002328 +12190 11.96394 -0.000004639 -0.000002327 +12191 11.96490 -0.000004637 -0.000002325 +12192 11.96587 -0.000004635 -0.000002324 +12193 11.96683 -0.000004632 -0.000002323 +12194 11.96780 -0.000004630 -0.000002321 +12195 11.96876 -0.000004628 -0.000002320 +12196 11.96973 -0.000004626 -0.000002319 +12197 11.97069 -0.000004623 -0.000002317 +12198 11.97166 -0.000004621 -0.000002316 +12199 11.97262 -0.000004619 -0.000002315 +12200 11.97359 -0.000004617 -0.000002313 +12201 11.97455 -0.000004615 -0.000002312 +12202 11.97552 -0.000004612 -0.000002311 +12203 11.97648 -0.000004610 -0.000002310 +12204 11.97745 -0.000004608 -0.000002308 +12205 11.97841 -0.000004606 -0.000002307 +12206 11.97938 -0.000004603 -0.000002306 +12207 11.98034 -0.000004601 -0.000002304 +12208 11.98131 -0.000004599 -0.000002303 +12209 11.98227 -0.000004597 -0.000002302 +12210 11.98324 -0.000004594 -0.000002300 +12211 11.98420 -0.000004592 -0.000002299 +12212 11.98517 -0.000004590 -0.000002298 +12213 11.98613 -0.000004588 -0.000002297 +12214 11.98710 -0.000004586 -0.000002295 +12215 11.98806 -0.000004583 -0.000002294 +12216 11.98903 -0.000004581 -0.000002293 +12217 11.98999 -0.000004579 -0.000002291 +12218 11.99096 -0.000004577 -0.000002290 +12219 11.99192 -0.000004575 -0.000002289 +12220 11.99289 -0.000004572 -0.000002288 +12221 11.99385 -0.000004570 -0.000002286 +12222 11.99482 -0.000004568 -0.000002285 +12223 11.99578 -0.000004566 -0.000002284 +12224 11.99675 -0.000004564 -0.000002282 +12225 11.99771 -0.000004561 -0.000002281 +12226 11.99868 -0.000004559 -0.000002280 +12227 11.99964 -0.000004557 -0.000002279 +12228 12.00061 -0.000004555 -0.000002277 +12229 12.00158 -0.000004553 -0.000002276 +12230 12.00254 -0.000004550 -0.000002275 +12231 12.00351 -0.000004548 -0.000002273 +12232 12.00447 -0.000004546 -0.000002272 +12233 12.00544 -0.000004544 -0.000002271 +12234 12.00640 -0.000004542 -0.000002270 +12235 12.00737 -0.000004539 -0.000002268 +12236 12.00833 -0.000004537 -0.000002267 +12237 12.00930 -0.000004535 -0.000002266 +12238 12.01026 -0.000004533 -0.000002264 +12239 12.01123 -0.000004531 -0.000002263 +12240 12.01219 -0.000004528 -0.000002262 +12241 12.01316 -0.000004526 -0.000002261 +12242 12.01412 -0.000004524 -0.000002259 +12243 12.01509 -0.000004522 -0.000002258 +12244 12.01605 -0.000004520 -0.000002257 +12245 12.01702 -0.000004518 -0.000002256 +12246 12.01798 -0.000004515 -0.000002254 +12247 12.01895 -0.000004513 -0.000002253 +12248 12.01991 -0.000004511 -0.000002252 +12249 12.02088 -0.000004509 -0.000002251 +12250 12.02184 -0.000004507 -0.000002249 +12251 12.02281 -0.000004505 -0.000002248 +12252 12.02377 -0.000004502 -0.000002247 +12253 12.02474 -0.000004500 -0.000002245 +12254 12.02570 -0.000004498 -0.000002244 +12255 12.02667 -0.000004496 -0.000002243 +12256 12.02763 -0.000004494 -0.000002242 +12257 12.02860 -0.000004492 -0.000002240 +12258 12.02956 -0.000004489 -0.000002239 +12259 12.03053 -0.000004487 -0.000002238 +12260 12.03149 -0.000004485 -0.000002237 +12261 12.03246 -0.000004483 -0.000002235 +12262 12.03342 -0.000004481 -0.000002234 +12263 12.03439 -0.000004479 -0.000002233 +12264 12.03535 -0.000004476 -0.000002232 +12265 12.03632 -0.000004474 -0.000002230 +12266 12.03728 -0.000004472 -0.000002229 +12267 12.03825 -0.000004470 -0.000002228 +12268 12.03921 -0.000004468 -0.000002227 +12269 12.04018 -0.000004466 -0.000002225 +12270 12.04114 -0.000004464 -0.000002224 +12271 12.04211 -0.000004461 -0.000002223 +12272 12.04307 -0.000004459 -0.000002222 +12273 12.04404 -0.000004457 -0.000002220 +12274 12.04500 -0.000004455 -0.000002219 +12275 12.04597 -0.000004453 -0.000002218 +12276 12.04693 -0.000004451 -0.000002217 +12277 12.04790 -0.000004449 -0.000002215 +12278 12.04886 -0.000004446 -0.000002214 +12279 12.04983 -0.000004444 -0.000002213 +12280 12.05079 -0.000004442 -0.000002212 +12281 12.05176 -0.000004440 -0.000002210 +12282 12.05272 -0.000004438 -0.000002209 +12283 12.05369 -0.000004436 -0.000002208 +12284 12.05465 -0.000004434 -0.000002207 +12285 12.05562 -0.000004431 -0.000002206 +12286 12.05658 -0.000004429 -0.000002204 +12287 12.05755 -0.000004427 -0.000002203 +12288 12.05851 -0.000004425 -0.000002202 +12289 12.05948 -0.000004423 -0.000002201 +12290 12.06044 -0.000004421 -0.000002199 +12291 12.06141 -0.000004419 -0.000002198 +12292 12.06237 -0.000004417 -0.000002197 +12293 12.06334 -0.000004414 -0.000002196 +12294 12.06430 -0.000004412 -0.000002194 +12295 12.06527 -0.000004410 -0.000002193 +12296 12.06623 -0.000004408 -0.000002192 +12297 12.06720 -0.000004406 -0.000002191 +12298 12.06816 -0.000004404 -0.000002190 +12299 12.06913 -0.000004402 -0.000002188 +12300 12.07009 -0.000004400 -0.000002187 +12301 12.07106 -0.000004398 -0.000002186 +12302 12.07202 -0.000004395 -0.000002185 +12303 12.07299 -0.000004393 -0.000002183 +12304 12.07395 -0.000004391 -0.000002182 +12305 12.07492 -0.000004389 -0.000002181 +12306 12.07588 -0.000004387 -0.000002180 +12307 12.07685 -0.000004385 -0.000002179 +12308 12.07781 -0.000004383 -0.000002177 +12309 12.07878 -0.000004381 -0.000002176 +12310 12.07974 -0.000004379 -0.000002175 +12311 12.08071 -0.000004377 -0.000002174 +12312 12.08167 -0.000004374 -0.000002172 +12313 12.08264 -0.000004372 -0.000002171 +12314 12.08360 -0.000004370 -0.000002170 +12315 12.08457 -0.000004368 -0.000002169 +12316 12.08553 -0.000004366 -0.000002168 +12317 12.08650 -0.000004364 -0.000002166 +12318 12.08746 -0.000004362 -0.000002165 +12319 12.08843 -0.000004360 -0.000002164 +12320 12.08939 -0.000004358 -0.000002163 +12321 12.09036 -0.000004356 -0.000002162 +12322 12.09132 -0.000004354 -0.000002160 +12323 12.09229 -0.000004351 -0.000002159 +12324 12.09325 -0.000004349 -0.000002158 +12325 12.09422 -0.000004347 -0.000002157 +12326 12.09518 -0.000004345 -0.000002155 +12327 12.09615 -0.000004343 -0.000002154 +12328 12.09711 -0.000004341 -0.000002153 +12329 12.09808 -0.000004339 -0.000002152 +12330 12.09904 -0.000004337 -0.000002151 +12331 12.10001 -0.000004335 -0.000002149 +12332 12.10098 -0.000004333 -0.000002148 +12333 12.10194 -0.000004331 -0.000002147 +12334 12.10291 -0.000004329 -0.000002146 +12335 12.10387 -0.000004327 -0.000002145 +12336 12.10484 -0.000004324 -0.000002143 +12337 12.10580 -0.000004322 -0.000002142 +12338 12.10677 -0.000004320 -0.000002141 +12339 12.10773 -0.000004318 -0.000002140 +12340 12.10870 -0.000004316 -0.000002139 +12341 12.10966 -0.000004314 -0.000002138 +12342 12.11063 -0.000004312 -0.000002136 +12343 12.11159 -0.000004310 -0.000002135 +12344 12.11256 -0.000004308 -0.000002134 +12345 12.11352 -0.000004306 -0.000002133 +12346 12.11449 -0.000004304 -0.000002132 +12347 12.11545 -0.000004302 -0.000002130 +12348 12.11642 -0.000004300 -0.000002129 +12349 12.11738 -0.000004298 -0.000002128 +12350 12.11835 -0.000004296 -0.000002127 +12351 12.11931 -0.000004294 -0.000002126 +12352 12.12028 -0.000004291 -0.000002124 +12353 12.12124 -0.000004289 -0.000002123 +12354 12.12221 -0.000004287 -0.000002122 +12355 12.12317 -0.000004285 -0.000002121 +12356 12.12414 -0.000004283 -0.000002120 +12357 12.12510 -0.000004281 -0.000002119 +12358 12.12607 -0.000004279 -0.000002117 +12359 12.12703 -0.000004277 -0.000002116 +12360 12.12800 -0.000004275 -0.000002115 +12361 12.12896 -0.000004273 -0.000002114 +12362 12.12993 -0.000004271 -0.000002113 +12363 12.13089 -0.000004269 -0.000002111 +12364 12.13186 -0.000004267 -0.000002110 +12365 12.13282 -0.000004265 -0.000002109 +12366 12.13379 -0.000004263 -0.000002108 +12367 12.13475 -0.000004261 -0.000002107 +12368 12.13572 -0.000004259 -0.000002106 +12369 12.13668 -0.000004257 -0.000002104 +12370 12.13765 -0.000004255 -0.000002103 +12371 12.13861 -0.000004253 -0.000002102 +12372 12.13958 -0.000004251 -0.000002101 +12373 12.14054 -0.000004249 -0.000002100 +12374 12.14151 -0.000004247 -0.000002099 +12375 12.14247 -0.000004245 -0.000002097 +12376 12.14344 -0.000004243 -0.000002096 +12377 12.14440 -0.000004241 -0.000002095 +12378 12.14537 -0.000004239 -0.000002094 +12379 12.14633 -0.000004237 -0.000002093 +12380 12.14730 -0.000004235 -0.000002092 +12381 12.14826 -0.000004233 -0.000002090 +12382 12.14923 -0.000004230 -0.000002089 +12383 12.15019 -0.000004228 -0.000002088 +12384 12.15116 -0.000004226 -0.000002087 +12385 12.15212 -0.000004224 -0.000002086 +12386 12.15309 -0.000004222 -0.000002085 +12387 12.15405 -0.000004220 -0.000002083 +12388 12.15502 -0.000004218 -0.000002082 +12389 12.15598 -0.000004216 -0.000002081 +12390 12.15695 -0.000004214 -0.000002080 +12391 12.15791 -0.000004212 -0.000002079 +12392 12.15888 -0.000004210 -0.000002078 +12393 12.15984 -0.000004208 -0.000002077 +12394 12.16081 -0.000004206 -0.000002075 +12395 12.16177 -0.000004204 -0.000002074 +12396 12.16274 -0.000004202 -0.000002073 +12397 12.16370 -0.000004200 -0.000002072 +12398 12.16467 -0.000004198 -0.000002071 +12399 12.16563 -0.000004196 -0.000002070 +12400 12.16660 -0.000004194 -0.000002068 +12401 12.16756 -0.000004192 -0.000002067 +12402 12.16853 -0.000004190 -0.000002066 +12403 12.16949 -0.000004188 -0.000002065 +12404 12.17046 -0.000004186 -0.000002064 +12405 12.17142 -0.000004184 -0.000002063 +12406 12.17239 -0.000004182 -0.000002062 +12407 12.17335 -0.000004180 -0.000002060 +12408 12.17432 -0.000004178 -0.000002059 +12409 12.17528 -0.000004176 -0.000002058 +12410 12.17625 -0.000004174 -0.000002057 +12411 12.17721 -0.000004172 -0.000002056 +12412 12.17818 -0.000004171 -0.000002055 +12413 12.17914 -0.000004169 -0.000002054 +12414 12.18011 -0.000004167 -0.000002052 +12415 12.18107 -0.000004165 -0.000002051 +12416 12.18204 -0.000004163 -0.000002050 +12417 12.18300 -0.000004161 -0.000002049 +12418 12.18397 -0.000004159 -0.000002048 +12419 12.18493 -0.000004157 -0.000002047 +12420 12.18590 -0.000004155 -0.000002046 +12421 12.18686 -0.000004153 -0.000002045 +12422 12.18783 -0.000004151 -0.000002043 +12423 12.18879 -0.000004149 -0.000002042 +12424 12.18976 -0.000004147 -0.000002041 +12425 12.19072 -0.000004145 -0.000002040 +12426 12.19169 -0.000004143 -0.000002039 +12427 12.19265 -0.000004141 -0.000002038 +12428 12.19362 -0.000004139 -0.000002037 +12429 12.19458 -0.000004137 -0.000002035 +12430 12.19555 -0.000004135 -0.000002034 +12431 12.19651 -0.000004133 -0.000002033 +12432 12.19748 -0.000004131 -0.000002032 +12433 12.19844 -0.000004129 -0.000002031 +12434 12.19941 -0.000004127 -0.000002030 +12435 12.20038 -0.000004125 -0.000002029 +12436 12.20134 -0.000004123 -0.000002028 +12437 12.20231 -0.000004121 -0.000002026 +12438 12.20327 -0.000004119 -0.000002025 +12439 12.20424 -0.000004117 -0.000002024 +12440 12.20520 -0.000004115 -0.000002023 +12441 12.20617 -0.000004113 -0.000002022 +12442 12.20713 -0.000004112 -0.000002021 +12443 12.20810 -0.000004110 -0.000002020 +12444 12.20906 -0.000004108 -0.000002019 +12445 12.21003 -0.000004106 -0.000002018 +12446 12.21099 -0.000004104 -0.000002016 +12447 12.21196 -0.000004102 -0.000002015 +12448 12.21292 -0.000004100 -0.000002014 +12449 12.21389 -0.000004098 -0.000002013 +12450 12.21485 -0.000004096 -0.000002012 +12451 12.21582 -0.000004094 -0.000002011 +12452 12.21678 -0.000004092 -0.000002010 +12453 12.21775 -0.000004090 -0.000002009 +12454 12.21871 -0.000004088 -0.000002008 +12455 12.21968 -0.000004086 -0.000002006 +12456 12.22064 -0.000004084 -0.000002005 +12457 12.22161 -0.000004082 -0.000002004 +12458 12.22257 -0.000004080 -0.000002003 +12459 12.22354 -0.000004079 -0.000002002 +12460 12.22450 -0.000004077 -0.000002001 +12461 12.22547 -0.000004075 -0.000002000 +12462 12.22643 -0.000004073 -0.000001999 +12463 12.22740 -0.000004071 -0.000001998 +12464 12.22836 -0.000004069 -0.000001996 +12465 12.22933 -0.000004067 -0.000001995 +12466 12.23029 -0.000004065 -0.000001994 +12467 12.23126 -0.000004063 -0.000001993 +12468 12.23222 -0.000004061 -0.000001992 +12469 12.23319 -0.000004059 -0.000001991 +12470 12.23415 -0.000004057 -0.000001990 +12471 12.23512 -0.000004055 -0.000001989 +12472 12.23608 -0.000004053 -0.000001988 +12473 12.23705 -0.000004052 -0.000001987 +12474 12.23801 -0.000004050 -0.000001985 +12475 12.23898 -0.000004048 -0.000001984 +12476 12.23994 -0.000004046 -0.000001983 +12477 12.24091 -0.000004044 -0.000001982 +12478 12.24187 -0.000004042 -0.000001981 +12479 12.24284 -0.000004040 -0.000001980 +12480 12.24380 -0.000004038 -0.000001979 +12481 12.24477 -0.000004036 -0.000001978 +12482 12.24573 -0.000004034 -0.000001977 +12483 12.24670 -0.000004032 -0.000001976 +12484 12.24766 -0.000004031 -0.000001975 +12485 12.24863 -0.000004029 -0.000001973 +12486 12.24959 -0.000004027 -0.000001972 +12487 12.25056 -0.000004025 -0.000001971 +12488 12.25152 -0.000004023 -0.000001970 +12489 12.25249 -0.000004021 -0.000001969 +12490 12.25345 -0.000004019 -0.000001968 +12491 12.25442 -0.000004017 -0.000001967 +12492 12.25538 -0.000004015 -0.000001966 +12493 12.25635 -0.000004013 -0.000001965 +12494 12.25731 -0.000004012 -0.000001964 +12495 12.25828 -0.000004010 -0.000001963 +12496 12.25924 -0.000004008 -0.000001961 +12497 12.26021 -0.000004006 -0.000001960 +12498 12.26117 -0.000004004 -0.000001959 +12499 12.26214 -0.000004002 -0.000001958 +12500 12.26310 -0.000004000 -0.000001957 +12501 12.26407 -0.000003998 -0.000001956 +12502 12.26503 -0.000003996 -0.000001955 +12503 12.26600 -0.000003995 -0.000001954 +12504 12.26696 -0.000003993 -0.000001953 +12505 12.26793 -0.000003991 -0.000001952 +12506 12.26889 -0.000003989 -0.000001951 +12507 12.26986 -0.000003987 -0.000001950 +12508 12.27082 -0.000003985 -0.000001949 +12509 12.27179 -0.000003983 -0.000001948 +12510 12.27275 -0.000003981 -0.000001946 +12511 12.27372 -0.000003979 -0.000001945 +12512 12.27468 -0.000003978 -0.000001944 +12513 12.27565 -0.000003976 -0.000001943 +12514 12.27661 -0.000003974 -0.000001942 +12515 12.27758 -0.000003972 -0.000001941 +12516 12.27854 -0.000003970 -0.000001940 +12517 12.27951 -0.000003968 -0.000001939 +12518 12.28047 -0.000003966 -0.000001938 +12519 12.28144 -0.000003964 -0.000001937 +12520 12.28240 -0.000003963 -0.000001936 +12521 12.28337 -0.000003961 -0.000001935 +12522 12.28433 -0.000003959 -0.000001934 +12523 12.28530 -0.000003957 -0.000001933 +12524 12.28626 -0.000003955 -0.000001931 +12525 12.28723 -0.000003953 -0.000001930 +12526 12.28819 -0.000003951 -0.000001929 +12527 12.28916 -0.000003950 -0.000001928 +12528 12.29012 -0.000003948 -0.000001927 +12529 12.29109 -0.000003946 -0.000001926 +12530 12.29205 -0.000003944 -0.000001925 +12531 12.29302 -0.000003942 -0.000001924 +12532 12.29398 -0.000003940 -0.000001923 +12533 12.29495 -0.000003938 -0.000001922 +12534 12.29591 -0.000003937 -0.000001921 +12535 12.29688 -0.000003935 -0.000001920 +12536 12.29784 -0.000003933 -0.000001919 +12537 12.29881 -0.000003931 -0.000001918 +12538 12.29977 -0.000003929 -0.000001917 +12539 12.30074 -0.000003927 -0.000001916 +12540 12.30171 -0.000003925 -0.000001915 +12541 12.30267 -0.000003924 -0.000001914 +12542 12.30364 -0.000003922 -0.000001912 +12543 12.30460 -0.000003920 -0.000001911 +12544 12.30557 -0.000003918 -0.000001910 +12545 12.30653 -0.000003916 -0.000001909 +12546 12.30750 -0.000003914 -0.000001908 +12547 12.30846 -0.000003913 -0.000001907 +12548 12.30943 -0.000003911 -0.000001906 +12549 12.31039 -0.000003909 -0.000001905 +12550 12.31136 -0.000003907 -0.000001904 +12551 12.31232 -0.000003905 -0.000001903 +12552 12.31329 -0.000003903 -0.000001902 +12553 12.31425 -0.000003902 -0.000001901 +12554 12.31522 -0.000003900 -0.000001900 +12555 12.31618 -0.000003898 -0.000001899 +12556 12.31715 -0.000003896 -0.000001898 +12557 12.31811 -0.000003894 -0.000001897 +12558 12.31908 -0.000003892 -0.000001896 +12559 12.32004 -0.000003891 -0.000001895 +12560 12.32101 -0.000003889 -0.000001894 +12561 12.32197 -0.000003887 -0.000001893 +12562 12.32294 -0.000003885 -0.000001892 +12563 12.32390 -0.000003883 -0.000001891 +12564 12.32487 -0.000003881 -0.000001890 +12565 12.32583 -0.000003880 -0.000001889 +12566 12.32680 -0.000003878 -0.000001887 +12567 12.32776 -0.000003876 -0.000001886 +12568 12.32873 -0.000003874 -0.000001885 +12569 12.32969 -0.000003872 -0.000001884 +12570 12.33066 -0.000003870 -0.000001883 +12571 12.33162 -0.000003869 -0.000001882 +12572 12.33259 -0.000003867 -0.000001881 +12573 12.33355 -0.000003865 -0.000001880 +12574 12.33452 -0.000003863 -0.000001879 +12575 12.33548 -0.000003861 -0.000001878 +12576 12.33645 -0.000003860 -0.000001877 +12577 12.33741 -0.000003858 -0.000001876 +12578 12.33838 -0.000003856 -0.000001875 +12579 12.33934 -0.000003854 -0.000001874 +12580 12.34031 -0.000003852 -0.000001873 +12581 12.34127 -0.000003851 -0.000001872 +12582 12.34224 -0.000003849 -0.000001871 +12583 12.34320 -0.000003847 -0.000001870 +12584 12.34417 -0.000003845 -0.000001869 +12585 12.34513 -0.000003843 -0.000001868 +12586 12.34610 -0.000003842 -0.000001867 +12587 12.34706 -0.000003840 -0.000001866 +12588 12.34803 -0.000003838 -0.000001865 +12589 12.34899 -0.000003836 -0.000001864 +12590 12.34996 -0.000003834 -0.000001863 +12591 12.35092 -0.000003833 -0.000001862 +12592 12.35189 -0.000003831 -0.000001861 +12593 12.35285 -0.000003829 -0.000001860 +12594 12.35382 -0.000003827 -0.000001859 +12595 12.35478 -0.000003825 -0.000001858 +12596 12.35575 -0.000003824 -0.000001857 +12597 12.35671 -0.000003822 -0.000001856 +12598 12.35768 -0.000003820 -0.000001855 +12599 12.35864 -0.000003818 -0.000001854 +12600 12.35961 -0.000003816 -0.000001853 +12601 12.36057 -0.000003815 -0.000001852 +12602 12.36154 -0.000003813 -0.000001851 +12603 12.36250 -0.000003811 -0.000001850 +12604 12.36347 -0.000003809 -0.000001849 +12605 12.36443 -0.000003807 -0.000001848 +12606 12.36540 -0.000003806 -0.000001847 +12607 12.36636 -0.000003804 -0.000001846 +12608 12.36733 -0.000003802 -0.000001845 +12609 12.36829 -0.000003800 -0.000001844 +12610 12.36926 -0.000003799 -0.000001843 +12611 12.37022 -0.000003797 -0.000001842 +12612 12.37119 -0.000003795 -0.000001841 +12613 12.37215 -0.000003793 -0.000001840 +12614 12.37312 -0.000003791 -0.000001839 +12615 12.37408 -0.000003790 -0.000001838 +12616 12.37505 -0.000003788 -0.000001837 +12617 12.37601 -0.000003786 -0.000001836 +12618 12.37698 -0.000003784 -0.000001835 +12619 12.37794 -0.000003783 -0.000001834 +12620 12.37891 -0.000003781 -0.000001833 +12621 12.37987 -0.000003779 -0.000001832 +12622 12.38084 -0.000003777 -0.000001831 +12623 12.38180 -0.000003776 -0.000001830 +12624 12.38277 -0.000003774 -0.000001829 +12625 12.38373 -0.000003772 -0.000001828 +12626 12.38470 -0.000003770 -0.000001827 +12627 12.38566 -0.000003768 -0.000001826 +12628 12.38663 -0.000003767 -0.000001825 +12629 12.38759 -0.000003765 -0.000001824 +12630 12.38856 -0.000003763 -0.000001823 +12631 12.38952 -0.000003761 -0.000001822 +12632 12.39049 -0.000003760 -0.000001821 +12633 12.39145 -0.000003758 -0.000001820 +12634 12.39242 -0.000003756 -0.000001819 +12635 12.39338 -0.000003754 -0.000001818 +12636 12.39435 -0.000003753 -0.000001817 +12637 12.39531 -0.000003751 -0.000001816 +12638 12.39628 -0.000003749 -0.000001815 +12639 12.39724 -0.000003747 -0.000001814 +12640 12.39821 -0.000003746 -0.000001813 +12641 12.39917 -0.000003744 -0.000001812 +12642 12.40014 -0.000003742 -0.000001811 +12643 12.40111 -0.000003740 -0.000001810 +12644 12.40207 -0.000003739 -0.000001809 +12645 12.40304 -0.000003737 -0.000001808 +12646 12.40400 -0.000003735 -0.000001807 +12647 12.40497 -0.000003733 -0.000001806 +12648 12.40593 -0.000003732 -0.000001805 +12649 12.40690 -0.000003730 -0.000001804 +12650 12.40786 -0.000003728 -0.000001803 +12651 12.40883 -0.000003726 -0.000001802 +12652 12.40979 -0.000003725 -0.000001801 +12653 12.41076 -0.000003723 -0.000001800 +12654 12.41172 -0.000003721 -0.000001799 +12655 12.41269 -0.000003720 -0.000001798 +12656 12.41365 -0.000003718 -0.000001797 +12657 12.41462 -0.000003716 -0.000001796 +12658 12.41558 -0.000003714 -0.000001795 +12659 12.41655 -0.000003713 -0.000001794 +12660 12.41751 -0.000003711 -0.000001793 +12661 12.41848 -0.000003709 -0.000001792 +12662 12.41944 -0.000003707 -0.000001791 +12663 12.42041 -0.000003706 -0.000001790 +12664 12.42137 -0.000003704 -0.000001789 +12665 12.42234 -0.000003702 -0.000001788 +12666 12.42330 -0.000003700 -0.000001787 +12667 12.42427 -0.000003699 -0.000001786 +12668 12.42523 -0.000003697 -0.000001785 +12669 12.42620 -0.000003695 -0.000001784 +12670 12.42716 -0.000003694 -0.000001783 +12671 12.42813 -0.000003692 -0.000001782 +12672 12.42909 -0.000003690 -0.000001781 +12673 12.43006 -0.000003688 -0.000001780 +12674 12.43102 -0.000003687 -0.000001779 +12675 12.43199 -0.000003685 -0.000001778 +12676 12.43295 -0.000003683 -0.000001778 +12677 12.43392 -0.000003682 -0.000001777 +12678 12.43488 -0.000003680 -0.000001776 +12679 12.43585 -0.000003678 -0.000001775 +12680 12.43681 -0.000003676 -0.000001774 +12681 12.43778 -0.000003675 -0.000001773 +12682 12.43874 -0.000003673 -0.000001772 +12683 12.43971 -0.000003671 -0.000001771 +12684 12.44067 -0.000003670 -0.000001770 +12685 12.44164 -0.000003668 -0.000001769 +12686 12.44260 -0.000003666 -0.000001768 +12687 12.44357 -0.000003664 -0.000001767 +12688 12.44453 -0.000003663 -0.000001766 +12689 12.44550 -0.000003661 -0.000001765 +12690 12.44646 -0.000003659 -0.000001764 +12691 12.44743 -0.000003658 -0.000001763 +12692 12.44839 -0.000003656 -0.000001762 +12693 12.44936 -0.000003654 -0.000001761 +12694 12.45032 -0.000003653 -0.000001760 +12695 12.45129 -0.000003651 -0.000001759 +12696 12.45225 -0.000003649 -0.000001758 +12697 12.45322 -0.000003647 -0.000001757 +12698 12.45418 -0.000003646 -0.000001756 +12699 12.45515 -0.000003644 -0.000001755 +12700 12.45611 -0.000003642 -0.000001755 +12701 12.45708 -0.000003641 -0.000001754 +12702 12.45804 -0.000003639 -0.000001753 +12703 12.45901 -0.000003637 -0.000001752 +12704 12.45997 -0.000003636 -0.000001751 +12705 12.46094 -0.000003634 -0.000001750 +12706 12.46190 -0.000003632 -0.000001749 +12707 12.46287 -0.000003631 -0.000001748 +12708 12.46383 -0.000003629 -0.000001747 +12709 12.46480 -0.000003627 -0.000001746 +12710 12.46576 -0.000003626 -0.000001745 +12711 12.46673 -0.000003624 -0.000001744 +12712 12.46769 -0.000003622 -0.000001743 +12713 12.46866 -0.000003620 -0.000001742 +12714 12.46962 -0.000003619 -0.000001741 +12715 12.47059 -0.000003617 -0.000001740 +12716 12.47155 -0.000003615 -0.000001739 +12717 12.47252 -0.000003614 -0.000001738 +12718 12.47348 -0.000003612 -0.000001737 +12719 12.47445 -0.000003610 -0.000001737 +12720 12.47541 -0.000003609 -0.000001736 +12721 12.47638 -0.000003607 -0.000001735 +12722 12.47734 -0.000003605 -0.000001734 +12723 12.47831 -0.000003604 -0.000001733 +12724 12.47927 -0.000003602 -0.000001732 +12725 12.48024 -0.000003600 -0.000001731 +12726 12.48120 -0.000003599 -0.000001730 +12727 12.48217 -0.000003597 -0.000001729 +12728 12.48313 -0.000003595 -0.000001728 +12729 12.48410 -0.000003594 -0.000001727 +12730 12.48506 -0.000003592 -0.000001726 +12731 12.48603 -0.000003590 -0.000001725 +12732 12.48699 -0.000003589 -0.000001724 +12733 12.48796 -0.000003587 -0.000001723 +12734 12.48892 -0.000003585 -0.000001722 +12735 12.48989 -0.000003584 -0.000001722 +12736 12.49085 -0.000003582 -0.000001721 +12737 12.49182 -0.000003580 -0.000001720 +12738 12.49278 -0.000003579 -0.000001719 +12739 12.49375 -0.000003577 -0.000001718 +12740 12.49471 -0.000003575 -0.000001717 +12741 12.49568 -0.000003574 -0.000001716 +12742 12.49664 -0.000003572 -0.000001715 +12743 12.49761 -0.000003570 -0.000001714 +12744 12.49857 -0.000003569 -0.000001713 +12745 12.49954 -0.000003567 -0.000001712 +12746 12.50051 -0.000003565 -0.000001711 +12747 12.50147 -0.000003564 -0.000001710 +12748 12.50244 -0.000003562 -0.000001710 +12749 12.50340 -0.000003561 -0.000001709 +12750 12.50437 -0.000003559 -0.000001708 +12751 12.50533 -0.000003557 -0.000001707 +12752 12.50630 -0.000003556 -0.000001706 +12753 12.50726 -0.000003554 -0.000001705 +12754 12.50823 -0.000003552 -0.000001704 +12755 12.50919 -0.000003551 -0.000001703 +12756 12.51016 -0.000003549 -0.000001702 +12757 12.51112 -0.000003547 -0.000001701 +12758 12.51209 -0.000003546 -0.000001700 +12759 12.51305 -0.000003544 -0.000001699 +12760 12.51402 -0.000003542 -0.000001698 +12761 12.51498 -0.000003541 -0.000001698 +12762 12.51595 -0.000003539 -0.000001697 +12763 12.51691 -0.000003538 -0.000001696 +12764 12.51788 -0.000003536 -0.000001695 +12765 12.51884 -0.000003534 -0.000001694 +12766 12.51981 -0.000003533 -0.000001693 +12767 12.52077 -0.000003531 -0.000001692 +12768 12.52174 -0.000003529 -0.000001691 +12769 12.52270 -0.000003528 -0.000001690 +12770 12.52367 -0.000003526 -0.000001689 +12771 12.52463 -0.000003524 -0.000001688 +12772 12.52560 -0.000003523 -0.000001688 +12773 12.52656 -0.000003521 -0.000001687 +12774 12.52753 -0.000003520 -0.000001686 +12775 12.52849 -0.000003518 -0.000001685 +12776 12.52946 -0.000003516 -0.000001684 +12777 12.53042 -0.000003515 -0.000001683 +12778 12.53139 -0.000003513 -0.000001682 +12779 12.53235 -0.000003511 -0.000001681 +12780 12.53332 -0.000003510 -0.000001680 +12781 12.53428 -0.000003508 -0.000001679 +12782 12.53525 -0.000003507 -0.000001678 +12783 12.53621 -0.000003505 -0.000001678 +12784 12.53718 -0.000003503 -0.000001677 +12785 12.53814 -0.000003502 -0.000001676 +12786 12.53911 -0.000003500 -0.000001675 +12787 12.54007 -0.000003499 -0.000001674 +12788 12.54104 -0.000003497 -0.000001673 +12789 12.54200 -0.000003495 -0.000001672 +12790 12.54297 -0.000003494 -0.000001671 +12791 12.54393 -0.000003492 -0.000001670 +12792 12.54490 -0.000003490 -0.000001669 +12793 12.54586 -0.000003489 -0.000001669 +12794 12.54683 -0.000003487 -0.000001668 +12795 12.54779 -0.000003486 -0.000001667 +12796 12.54876 -0.000003484 -0.000001666 +12797 12.54972 -0.000003482 -0.000001665 +12798 12.55069 -0.000003481 -0.000001664 +12799 12.55165 -0.000003479 -0.000001663 +12800 12.55262 -0.000003478 -0.000001662 +12801 12.55358 -0.000003476 -0.000001661 +12802 12.55455 -0.000003474 -0.000001660 +12803 12.55551 -0.000003473 -0.000001660 +12804 12.55648 -0.000003471 -0.000001659 +12805 12.55744 -0.000003470 -0.000001658 +12806 12.55841 -0.000003468 -0.000001657 +12807 12.55937 -0.000003466 -0.000001656 +12808 12.56034 -0.000003465 -0.000001655 +12809 12.56130 -0.000003463 -0.000001654 +12810 12.56227 -0.000003462 -0.000001653 +12811 12.56323 -0.000003460 -0.000001652 +12812 12.56420 -0.000003458 -0.000001652 +12813 12.56516 -0.000003457 -0.000001651 +12814 12.56613 -0.000003455 -0.000001650 +12815 12.56709 -0.000003454 -0.000001649 +12816 12.56806 -0.000003452 -0.000001648 +12817 12.56902 -0.000003450 -0.000001647 +12818 12.56999 -0.000003449 -0.000001646 +12819 12.57095 -0.000003447 -0.000001645 +12820 12.57192 -0.000003446 -0.000001644 +12821 12.57288 -0.000003444 -0.000001644 +12822 12.57385 -0.000003442 -0.000001643 +12823 12.57481 -0.000003441 -0.000001642 +12824 12.57578 -0.000003439 -0.000001641 +12825 12.57674 -0.000003438 -0.000001640 +12826 12.57771 -0.000003436 -0.000001639 +12827 12.57867 -0.000003435 -0.000001638 +12828 12.57964 -0.000003433 -0.000001637 +12829 12.58060 -0.000003431 -0.000001637 +12830 12.58157 -0.000003430 -0.000001636 +12831 12.58253 -0.000003428 -0.000001635 +12832 12.58350 -0.000003427 -0.000001634 +12833 12.58446 -0.000003425 -0.000001633 +12834 12.58543 -0.000003424 -0.000001632 +12835 12.58639 -0.000003422 -0.000001631 +12836 12.58736 -0.000003420 -0.000001630 +12837 12.58832 -0.000003419 -0.000001630 +12838 12.58929 -0.000003417 -0.000001629 +12839 12.59025 -0.000003416 -0.000001628 +12840 12.59122 -0.000003414 -0.000001627 +12841 12.59218 -0.000003413 -0.000001626 +12842 12.59315 -0.000003411 -0.000001625 +12843 12.59411 -0.000003409 -0.000001624 +12844 12.59508 -0.000003408 -0.000001623 +12845 12.59604 -0.000003406 -0.000001623 +12846 12.59701 -0.000003405 -0.000001622 +12847 12.59797 -0.000003403 -0.000001621 +12848 12.59894 -0.000003402 -0.000001620 +12849 12.59990 -0.000003400 -0.000001619 +12850 12.60087 -0.000003398 -0.000001618 +12851 12.60184 -0.000003397 -0.000001617 +12852 12.60280 -0.000003395 -0.000001616 +12853 12.60377 -0.000003394 -0.000001616 +12854 12.60473 -0.000003392 -0.000001615 +12855 12.60570 -0.000003391 -0.000001614 +12856 12.60666 -0.000003389 -0.000001613 +12857 12.60763 -0.000003388 -0.000001612 +12858 12.60859 -0.000003386 -0.000001611 +12859 12.60956 -0.000003384 -0.000001610 +12860 12.61052 -0.000003383 -0.000001610 +12861 12.61149 -0.000003381 -0.000001609 +12862 12.61245 -0.000003380 -0.000001608 +12863 12.61342 -0.000003378 -0.000001607 +12864 12.61438 -0.000003377 -0.000001606 +12865 12.61535 -0.000003375 -0.000001605 +12866 12.61631 -0.000003374 -0.000001604 +12867 12.61728 -0.000003372 -0.000001604 +12868 12.61824 -0.000003370 -0.000001603 +12869 12.61921 -0.000003369 -0.000001602 +12870 12.62017 -0.000003367 -0.000001601 +12871 12.62114 -0.000003366 -0.000001600 +12872 12.62210 -0.000003364 -0.000001599 +12873 12.62307 -0.000003363 -0.000001598 +12874 12.62403 -0.000003361 -0.000001598 +12875 12.62500 -0.000003360 -0.000001597 +12876 12.62596 -0.000003358 -0.000001596 +12877 12.62693 -0.000003357 -0.000001595 +12878 12.62789 -0.000003355 -0.000001594 +12879 12.62886 -0.000003353 -0.000001593 +12880 12.62982 -0.000003352 -0.000001592 +12881 12.63079 -0.000003350 -0.000001592 +12882 12.63175 -0.000003349 -0.000001591 +12883 12.63272 -0.000003347 -0.000001590 +12884 12.63368 -0.000003346 -0.000001589 +12885 12.63465 -0.000003344 -0.000001588 +12886 12.63561 -0.000003343 -0.000001587 +12887 12.63658 -0.000003341 -0.000001586 +12888 12.63754 -0.000003340 -0.000001586 +12889 12.63851 -0.000003338 -0.000001585 +12890 12.63947 -0.000003337 -0.000001584 +12891 12.64044 -0.000003335 -0.000001583 +12892 12.64140 -0.000003334 -0.000001582 +12893 12.64237 -0.000003332 -0.000001581 +12894 12.64333 -0.000003331 -0.000001581 +12895 12.64430 -0.000003329 -0.000001580 +12896 12.64526 -0.000003327 -0.000001579 +12897 12.64623 -0.000003326 -0.000001578 +12898 12.64719 -0.000003324 -0.000001577 +12899 12.64816 -0.000003323 -0.000001576 +12900 12.64912 -0.000003321 -0.000001575 +12901 12.65009 -0.000003320 -0.000001575 +12902 12.65105 -0.000003318 -0.000001574 +12903 12.65202 -0.000003317 -0.000001573 +12904 12.65298 -0.000003315 -0.000001572 +12905 12.65395 -0.000003314 -0.000001571 +12906 12.65491 -0.000003312 -0.000001570 +12907 12.65588 -0.000003311 -0.000001570 +12908 12.65684 -0.000003309 -0.000001569 +12909 12.65781 -0.000003308 -0.000001568 +12910 12.65877 -0.000003306 -0.000001567 +12911 12.65974 -0.000003305 -0.000001566 +12912 12.66070 -0.000003303 -0.000001565 +12913 12.66167 -0.000003302 -0.000001565 +12914 12.66263 -0.000003300 -0.000001564 +12915 12.66360 -0.000003299 -0.000001563 +12916 12.66456 -0.000003297 -0.000001562 +12917 12.66553 -0.000003296 -0.000001561 +12918 12.66649 -0.000003294 -0.000001560 +12919 12.66746 -0.000003293 -0.000001560 +12920 12.66842 -0.000003291 -0.000001559 +12921 12.66939 -0.000003290 -0.000001558 +12922 12.67035 -0.000003288 -0.000001557 +12923 12.67132 -0.000003287 -0.000001556 +12924 12.67228 -0.000003285 -0.000001555 +12925 12.67325 -0.000003284 -0.000001555 +12926 12.67421 -0.000003282 -0.000001554 +12927 12.67518 -0.000003281 -0.000001553 +12928 12.67614 -0.000003279 -0.000001552 +12929 12.67711 -0.000003278 -0.000001551 +12930 12.67807 -0.000003276 -0.000001550 +12931 12.67904 -0.000003275 -0.000001550 +12932 12.68000 -0.000003273 -0.000001549 +12933 12.68097 -0.000003272 -0.000001548 +12934 12.68193 -0.000003270 -0.000001547 +12935 12.68290 -0.000003269 -0.000001546 +12936 12.68386 -0.000003267 -0.000001546 +12937 12.68483 -0.000003266 -0.000001545 +12938 12.68579 -0.000003264 -0.000001544 +12939 12.68676 -0.000003263 -0.000001543 +12940 12.68772 -0.000003261 -0.000001542 +12941 12.68869 -0.000003260 -0.000001541 +12942 12.68965 -0.000003258 -0.000001541 +12943 12.69062 -0.000003257 -0.000001540 +12944 12.69158 -0.000003255 -0.000001539 +12945 12.69255 -0.000003254 -0.000001538 +12946 12.69351 -0.000003252 -0.000001537 +12947 12.69448 -0.000003251 -0.000001536 +12948 12.69544 -0.000003249 -0.000001536 +12949 12.69641 -0.000003248 -0.000001535 +12950 12.69737 -0.000003246 -0.000001534 +12951 12.69834 -0.000003245 -0.000001533 +12952 12.69930 -0.000003243 -0.000001532 +12953 12.70027 -0.000003242 -0.000001532 +12954 12.70124 -0.000003240 -0.000001531 +12955 12.70220 -0.000003239 -0.000001530 +12956 12.70317 -0.000003238 -0.000001529 +12957 12.70413 -0.000003236 -0.000001528 +12958 12.70510 -0.000003235 -0.000001528 +12959 12.70606 -0.000003233 -0.000001527 +12960 12.70703 -0.000003232 -0.000001526 +12961 12.70799 -0.000003230 -0.000001525 +12962 12.70896 -0.000003229 -0.000001524 +12963 12.70992 -0.000003227 -0.000001523 +12964 12.71089 -0.000003226 -0.000001523 +12965 12.71185 -0.000003224 -0.000001522 +12966 12.71282 -0.000003223 -0.000001521 +12967 12.71378 -0.000003221 -0.000001520 +12968 12.71475 -0.000003220 -0.000001519 +12969 12.71571 -0.000003218 -0.000001519 +12970 12.71668 -0.000003217 -0.000001518 +12971 12.71764 -0.000003215 -0.000001517 +12972 12.71861 -0.000003214 -0.000001516 +12973 12.71957 -0.000003213 -0.000001515 +12974 12.72054 -0.000003211 -0.000001515 +12975 12.72150 -0.000003210 -0.000001514 +12976 12.72247 -0.000003208 -0.000001513 +12977 12.72343 -0.000003207 -0.000001512 +12978 12.72440 -0.000003205 -0.000001511 +12979 12.72536 -0.000003204 -0.000001511 +12980 12.72633 -0.000003202 -0.000001510 +12981 12.72729 -0.000003201 -0.000001509 +12982 12.72826 -0.000003199 -0.000001508 +12983 12.72922 -0.000003198 -0.000001507 +12984 12.73019 -0.000003196 -0.000001507 +12985 12.73115 -0.000003195 -0.000001506 +12986 12.73212 -0.000003194 -0.000001505 +12987 12.73308 -0.000003192 -0.000001504 +12988 12.73405 -0.000003191 -0.000001503 +12989 12.73501 -0.000003189 -0.000001503 +12990 12.73598 -0.000003188 -0.000001502 +12991 12.73694 -0.000003186 -0.000001501 +12992 12.73791 -0.000003185 -0.000001500 +12993 12.73887 -0.000003183 -0.000001499 +12994 12.73984 -0.000003182 -0.000001499 +12995 12.74080 -0.000003181 -0.000001498 +12996 12.74177 -0.000003179 -0.000001497 +12997 12.74273 -0.000003178 -0.000001496 +12998 12.74370 -0.000003176 -0.000001495 +12999 12.74466 -0.000003175 -0.000001495 +13000 12.74563 -0.000003173 -0.000001494 +13001 12.74659 -0.000003172 -0.000001493 +13002 12.74756 -0.000003170 -0.000001492 +13003 12.74852 -0.000003169 -0.000001491 +13004 12.74949 -0.000003168 -0.000001491 +13005 12.75045 -0.000003166 -0.000001490 +13006 12.75142 -0.000003165 -0.000001489 +13007 12.75238 -0.000003163 -0.000001488 +13008 12.75335 -0.000003162 -0.000001488 +13009 12.75431 -0.000003160 -0.000001487 +13010 12.75528 -0.000003159 -0.000001486 +13011 12.75624 -0.000003158 -0.000001485 +13012 12.75721 -0.000003156 -0.000001484 +13013 12.75817 -0.000003155 -0.000001484 +13014 12.75914 -0.000003153 -0.000001483 +13015 12.76010 -0.000003152 -0.000001482 +13016 12.76107 -0.000003150 -0.000001481 +13017 12.76203 -0.000003149 -0.000001480 +13018 12.76300 -0.000003148 -0.000001480 +13019 12.76396 -0.000003146 -0.000001479 +13020 12.76493 -0.000003145 -0.000001478 +13021 12.76589 -0.000003143 -0.000001477 +13022 12.76686 -0.000003142 -0.000001477 +13023 12.76782 -0.000003140 -0.000001476 +13024 12.76879 -0.000003139 -0.000001475 +13025 12.76975 -0.000003138 -0.000001474 +13026 12.77072 -0.000003136 -0.000001473 +13027 12.77168 -0.000003135 -0.000001473 +13028 12.77265 -0.000003133 -0.000001472 +13029 12.77361 -0.000003132 -0.000001471 +13030 12.77458 -0.000003130 -0.000001470 +13031 12.77554 -0.000003129 -0.000001470 +13032 12.77651 -0.000003128 -0.000001469 +13033 12.77747 -0.000003126 -0.000001468 +13034 12.77844 -0.000003125 -0.000001467 +13035 12.77940 -0.000003123 -0.000001466 +13036 12.78037 -0.000003122 -0.000001466 +13037 12.78133 -0.000003121 -0.000001465 +13038 12.78230 -0.000003119 -0.000001464 +13039 12.78326 -0.000003118 -0.000001463 +13040 12.78423 -0.000003116 -0.000001463 +13041 12.78519 -0.000003115 -0.000001462 +13042 12.78616 -0.000003113 -0.000001461 +13043 12.78712 -0.000003112 -0.000001460 +13044 12.78809 -0.000003111 -0.000001459 +13045 12.78905 -0.000003109 -0.000001459 +13046 12.79002 -0.000003108 -0.000001458 +13047 12.79098 -0.000003106 -0.000001457 +13048 12.79195 -0.000003105 -0.000001456 +13049 12.79291 -0.000003104 -0.000001456 +13050 12.79388 -0.000003102 -0.000001455 +13051 12.79484 -0.000003101 -0.000001454 +13052 12.79581 -0.000003099 -0.000001453 +13053 12.79677 -0.000003098 -0.000001453 +13054 12.79774 -0.000003097 -0.000001452 +13055 12.79870 -0.000003095 -0.000001451 +13056 12.79967 -0.000003094 -0.000001450 +13057 12.80064 -0.000003092 -0.000001449 +13058 12.80160 -0.000003091 -0.000001449 +13059 12.80257 -0.000003090 -0.000001448 +13060 12.80353 -0.000003088 -0.000001447 +13061 12.80450 -0.000003087 -0.000001446 +13062 12.80546 -0.000003085 -0.000001446 +13063 12.80643 -0.000003084 -0.000001445 +13064 12.80739 -0.000003083 -0.000001444 +13065 12.80836 -0.000003081 -0.000001443 +13066 12.80932 -0.000003080 -0.000001443 +13067 12.81029 -0.000003078 -0.000001442 +13068 12.81125 -0.000003077 -0.000001441 +13069 12.81222 -0.000003076 -0.000001440 +13070 12.81318 -0.000003074 -0.000001440 +13071 12.81415 -0.000003073 -0.000001439 +13072 12.81511 -0.000003071 -0.000001438 +13073 12.81608 -0.000003070 -0.000001437 +13074 12.81704 -0.000003069 -0.000001437 +13075 12.81801 -0.000003067 -0.000001436 +13076 12.81897 -0.000003066 -0.000001435 +13077 12.81994 -0.000003065 -0.000001434 +13078 12.82090 -0.000003063 -0.000001434 +13079 12.82187 -0.000003062 -0.000001433 +13080 12.82283 -0.000003060 -0.000001432 +13081 12.82380 -0.000003059 -0.000001431 +13082 12.82476 -0.000003058 -0.000001431 +13083 12.82573 -0.000003056 -0.000001430 +13084 12.82669 -0.000003055 -0.000001429 +13085 12.82766 -0.000003054 -0.000001428 +13086 12.82862 -0.000003052 -0.000001427 +13087 12.82959 -0.000003051 -0.000001427 +13088 12.83055 -0.000003049 -0.000001426 +13089 12.83152 -0.000003048 -0.000001425 +13090 12.83248 -0.000003047 -0.000001424 +13091 12.83345 -0.000003045 -0.000001424 +13092 12.83441 -0.000003044 -0.000001423 +13093 12.83538 -0.000003043 -0.000001422 +13094 12.83634 -0.000003041 -0.000001421 +13095 12.83731 -0.000003040 -0.000001421 +13096 12.83827 -0.000003038 -0.000001420 +13097 12.83924 -0.000003037 -0.000001419 +13098 12.84020 -0.000003036 -0.000001419 +13099 12.84117 -0.000003034 -0.000001418 +13100 12.84213 -0.000003033 -0.000001417 +13101 12.84310 -0.000003032 -0.000001416 +13102 12.84406 -0.000003030 -0.000001416 +13103 12.84503 -0.000003029 -0.000001415 +13104 12.84599 -0.000003027 -0.000001414 +13105 12.84696 -0.000003026 -0.000001413 +13106 12.84792 -0.000003025 -0.000001413 +13107 12.84889 -0.000003023 -0.000001412 +13108 12.84985 -0.000003022 -0.000001411 +13109 12.85082 -0.000003021 -0.000001410 +13110 12.85178 -0.000003019 -0.000001410 +13111 12.85275 -0.000003018 -0.000001409 +13112 12.85371 -0.000003017 -0.000001408 +13113 12.85468 -0.000003015 -0.000001407 +13114 12.85564 -0.000003014 -0.000001407 +13115 12.85661 -0.000003012 -0.000001406 +13116 12.85757 -0.000003011 -0.000001405 +13117 12.85854 -0.000003010 -0.000001404 +13118 12.85950 -0.000003008 -0.000001404 +13119 12.86047 -0.000003007 -0.000001403 +13120 12.86143 -0.000003006 -0.000001402 +13121 12.86240 -0.000003004 -0.000001401 +13122 12.86336 -0.000003003 -0.000001401 +13123 12.86433 -0.000003002 -0.000001400 +13124 12.86529 -0.000003000 -0.000001399 +13125 12.86626 -0.000002999 -0.000001399 +13126 12.86722 -0.000002998 -0.000001398 +13127 12.86819 -0.000002996 -0.000001397 +13128 12.86915 -0.000002995 -0.000001396 +13129 12.87012 -0.000002994 -0.000001396 +13130 12.87108 -0.000002992 -0.000001395 +13131 12.87205 -0.000002991 -0.000001394 +13132 12.87301 -0.000002990 -0.000001393 +13133 12.87398 -0.000002988 -0.000001393 +13134 12.87494 -0.000002987 -0.000001392 +13135 12.87591 -0.000002985 -0.000001391 +13136 12.87687 -0.000002984 -0.000001390 +13137 12.87784 -0.000002983 -0.000001390 +13138 12.87880 -0.000002981 -0.000001389 +13139 12.87977 -0.000002980 -0.000001388 +13140 12.88073 -0.000002979 -0.000001388 +13141 12.88170 -0.000002977 -0.000001387 +13142 12.88266 -0.000002976 -0.000001386 +13143 12.88363 -0.000002975 -0.000001385 +13144 12.88459 -0.000002973 -0.000001385 +13145 12.88556 -0.000002972 -0.000001384 +13146 12.88652 -0.000002971 -0.000001383 +13147 12.88749 -0.000002969 -0.000001382 +13148 12.88845 -0.000002968 -0.000001382 +13149 12.88942 -0.000002967 -0.000001381 +13150 12.89038 -0.000002965 -0.000001380 +13151 12.89135 -0.000002964 -0.000001380 +13152 12.89231 -0.000002963 -0.000001379 +13153 12.89328 -0.000002961 -0.000001378 +13154 12.89424 -0.000002960 -0.000001377 +13155 12.89521 -0.000002959 -0.000001377 +13156 12.89617 -0.000002957 -0.000001376 +13157 12.89714 -0.000002956 -0.000001375 +13158 12.89810 -0.000002955 -0.000001375 +13159 12.89907 -0.000002953 -0.000001374 +13160 12.90004 -0.000002952 -0.000001373 +13161 12.90100 -0.000002951 -0.000001372 +13162 12.90197 -0.000002949 -0.000001372 +13163 12.90293 -0.000002948 -0.000001371 +13164 12.90390 -0.000002947 -0.000001370 +13165 12.90486 -0.000002946 -0.000001369 +13166 12.90583 -0.000002944 -0.000001369 +13167 12.90679 -0.000002943 -0.000001368 +13168 12.90776 -0.000002942 -0.000001367 +13169 12.90872 -0.000002940 -0.000001367 +13170 12.90969 -0.000002939 -0.000001366 +13171 12.91065 -0.000002938 -0.000001365 +13172 12.91162 -0.000002936 -0.000001364 +13173 12.91258 -0.000002935 -0.000001364 +13174 12.91355 -0.000002934 -0.000001363 +13175 12.91451 -0.000002932 -0.000001362 +13176 12.91548 -0.000002931 -0.000001362 +13177 12.91644 -0.000002930 -0.000001361 +13178 12.91741 -0.000002928 -0.000001360 +13179 12.91837 -0.000002927 -0.000001360 +13180 12.91934 -0.000002926 -0.000001359 +13181 12.92030 -0.000002924 -0.000001358 +13182 12.92127 -0.000002923 -0.000001357 +13183 12.92223 -0.000002922 -0.000001357 +13184 12.92320 -0.000002921 -0.000001356 +13185 12.92416 -0.000002919 -0.000001355 +13186 12.92513 -0.000002918 -0.000001355 +13187 12.92609 -0.000002917 -0.000001354 +13188 12.92706 -0.000002915 -0.000001353 +13189 12.92802 -0.000002914 -0.000001352 +13190 12.92899 -0.000002913 -0.000001352 +13191 12.92995 -0.000002911 -0.000001351 +13192 12.93092 -0.000002910 -0.000001350 +13193 12.93188 -0.000002909 -0.000001350 +13194 12.93285 -0.000002907 -0.000001349 +13195 12.93381 -0.000002906 -0.000001348 +13196 12.93478 -0.000002905 -0.000001347 +13197 12.93574 -0.000002904 -0.000001347 +13198 12.93671 -0.000002902 -0.000001346 +13199 12.93767 -0.000002901 -0.000001345 +13200 12.93864 -0.000002900 -0.000001345 +13201 12.93960 -0.000002898 -0.000001344 +13202 12.94057 -0.000002897 -0.000001343 +13203 12.94153 -0.000002896 -0.000001343 +13204 12.94250 -0.000002895 -0.000001342 +13205 12.94346 -0.000002893 -0.000001341 +13206 12.94443 -0.000002892 -0.000001340 +13207 12.94539 -0.000002891 -0.000001340 +13208 12.94636 -0.000002889 -0.000001339 +13209 12.94732 -0.000002888 -0.000001338 +13210 12.94829 -0.000002887 -0.000001338 +13211 12.94925 -0.000002885 -0.000001337 +13212 12.95022 -0.000002884 -0.000001336 +13213 12.95118 -0.000002883 -0.000001336 +13214 12.95215 -0.000002882 -0.000001335 +13215 12.95311 -0.000002880 -0.000001334 +13216 12.95408 -0.000002879 -0.000001333 +13217 12.95504 -0.000002878 -0.000001333 +13218 12.95601 -0.000002876 -0.000001332 +13219 12.95697 -0.000002875 -0.000001331 +13220 12.95794 -0.000002874 -0.000001331 +13221 12.95890 -0.000002873 -0.000001330 +13222 12.95987 -0.000002871 -0.000001329 +13223 12.96083 -0.000002870 -0.000001329 +13224 12.96180 -0.000002869 -0.000001328 +13225 12.96276 -0.000002867 -0.000001327 +13226 12.96373 -0.000002866 -0.000001327 +13227 12.96469 -0.000002865 -0.000001326 +13228 12.96566 -0.000002864 -0.000001325 +13229 12.96662 -0.000002862 -0.000001324 +13230 12.96759 -0.000002861 -0.000001324 +13231 12.96855 -0.000002860 -0.000001323 +13232 12.96952 -0.000002859 -0.000001322 +13233 12.97048 -0.000002857 -0.000001322 +13234 12.97145 -0.000002856 -0.000001321 +13235 12.97241 -0.000002855 -0.000001320 +13236 12.97338 -0.000002853 -0.000001320 +13237 12.97434 -0.000002852 -0.000001319 +13238 12.97531 -0.000002851 -0.000001318 +13239 12.97627 -0.000002850 -0.000001318 +13240 12.97724 -0.000002848 -0.000001317 +13241 12.97820 -0.000002847 -0.000001316 +13242 12.97917 -0.000002846 -0.000001316 +13243 12.98013 -0.000002845 -0.000001315 +13244 12.98110 -0.000002843 -0.000001314 +13245 12.98206 -0.000002842 -0.000001313 +13246 12.98303 -0.000002841 -0.000001313 +13247 12.98399 -0.000002839 -0.000001312 +13248 12.98496 -0.000002838 -0.000001311 +13249 12.98592 -0.000002837 -0.000001311 +13250 12.98689 -0.000002836 -0.000001310 +13251 12.98785 -0.000002834 -0.000001309 +13252 12.98882 -0.000002833 -0.000001309 +13253 12.98978 -0.000002832 -0.000001308 +13254 12.99075 -0.000002831 -0.000001307 +13255 12.99171 -0.000002829 -0.000001307 +13256 12.99268 -0.000002828 -0.000001306 +13257 12.99364 -0.000002827 -0.000001305 +13258 12.99461 -0.000002826 -0.000001305 +13259 12.99557 -0.000002824 -0.000001304 +13260 12.99654 -0.000002823 -0.000001303 +13261 12.99750 -0.000002822 -0.000001303 +13262 12.99847 -0.000002821 -0.000001302 +13263 12.99943 -0.000002819 -0.000001301 +13264 13.00040 -0.000002818 -0.000001301 +13265 13.00137 -0.000002817 -0.000001300 +13266 13.00233 -0.000002815 -0.000001299 +13267 13.00330 -0.000002814 -0.000001299 +13268 13.00426 -0.000002813 -0.000001298 +13269 13.00523 -0.000002812 -0.000001297 +13270 13.00619 -0.000002810 -0.000001297 +13271 13.00716 -0.000002809 -0.000001296 +13272 13.00812 -0.000002808 -0.000001295 +13273 13.00909 -0.000002807 -0.000001295 +13274 13.01005 -0.000002805 -0.000001294 +13275 13.01102 -0.000002804 -0.000001293 +13276 13.01198 -0.000002803 -0.000001292 +13277 13.01295 -0.000002802 -0.000001292 +13278 13.01391 -0.000002800 -0.000001291 +13279 13.01488 -0.000002799 -0.000001290 +13280 13.01584 -0.000002798 -0.000001290 +13281 13.01681 -0.000002797 -0.000001289 +13282 13.01777 -0.000002796 -0.000001288 +13283 13.01874 -0.000002794 -0.000001288 +13284 13.01970 -0.000002793 -0.000001287 +13285 13.02067 -0.000002792 -0.000001286 +13286 13.02163 -0.000002791 -0.000001286 +13287 13.02260 -0.000002789 -0.000001285 +13288 13.02356 -0.000002788 -0.000001284 +13289 13.02453 -0.000002787 -0.000001284 +13290 13.02549 -0.000002786 -0.000001283 +13291 13.02646 -0.000002784 -0.000001282 +13292 13.02742 -0.000002783 -0.000001282 +13293 13.02839 -0.000002782 -0.000001281 +13294 13.02935 -0.000002781 -0.000001280 +13295 13.03032 -0.000002779 -0.000001280 +13296 13.03128 -0.000002778 -0.000001279 +13297 13.03225 -0.000002777 -0.000001278 +13298 13.03321 -0.000002776 -0.000001278 +13299 13.03418 -0.000002774 -0.000001277 +13300 13.03514 -0.000002773 -0.000001277 +13301 13.03611 -0.000002772 -0.000001276 +13302 13.03707 -0.000002771 -0.000001275 +13303 13.03804 -0.000002770 -0.000001275 +13304 13.03900 -0.000002768 -0.000001274 +13305 13.03997 -0.000002767 -0.000001273 +13306 13.04093 -0.000002766 -0.000001273 +13307 13.04190 -0.000002765 -0.000001272 +13308 13.04286 -0.000002763 -0.000001271 +13309 13.04383 -0.000002762 -0.000001271 +13310 13.04479 -0.000002761 -0.000001270 +13311 13.04576 -0.000002760 -0.000001269 +13312 13.04672 -0.000002759 -0.000001269 +13313 13.04769 -0.000002757 -0.000001268 +13314 13.04865 -0.000002756 -0.000001267 +13315 13.04962 -0.000002755 -0.000001267 +13316 13.05058 -0.000002754 -0.000001266 +13317 13.05155 -0.000002752 -0.000001265 +13318 13.05251 -0.000002751 -0.000001265 +13319 13.05348 -0.000002750 -0.000001264 +13320 13.05444 -0.000002749 -0.000001263 +13321 13.05541 -0.000002748 -0.000001263 +13322 13.05637 -0.000002746 -0.000001262 +13323 13.05734 -0.000002745 -0.000001261 +13324 13.05830 -0.000002744 -0.000001261 +13325 13.05927 -0.000002743 -0.000001260 +13326 13.06023 -0.000002741 -0.000001259 +13327 13.06120 -0.000002740 -0.000001259 +13328 13.06216 -0.000002739 -0.000001258 +13329 13.06313 -0.000002738 -0.000001257 +13330 13.06409 -0.000002737 -0.000001257 +13331 13.06506 -0.000002735 -0.000001256 +13332 13.06602 -0.000002734 -0.000001256 +13333 13.06699 -0.000002733 -0.000001255 +13334 13.06795 -0.000002732 -0.000001254 +13335 13.06892 -0.000002731 -0.000001254 +13336 13.06988 -0.000002729 -0.000001253 +13337 13.07085 -0.000002728 -0.000001252 +13338 13.07181 -0.000002727 -0.000001252 +13339 13.07278 -0.000002726 -0.000001251 +13340 13.07374 -0.000002724 -0.000001250 +13341 13.07471 -0.000002723 -0.000001250 +13342 13.07567 -0.000002722 -0.000001249 +13343 13.07664 -0.000002721 -0.000001248 +13344 13.07760 -0.000002720 -0.000001248 +13345 13.07857 -0.000002718 -0.000001247 +13346 13.07953 -0.000002717 -0.000001246 +13347 13.08050 -0.000002716 -0.000001246 +13348 13.08146 -0.000002715 -0.000001245 +13349 13.08243 -0.000002714 -0.000001245 +13350 13.08339 -0.000002712 -0.000001244 +13351 13.08436 -0.000002711 -0.000001243 +13352 13.08532 -0.000002710 -0.000001243 +13353 13.08629 -0.000002709 -0.000001242 +13354 13.08725 -0.000002708 -0.000001241 +13355 13.08822 -0.000002706 -0.000001241 +13356 13.08918 -0.000002705 -0.000001240 +13357 13.09015 -0.000002704 -0.000001239 +13358 13.09111 -0.000002703 -0.000001239 +13359 13.09208 -0.000002702 -0.000001238 +13360 13.09304 -0.000002700 -0.000001238 +13361 13.09401 -0.000002699 -0.000001237 +13362 13.09497 -0.000002698 -0.000001236 +13363 13.09594 -0.000002697 -0.000001236 +13364 13.09690 -0.000002696 -0.000001235 +13365 13.09787 -0.000002695 -0.000001234 +13366 13.09883 -0.000002693 -0.000001234 +13367 13.09980 -0.000002692 -0.000001233 +13368 13.10077 -0.000002691 -0.000001232 +13369 13.10173 -0.000002690 -0.000001232 +13370 13.10270 -0.000002689 -0.000001231 +13371 13.10366 -0.000002687 -0.000001231 +13372 13.10463 -0.000002686 -0.000001230 +13373 13.10559 -0.000002685 -0.000001229 +13374 13.10656 -0.000002684 -0.000001229 +13375 13.10752 -0.000002683 -0.000001228 +13376 13.10849 -0.000002681 -0.000001227 +13377 13.10945 -0.000002680 -0.000001227 +13378 13.11042 -0.000002679 -0.000001226 +13379 13.11138 -0.000002678 -0.000001225 +13380 13.11235 -0.000002677 -0.000001225 +13381 13.11331 -0.000002676 -0.000001224 +13382 13.11428 -0.000002674 -0.000001224 +13383 13.11524 -0.000002673 -0.000001223 +13384 13.11621 -0.000002672 -0.000001222 +13385 13.11717 -0.000002671 -0.000001222 +13386 13.11814 -0.000002670 -0.000001221 +13387 13.11910 -0.000002668 -0.000001220 +13388 13.12007 -0.000002667 -0.000001220 +13389 13.12103 -0.000002666 -0.000001219 +13390 13.12200 -0.000002665 -0.000001219 +13391 13.12296 -0.000002664 -0.000001218 +13392 13.12393 -0.000002663 -0.000001217 +13393 13.12489 -0.000002661 -0.000001217 +13394 13.12586 -0.000002660 -0.000001216 +13395 13.12682 -0.000002659 -0.000001215 +13396 13.12779 -0.000002658 -0.000001215 +13397 13.12875 -0.000002657 -0.000001214 +13398 13.12972 -0.000002656 -0.000001214 +13399 13.13068 -0.000002654 -0.000001213 +13400 13.13165 -0.000002653 -0.000001212 +13401 13.13261 -0.000002652 -0.000001212 +13402 13.13358 -0.000002651 -0.000001211 +13403 13.13454 -0.000002650 -0.000001210 +13404 13.13551 -0.000002649 -0.000001210 +13405 13.13647 -0.000002647 -0.000001209 +13406 13.13744 -0.000002646 -0.000001209 +13407 13.13840 -0.000002645 -0.000001208 +13408 13.13937 -0.000002644 -0.000001207 +13409 13.14033 -0.000002643 -0.000001207 +13410 13.14130 -0.000002642 -0.000001206 +13411 13.14226 -0.000002640 -0.000001205 +13412 13.14323 -0.000002639 -0.000001205 +13413 13.14419 -0.000002638 -0.000001204 +13414 13.14516 -0.000002637 -0.000001204 +13415 13.14612 -0.000002636 -0.000001203 +13416 13.14709 -0.000002635 -0.000001202 +13417 13.14805 -0.000002633 -0.000001202 +13418 13.14902 -0.000002632 -0.000001201 +13419 13.14998 -0.000002631 -0.000001200 +13420 13.15095 -0.000002630 -0.000001200 +13421 13.15191 -0.000002629 -0.000001199 +13422 13.15288 -0.000002628 -0.000001199 +13423 13.15384 -0.000002626 -0.000001198 +13424 13.15481 -0.000002625 -0.000001197 +13425 13.15577 -0.000002624 -0.000001197 +13426 13.15674 -0.000002623 -0.000001196 +13427 13.15770 -0.000002622 -0.000001196 +13428 13.15867 -0.000002621 -0.000001195 +13429 13.15963 -0.000002620 -0.000001194 +13430 13.16060 -0.000002618 -0.000001194 +13431 13.16156 -0.000002617 -0.000001193 +13432 13.16253 -0.000002616 -0.000001193 +13433 13.16349 -0.000002615 -0.000001192 +13434 13.16446 -0.000002614 -0.000001191 +13435 13.16542 -0.000002613 -0.000001191 +13436 13.16639 -0.000002611 -0.000001190 +13437 13.16735 -0.000002610 -0.000001189 +13438 13.16832 -0.000002609 -0.000001189 +13439 13.16928 -0.000002608 -0.000001188 +13440 13.17025 -0.000002607 -0.000001188 +13441 13.17121 -0.000002606 -0.000001187 +13442 13.17218 -0.000002605 -0.000001186 +13443 13.17314 -0.000002603 -0.000001186 +13444 13.17411 -0.000002602 -0.000001185 +13445 13.17507 -0.000002601 -0.000001185 +13446 13.17604 -0.000002600 -0.000001184 +13447 13.17700 -0.000002599 -0.000001183 +13448 13.17797 -0.000002598 -0.000001183 +13449 13.17893 -0.000002597 -0.000001182 +13450 13.17990 -0.000002595 -0.000001182 +13451 13.18086 -0.000002594 -0.000001181 +13452 13.18183 -0.000002593 -0.000001180 +13453 13.18279 -0.000002592 -0.000001180 +13454 13.18376 -0.000002591 -0.000001179 +13455 13.18472 -0.000002590 -0.000001179 +13456 13.18569 -0.000002589 -0.000001178 +13457 13.18665 -0.000002587 -0.000001177 +13458 13.18762 -0.000002586 -0.000001177 +13459 13.18858 -0.000002585 -0.000001176 +13460 13.18955 -0.000002584 -0.000001176 +13461 13.19051 -0.000002583 -0.000001175 +13462 13.19148 -0.000002582 -0.000001174 +13463 13.19244 -0.000002581 -0.000001174 +13464 13.19341 -0.000002580 -0.000001173 +13465 13.19437 -0.000002578 -0.000001172 +13466 13.19534 -0.000002577 -0.000001172 +13467 13.19630 -0.000002576 -0.000001171 +13468 13.19727 -0.000002575 -0.000001171 +13469 13.19823 -0.000002574 -0.000001170 +13470 13.19920 -0.000002573 -0.000001170 +13471 13.20017 -0.000002572 -0.000001169 +13472 13.20113 -0.000002570 -0.000001168 +13473 13.20210 -0.000002569 -0.000001168 +13474 13.20306 -0.000002568 -0.000001167 +13475 13.20403 -0.000002567 -0.000001167 +13476 13.20499 -0.000002566 -0.000001166 +13477 13.20596 -0.000002565 -0.000001165 +13478 13.20692 -0.000002564 -0.000001165 +13479 13.20789 -0.000002563 -0.000001164 +13480 13.20885 -0.000002561 -0.000001164 +13481 13.20982 -0.000002560 -0.000001163 +13482 13.21078 -0.000002559 -0.000001162 +13483 13.21175 -0.000002558 -0.000001162 +13484 13.21271 -0.000002557 -0.000001161 +13485 13.21368 -0.000002556 -0.000001161 +13486 13.21464 -0.000002555 -0.000001160 +13487 13.21561 -0.000002554 -0.000001159 +13488 13.21657 -0.000002553 -0.000001159 +13489 13.21754 -0.000002551 -0.000001158 +13490 13.21850 -0.000002550 -0.000001158 +13491 13.21947 -0.000002549 -0.000001157 +13492 13.22043 -0.000002548 -0.000001156 +13493 13.22140 -0.000002547 -0.000001156 +13494 13.22236 -0.000002546 -0.000001155 +13495 13.22333 -0.000002545 -0.000001155 +13496 13.22429 -0.000002544 -0.000001154 +13497 13.22526 -0.000002542 -0.000001153 +13498 13.22622 -0.000002541 -0.000001153 +13499 13.22719 -0.000002540 -0.000001152 +13500 13.22815 -0.000002539 -0.000001152 +13501 13.22912 -0.000002538 -0.000001151 +13502 13.23008 -0.000002537 -0.000001151 +13503 13.23105 -0.000002536 -0.000001150 +13504 13.23201 -0.000002535 -0.000001149 +13505 13.23298 -0.000002534 -0.000001149 +13506 13.23394 -0.000002532 -0.000001148 +13507 13.23491 -0.000002531 -0.000001148 +13508 13.23587 -0.000002530 -0.000001147 +13509 13.23684 -0.000002529 -0.000001146 +13510 13.23780 -0.000002528 -0.000001146 +13511 13.23877 -0.000002527 -0.000001145 +13512 13.23973 -0.000002526 -0.000001145 +13513 13.24070 -0.000002525 -0.000001144 +13514 13.24166 -0.000002524 -0.000001143 +13515 13.24263 -0.000002523 -0.000001143 +13516 13.24359 -0.000002521 -0.000001142 +13517 13.24456 -0.000002520 -0.000001142 +13518 13.24552 -0.000002519 -0.000001141 +13519 13.24649 -0.000002518 -0.000001141 +13520 13.24745 -0.000002517 -0.000001140 +13521 13.24842 -0.000002516 -0.000001139 +13522 13.24938 -0.000002515 -0.000001139 +13523 13.25035 -0.000002514 -0.000001138 +13524 13.25131 -0.000002513 -0.000001138 +13525 13.25228 -0.000002512 -0.000001137 +13526 13.25324 -0.000002510 -0.000001137 +13527 13.25421 -0.000002509 -0.000001136 +13528 13.25517 -0.000002508 -0.000001135 +13529 13.25614 -0.000002507 -0.000001135 +13530 13.25710 -0.000002506 -0.000001134 +13531 13.25807 -0.000002505 -0.000001134 +13532 13.25903 -0.000002504 -0.000001133 +13533 13.26000 -0.000002503 -0.000001132 +13534 13.26096 -0.000002502 -0.000001132 +13535 13.26193 -0.000002501 -0.000001131 +13536 13.26289 -0.000002499 -0.000001131 +13537 13.26386 -0.000002498 -0.000001130 +13538 13.26482 -0.000002497 -0.000001130 +13539 13.26579 -0.000002496 -0.000001129 +13540 13.26675 -0.000002495 -0.000001128 +13541 13.26772 -0.000002494 -0.000001128 +13542 13.26868 -0.000002493 -0.000001127 +13543 13.26965 -0.000002492 -0.000001127 +13544 13.27061 -0.000002491 -0.000001126 +13545 13.27158 -0.000002490 -0.000001126 +13546 13.27254 -0.000002489 -0.000001125 +13547 13.27351 -0.000002488 -0.000001124 +13548 13.27447 -0.000002486 -0.000001124 +13549 13.27544 -0.000002485 -0.000001123 +13550 13.27640 -0.000002484 -0.000001123 +13551 13.27737 -0.000002483 -0.000001122 +13552 13.27833 -0.000002482 -0.000001122 +13553 13.27930 -0.000002481 -0.000001121 +13554 13.28026 -0.000002480 -0.000001120 +13555 13.28123 -0.000002479 -0.000001120 +13556 13.28219 -0.000002478 -0.000001119 +13557 13.28316 -0.000002477 -0.000001119 +13558 13.28412 -0.000002476 -0.000001118 +13559 13.28509 -0.000002475 -0.000001118 +13560 13.28605 -0.000002473 -0.000001117 +13561 13.28702 -0.000002472 -0.000001116 +13562 13.28798 -0.000002471 -0.000001116 +13563 13.28895 -0.000002470 -0.000001115 +13564 13.28991 -0.000002469 -0.000001115 +13565 13.29088 -0.000002468 -0.000001114 +13566 13.29184 -0.000002467 -0.000001114 +13567 13.29281 -0.000002466 -0.000001113 +13568 13.29377 -0.000002465 -0.000001112 +13569 13.29474 -0.000002464 -0.000001112 +13570 13.29570 -0.000002463 -0.000001111 +13571 13.29667 -0.000002462 -0.000001111 +13572 13.29763 -0.000002461 -0.000001110 +13573 13.29860 -0.000002459 -0.000001110 +13574 13.29956 -0.000002458 -0.000001109 +13575 13.30053 -0.000002457 -0.000001109 +13576 13.30150 -0.000002456 -0.000001108 +13577 13.30246 -0.000002455 -0.000001107 +13578 13.30343 -0.000002454 -0.000001107 +13579 13.30439 -0.000002453 -0.000001106 +13580 13.30536 -0.000002452 -0.000001106 +13581 13.30632 -0.000002451 -0.000001105 +13582 13.30729 -0.000002450 -0.000001105 +13583 13.30825 -0.000002449 -0.000001104 +13584 13.30922 -0.000002448 -0.000001103 +13585 13.31018 -0.000002447 -0.000001103 +13586 13.31115 -0.000002446 -0.000001102 +13587 13.31211 -0.000002445 -0.000001102 +13588 13.31308 -0.000002443 -0.000001101 +13589 13.31404 -0.000002442 -0.000001101 +13590 13.31501 -0.000002441 -0.000001100 +13591 13.31597 -0.000002440 -0.000001100 +13592 13.31694 -0.000002439 -0.000001099 +13593 13.31790 -0.000002438 -0.000001098 +13594 13.31887 -0.000002437 -0.000001098 +13595 13.31983 -0.000002436 -0.000001097 +13596 13.32080 -0.000002435 -0.000001097 +13597 13.32176 -0.000002434 -0.000001096 +13598 13.32273 -0.000002433 -0.000001096 +13599 13.32369 -0.000002432 -0.000001095 +13600 13.32466 -0.000002431 -0.000001095 +13601 13.32562 -0.000002430 -0.000001094 +13602 13.32659 -0.000002429 -0.000001093 +13603 13.32755 -0.000002428 -0.000001093 +13604 13.32852 -0.000002427 -0.000001092 +13605 13.32948 -0.000002426 -0.000001092 +13606 13.33045 -0.000002424 -0.000001091 +13607 13.33141 -0.000002423 -0.000001091 +13608 13.33238 -0.000002422 -0.000001090 +13609 13.33334 -0.000002421 -0.000001090 +13610 13.33431 -0.000002420 -0.000001089 +13611 13.33527 -0.000002419 -0.000001088 +13612 13.33624 -0.000002418 -0.000001088 +13613 13.33720 -0.000002417 -0.000001087 +13614 13.33817 -0.000002416 -0.000001087 +13615 13.33913 -0.000002415 -0.000001086 +13616 13.34010 -0.000002414 -0.000001086 +13617 13.34106 -0.000002413 -0.000001085 +13618 13.34203 -0.000002412 -0.000001085 +13619 13.34299 -0.000002411 -0.000001084 +13620 13.34396 -0.000002410 -0.000001084 +13621 13.34492 -0.000002409 -0.000001083 +13622 13.34589 -0.000002408 -0.000001082 +13623 13.34685 -0.000002407 -0.000001082 +13624 13.34782 -0.000002406 -0.000001081 +13625 13.34878 -0.000002405 -0.000001081 +13626 13.34975 -0.000002403 -0.000001080 +13627 13.35071 -0.000002402 -0.000001080 +13628 13.35168 -0.000002401 -0.000001079 +13629 13.35264 -0.000002400 -0.000001079 +13630 13.35361 -0.000002399 -0.000001078 +13631 13.35457 -0.000002398 -0.000001078 +13632 13.35554 -0.000002397 -0.000001077 +13633 13.35650 -0.000002396 -0.000001076 +13634 13.35747 -0.000002395 -0.000001076 +13635 13.35843 -0.000002394 -0.000001075 +13636 13.35940 -0.000002393 -0.000001075 +13637 13.36036 -0.000002392 -0.000001074 +13638 13.36133 -0.000002391 -0.000001074 +13639 13.36229 -0.000002390 -0.000001073 +13640 13.36326 -0.000002389 -0.000001073 +13641 13.36422 -0.000002388 -0.000001072 +13642 13.36519 -0.000002387 -0.000001072 +13643 13.36615 -0.000002386 -0.000001071 +13644 13.36712 -0.000002385 -0.000001070 +13645 13.36808 -0.000002384 -0.000001070 +13646 13.36905 -0.000002383 -0.000001069 +13647 13.37001 -0.000002382 -0.000001069 +13648 13.37098 -0.000002381 -0.000001068 +13649 13.37194 -0.000002380 -0.000001068 +13650 13.37291 -0.000002379 -0.000001067 +13651 13.37387 -0.000002378 -0.000001067 +13652 13.37484 -0.000002377 -0.000001066 +13653 13.37580 -0.000002376 -0.000001066 +13654 13.37677 -0.000002375 -0.000001065 +13655 13.37773 -0.000002373 -0.000001065 +13656 13.37870 -0.000002372 -0.000001064 +13657 13.37966 -0.000002371 -0.000001063 +13658 13.38063 -0.000002370 -0.000001063 +13659 13.38159 -0.000002369 -0.000001062 +13660 13.38256 -0.000002368 -0.000001062 +13661 13.38352 -0.000002367 -0.000001061 +13662 13.38449 -0.000002366 -0.000001061 +13663 13.38545 -0.000002365 -0.000001060 +13664 13.38642 -0.000002364 -0.000001060 +13665 13.38738 -0.000002363 -0.000001059 +13666 13.38835 -0.000002362 -0.000001059 +13667 13.38931 -0.000002361 -0.000001058 +13668 13.39028 -0.000002360 -0.000001058 +13669 13.39124 -0.000002359 -0.000001057 +13670 13.39221 -0.000002358 -0.000001056 +13671 13.39317 -0.000002357 -0.000001056 +13672 13.39414 -0.000002356 -0.000001055 +13673 13.39510 -0.000002355 -0.000001055 +13674 13.39607 -0.000002354 -0.000001054 +13675 13.39703 -0.000002353 -0.000001054 +13676 13.39800 -0.000002352 -0.000001053 +13677 13.39896 -0.000002351 -0.000001053 +13678 13.39993 -0.000002350 -0.000001052 +13679 13.40090 -0.000002349 -0.000001052 +13680 13.40186 -0.000002348 -0.000001051 +13681 13.40283 -0.000002347 -0.000001051 +13682 13.40379 -0.000002346 -0.000001050 +13683 13.40476 -0.000002345 -0.000001050 +13684 13.40572 -0.000002344 -0.000001049 +13685 13.40669 -0.000002343 -0.000001049 +13686 13.40765 -0.000002342 -0.000001048 +13687 13.40862 -0.000002341 -0.000001047 +13688 13.40958 -0.000002340 -0.000001047 +13689 13.41055 -0.000002339 -0.000001046 +13690 13.41151 -0.000002338 -0.000001046 +13691 13.41248 -0.000002337 -0.000001045 +13692 13.41344 -0.000002336 -0.000001045 +13693 13.41441 -0.000002335 -0.000001044 +13694 13.41537 -0.000002334 -0.000001044 +13695 13.41634 -0.000002333 -0.000001043 +13696 13.41730 -0.000002332 -0.000001043 +13697 13.41827 -0.000002331 -0.000001042 +13698 13.41923 -0.000002330 -0.000001042 +13699 13.42020 -0.000002329 -0.000001041 +13700 13.42116 -0.000002328 -0.000001041 +13701 13.42213 -0.000002327 -0.000001040 +13702 13.42309 -0.000002326 -0.000001040 +13703 13.42406 -0.000002325 -0.000001039 +13704 13.42502 -0.000002324 -0.000001039 +13705 13.42599 -0.000002323 -0.000001038 +13706 13.42695 -0.000002322 -0.000001038 +13707 13.42792 -0.000002321 -0.000001037 +13708 13.42888 -0.000002320 -0.000001036 +13709 13.42985 -0.000002319 -0.000001036 +13710 13.43081 -0.000002318 -0.000001035 +13711 13.43178 -0.000002317 -0.000001035 +13712 13.43274 -0.000002316 -0.000001034 +13713 13.43371 -0.000002315 -0.000001034 +13714 13.43467 -0.000002314 -0.000001033 +13715 13.43564 -0.000002313 -0.000001033 +13716 13.43660 -0.000002312 -0.000001032 +13717 13.43757 -0.000002311 -0.000001032 +13718 13.43853 -0.000002310 -0.000001031 +13719 13.43950 -0.000002309 -0.000001031 +13720 13.44046 -0.000002308 -0.000001030 +13721 13.44143 -0.000002307 -0.000001030 +13722 13.44239 -0.000002306 -0.000001029 +13723 13.44336 -0.000002305 -0.000001029 +13724 13.44432 -0.000002304 -0.000001028 +13725 13.44529 -0.000002303 -0.000001028 +13726 13.44625 -0.000002302 -0.000001027 +13727 13.44722 -0.000002301 -0.000001027 +13728 13.44818 -0.000002300 -0.000001026 +13729 13.44915 -0.000002299 -0.000001026 +13730 13.45011 -0.000002298 -0.000001025 +13731 13.45108 -0.000002297 -0.000001025 +13732 13.45204 -0.000002296 -0.000001024 +13733 13.45301 -0.000002295 -0.000001024 +13734 13.45397 -0.000002294 -0.000001023 +13735 13.45494 -0.000002293 -0.000001022 +13736 13.45590 -0.000002292 -0.000001022 +13737 13.45687 -0.000002291 -0.000001021 +13738 13.45783 -0.000002290 -0.000001021 +13739 13.45880 -0.000002289 -0.000001020 +13740 13.45976 -0.000002288 -0.000001020 +13741 13.46073 -0.000002287 -0.000001019 +13742 13.46169 -0.000002286 -0.000001019 +13743 13.46266 -0.000002285 -0.000001018 +13744 13.46362 -0.000002284 -0.000001018 +13745 13.46459 -0.000002283 -0.000001017 +13746 13.46555 -0.000002282 -0.000001017 +13747 13.46652 -0.000002281 -0.000001016 +13748 13.46748 -0.000002280 -0.000001016 +13749 13.46845 -0.000002279 -0.000001015 +13750 13.46941 -0.000002278 -0.000001015 +13751 13.47038 -0.000002277 -0.000001014 +13752 13.47134 -0.000002276 -0.000001014 +13753 13.47231 -0.000002275 -0.000001013 +13754 13.47327 -0.000002274 -0.000001013 +13755 13.47424 -0.000002273 -0.000001012 +13756 13.47520 -0.000002272 -0.000001012 +13757 13.47617 -0.000002271 -0.000001011 +13758 13.47713 -0.000002270 -0.000001011 +13759 13.47810 -0.000002269 -0.000001010 +13760 13.47906 -0.000002268 -0.000001010 +13761 13.48003 -0.000002267 -0.000001009 +13762 13.48099 -0.000002266 -0.000001009 +13763 13.48196 -0.000002266 -0.000001008 +13764 13.48292 -0.000002265 -0.000001008 +13765 13.48389 -0.000002264 -0.000001007 +13766 13.48485 -0.000002263 -0.000001007 +13767 13.48582 -0.000002262 -0.000001006 +13768 13.48678 -0.000002261 -0.000001006 +13769 13.48775 -0.000002260 -0.000001005 +13770 13.48871 -0.000002259 -0.000001005 +13771 13.48968 -0.000002258 -0.000001004 +13772 13.49064 -0.000002257 -0.000001004 +13773 13.49161 -0.000002256 -0.000001003 +13774 13.49257 -0.000002255 -0.000001003 +13775 13.49354 -0.000002254 -0.000001002 +13776 13.49450 -0.000002253 -0.000001002 +13777 13.49547 -0.000002252 -0.000001001 +13778 13.49643 -0.000002251 -0.000001001 +13779 13.49740 -0.000002250 -0.000001000 +13780 13.49836 -0.000002249 -0.000001000 +13781 13.49933 -0.000002248 -0.000000999 +13782 13.50030 -0.000002247 -0.000000999 +13783 13.50126 -0.000002246 -0.000000998 +13784 13.50223 -0.000002245 -0.000000998 +13785 13.50319 -0.000002244 -0.000000997 +13786 13.50416 -0.000002243 -0.000000997 +13787 13.50512 -0.000002242 -0.000000996 +13788 13.50609 -0.000002241 -0.000000996 +13789 13.50705 -0.000002240 -0.000000995 +13790 13.50802 -0.000002239 -0.000000995 +13791 13.50898 -0.000002238 -0.000000994 +13792 13.50995 -0.000002237 -0.000000994 +13793 13.51091 -0.000002237 -0.000000993 +13794 13.51188 -0.000002236 -0.000000993 +13795 13.51284 -0.000002235 -0.000000992 +13796 13.51381 -0.000002234 -0.000000992 +13797 13.51477 -0.000002233 -0.000000991 +13798 13.51574 -0.000002232 -0.000000991 +13799 13.51670 -0.000002231 -0.000000990 +13800 13.51767 -0.000002230 -0.000000990 +13801 13.51863 -0.000002229 -0.000000989 +13802 13.51960 -0.000002228 -0.000000989 +13803 13.52056 -0.000002227 -0.000000988 +13804 13.52153 -0.000002226 -0.000000988 +13805 13.52249 -0.000002225 -0.000000987 +13806 13.52346 -0.000002224 -0.000000987 +13807 13.52442 -0.000002223 -0.000000986 +13808 13.52539 -0.000002222 -0.000000986 +13809 13.52635 -0.000002221 -0.000000985 +13810 13.52732 -0.000002220 -0.000000985 +13811 13.52828 -0.000002219 -0.000000984 +13812 13.52925 -0.000002218 -0.000000984 +13813 13.53021 -0.000002217 -0.000000983 +13814 13.53118 -0.000002217 -0.000000983 +13815 13.53214 -0.000002216 -0.000000982 +13816 13.53311 -0.000002215 -0.000000982 +13817 13.53407 -0.000002214 -0.000000981 +13818 13.53504 -0.000002213 -0.000000981 +13819 13.53600 -0.000002212 -0.000000980 +13820 13.53697 -0.000002211 -0.000000980 +13821 13.53793 -0.000002210 -0.000000979 +13822 13.53890 -0.000002209 -0.000000979 +13823 13.53986 -0.000002208 -0.000000978 +13824 13.54083 -0.000002207 -0.000000978 +13825 13.54179 -0.000002206 -0.000000977 +13826 13.54276 -0.000002205 -0.000000977 +13827 13.54372 -0.000002204 -0.000000976 +13828 13.54469 -0.000002203 -0.000000976 +13829 13.54565 -0.000002202 -0.000000976 +13830 13.54662 -0.000002201 -0.000000975 +13831 13.54758 -0.000002200 -0.000000975 +13832 13.54855 -0.000002200 -0.000000974 +13833 13.54951 -0.000002199 -0.000000974 +13834 13.55048 -0.000002198 -0.000000973 +13835 13.55144 -0.000002197 -0.000000973 +13836 13.55241 -0.000002196 -0.000000972 +13837 13.55337 -0.000002195 -0.000000972 +13838 13.55434 -0.000002194 -0.000000971 +13839 13.55530 -0.000002193 -0.000000971 +13840 13.55627 -0.000002192 -0.000000970 +13841 13.55723 -0.000002191 -0.000000970 +13842 13.55820 -0.000002190 -0.000000969 +13843 13.55916 -0.000002189 -0.000000969 +13844 13.56013 -0.000002188 -0.000000968 +13845 13.56109 -0.000002187 -0.000000968 +13846 13.56206 -0.000002186 -0.000000967 +13847 13.56302 -0.000002185 -0.000000967 +13848 13.56399 -0.000002185 -0.000000966 +13849 13.56495 -0.000002184 -0.000000966 +13850 13.56592 -0.000002183 -0.000000965 +13851 13.56688 -0.000002182 -0.000000965 +13852 13.56785 -0.000002181 -0.000000964 +13853 13.56881 -0.000002180 -0.000000964 +13854 13.56978 -0.000002179 -0.000000963 +13855 13.57074 -0.000002178 -0.000000963 +13856 13.57171 -0.000002177 -0.000000962 +13857 13.57267 -0.000002176 -0.000000962 +13858 13.57364 -0.000002175 -0.000000962 +13859 13.57460 -0.000002174 -0.000000961 +13860 13.57557 -0.000002173 -0.000000961 +13861 13.57653 -0.000002172 -0.000000960 +13862 13.57750 -0.000002172 -0.000000960 +13863 13.57846 -0.000002171 -0.000000959 +13864 13.57943 -0.000002170 -0.000000959 +13865 13.58039 -0.000002169 -0.000000958 +13866 13.58136 -0.000002168 -0.000000958 +13867 13.58232 -0.000002167 -0.000000957 +13868 13.58329 -0.000002166 -0.000000957 +13869 13.58425 -0.000002165 -0.000000956 +13870 13.58522 -0.000002164 -0.000000956 +13871 13.58618 -0.000002163 -0.000000955 +13872 13.58715 -0.000002162 -0.000000955 +13873 13.58811 -0.000002161 -0.000000954 +13874 13.58908 -0.000002160 -0.000000954 +13875 13.59004 -0.000002160 -0.000000953 +13876 13.59101 -0.000002159 -0.000000953 +13877 13.59197 -0.000002158 -0.000000952 +13878 13.59294 -0.000002157 -0.000000952 +13879 13.59390 -0.000002156 -0.000000952 +13880 13.59487 -0.000002155 -0.000000951 +13881 13.59583 -0.000002154 -0.000000951 +13882 13.59680 -0.000002153 -0.000000950 +13883 13.59776 -0.000002152 -0.000000950 +13884 13.59873 -0.000002151 -0.000000949 +13885 13.59969 -0.000002150 -0.000000949 +13886 13.60066 -0.000002149 -0.000000948 +13887 13.60163 -0.000002149 -0.000000948 +13888 13.60259 -0.000002148 -0.000000947 +13889 13.60356 -0.000002147 -0.000000947 +13890 13.60452 -0.000002146 -0.000000946 +13891 13.60549 -0.000002145 -0.000000946 +13892 13.60645 -0.000002144 -0.000000945 +13893 13.60742 -0.000002143 -0.000000945 +13894 13.60838 -0.000002142 -0.000000944 +13895 13.60935 -0.000002141 -0.000000944 +13896 13.61031 -0.000002140 -0.000000944 +13897 13.61128 -0.000002139 -0.000000943 +13898 13.61224 -0.000002138 -0.000000943 +13899 13.61321 -0.000002138 -0.000000942 +13900 13.61417 -0.000002137 -0.000000942 +13901 13.61514 -0.000002136 -0.000000941 +13902 13.61610 -0.000002135 -0.000000941 +13903 13.61707 -0.000002134 -0.000000940 +13904 13.61803 -0.000002133 -0.000000940 +13905 13.61900 -0.000002132 -0.000000939 +13906 13.61996 -0.000002131 -0.000000939 +13907 13.62093 -0.000002130 -0.000000938 +13908 13.62189 -0.000002129 -0.000000938 +13909 13.62286 -0.000002128 -0.000000937 +13910 13.62382 -0.000002128 -0.000000937 +13911 13.62479 -0.000002127 -0.000000937 +13912 13.62575 -0.000002126 -0.000000936 +13913 13.62672 -0.000002125 -0.000000936 +13914 13.62768 -0.000002124 -0.000000935 +13915 13.62865 -0.000002123 -0.000000935 +13916 13.62961 -0.000002122 -0.000000934 +13917 13.63058 -0.000002121 -0.000000934 +13918 13.63154 -0.000002120 -0.000000933 +13919 13.63251 -0.000002119 -0.000000933 +13920 13.63347 -0.000002119 -0.000000932 +13921 13.63444 -0.000002118 -0.000000932 +13922 13.63540 -0.000002117 -0.000000931 +13923 13.63637 -0.000002116 -0.000000931 +13924 13.63733 -0.000002115 -0.000000931 +13925 13.63830 -0.000002114 -0.000000930 +13926 13.63926 -0.000002113 -0.000000930 +13927 13.64023 -0.000002112 -0.000000929 +13928 13.64119 -0.000002111 -0.000000929 +13929 13.64216 -0.000002110 -0.000000928 +13930 13.64312 -0.000002110 -0.000000928 +13931 13.64409 -0.000002109 -0.000000927 +13932 13.64505 -0.000002108 -0.000000927 +13933 13.64602 -0.000002107 -0.000000926 +13934 13.64698 -0.000002106 -0.000000926 +13935 13.64795 -0.000002105 -0.000000925 +13936 13.64891 -0.000002104 -0.000000925 +13937 13.64988 -0.000002103 -0.000000925 +13938 13.65084 -0.000002102 -0.000000924 +13939 13.65181 -0.000002102 -0.000000924 +13940 13.65277 -0.000002101 -0.000000923 +13941 13.65374 -0.000002100 -0.000000923 +13942 13.65470 -0.000002099 -0.000000922 +13943 13.65567 -0.000002098 -0.000000922 +13944 13.65663 -0.000002097 -0.000000921 +13945 13.65760 -0.000002096 -0.000000921 +13946 13.65856 -0.000002095 -0.000000920 +13947 13.65953 -0.000002094 -0.000000920 +13948 13.66049 -0.000002094 -0.000000920 +13949 13.66146 -0.000002093 -0.000000919 +13950 13.66242 -0.000002092 -0.000000919 +13951 13.66339 -0.000002091 -0.000000918 +13952 13.66435 -0.000002090 -0.000000918 +13953 13.66532 -0.000002089 -0.000000917 +13954 13.66628 -0.000002088 -0.000000917 +13955 13.66725 -0.000002087 -0.000000916 +13956 13.66821 -0.000002086 -0.000000916 +13957 13.66918 -0.000002086 -0.000000915 +13958 13.67014 -0.000002085 -0.000000915 +13959 13.67111 -0.000002084 -0.000000915 +13960 13.67207 -0.000002083 -0.000000914 +13961 13.67304 -0.000002082 -0.000000914 +13962 13.67400 -0.000002081 -0.000000913 +13963 13.67497 -0.000002080 -0.000000913 +13964 13.67593 -0.000002079 -0.000000912 +13965 13.67690 -0.000002079 -0.000000912 +13966 13.67786 -0.000002078 -0.000000911 +13967 13.67883 -0.000002077 -0.000000911 +13968 13.67979 -0.000002076 -0.000000910 +13969 13.68076 -0.000002075 -0.000000910 +13970 13.68172 -0.000002074 -0.000000910 +13971 13.68269 -0.000002073 -0.000000909 +13972 13.68365 -0.000002072 -0.000000909 +13973 13.68462 -0.000002072 -0.000000908 +13974 13.68558 -0.000002071 -0.000000908 +13975 13.68655 -0.000002070 -0.000000907 +13976 13.68751 -0.000002069 -0.000000907 +13977 13.68848 -0.000002068 -0.000000906 +13978 13.68944 -0.000002067 -0.000000906 +13979 13.69041 -0.000002066 -0.000000906 +13980 13.69137 -0.000002065 -0.000000905 +13981 13.69234 -0.000002065 -0.000000905 +13982 13.69330 -0.000002064 -0.000000904 +13983 13.69427 -0.000002063 -0.000000904 +13984 13.69523 -0.000002062 -0.000000903 +13985 13.69620 -0.000002061 -0.000000903 +13986 13.69716 -0.000002060 -0.000000902 +13987 13.69813 -0.000002059 -0.000000902 +13988 13.69909 -0.000002058 -0.000000902 +13989 13.70006 -0.000002058 -0.000000901 +13990 13.70103 -0.000002057 -0.000000901 +13991 13.70199 -0.000002056 -0.000000900 +13992 13.70296 -0.000002055 -0.000000900 +13993 13.70392 -0.000002054 -0.000000899 +13994 13.70489 -0.000002053 -0.000000899 +13995 13.70585 -0.000002052 -0.000000898 +13996 13.70682 -0.000002051 -0.000000898 +13997 13.70778 -0.000002051 -0.000000898 +13998 13.70875 -0.000002050 -0.000000897 +13999 13.70971 -0.000002049 -0.000000897 +14000 13.71068 -0.000002048 -0.000000896 +14001 13.71164 -0.000002047 -0.000000896 +14002 13.71261 -0.000002046 -0.000000895 +14003 13.71357 -0.000002045 -0.000000895 +14004 13.71454 -0.000002045 -0.000000894 +14005 13.71550 -0.000002044 -0.000000894 +14006 13.71647 -0.000002043 -0.000000894 +14007 13.71743 -0.000002042 -0.000000893 +14008 13.71840 -0.000002041 -0.000000893 +14009 13.71936 -0.000002040 -0.000000892 +14010 13.72033 -0.000002039 -0.000000892 +14011 13.72129 -0.000002039 -0.000000891 +14012 13.72226 -0.000002038 -0.000000891 +14013 13.72322 -0.000002037 -0.000000891 +14014 13.72419 -0.000002036 -0.000000890 +14015 13.72515 -0.000002035 -0.000000890 +14016 13.72612 -0.000002034 -0.000000889 +14017 13.72708 -0.000002033 -0.000000889 +14018 13.72805 -0.000002032 -0.000000888 +14019 13.72901 -0.000002032 -0.000000888 +14020 13.72998 -0.000002031 -0.000000887 +14021 13.73094 -0.000002030 -0.000000887 +14022 13.73191 -0.000002029 -0.000000887 +14023 13.73287 -0.000002028 -0.000000886 +14024 13.73384 -0.000002027 -0.000000886 +14025 13.73480 -0.000002027 -0.000000885 +14026 13.73577 -0.000002026 -0.000000885 +14027 13.73673 -0.000002025 -0.000000884 +14028 13.73770 -0.000002024 -0.000000884 +14029 13.73866 -0.000002023 -0.000000884 +14030 13.73963 -0.000002022 -0.000000883 +14031 13.74059 -0.000002021 -0.000000883 +14032 13.74156 -0.000002021 -0.000000882 +14033 13.74252 -0.000002020 -0.000000882 +14034 13.74349 -0.000002019 -0.000000881 +14035 13.74445 -0.000002018 -0.000000881 +14036 13.74542 -0.000002017 -0.000000880 +14037 13.74638 -0.000002016 -0.000000880 +14038 13.74735 -0.000002015 -0.000000880 +14039 13.74831 -0.000002015 -0.000000879 +14040 13.74928 -0.000002014 -0.000000879 +14041 13.75024 -0.000002013 -0.000000878 +14042 13.75121 -0.000002012 -0.000000878 +14043 13.75217 -0.000002011 -0.000000877 +14044 13.75314 -0.000002010 -0.000000877 +14045 13.75410 -0.000002010 -0.000000877 +14046 13.75507 -0.000002009 -0.000000876 +14047 13.75603 -0.000002008 -0.000000876 +14048 13.75700 -0.000002007 -0.000000875 +14049 13.75796 -0.000002006 -0.000000875 +14050 13.75893 -0.000002005 -0.000000874 +14051 13.75989 -0.000002004 -0.000000874 +14052 13.76086 -0.000002004 -0.000000874 +14053 13.76182 -0.000002003 -0.000000873 +14054 13.76279 -0.000002002 -0.000000873 +14055 13.76375 -0.000002001 -0.000000872 +14056 13.76472 -0.000002000 -0.000000872 +14057 13.76568 -0.000001999 -0.000000871 +14058 13.76665 -0.000001999 -0.000000871 +14059 13.76761 -0.000001998 -0.000000871 +14060 13.76858 -0.000001997 -0.000000870 +14061 13.76954 -0.000001996 -0.000000870 +14062 13.77051 -0.000001995 -0.000000869 +14063 13.77147 -0.000001994 -0.000000869 +14064 13.77244 -0.000001994 -0.000000868 +14065 13.77340 -0.000001993 -0.000000868 +14066 13.77437 -0.000001992 -0.000000868 +14067 13.77533 -0.000001991 -0.000000867 +14068 13.77630 -0.000001990 -0.000000867 +14069 13.77726 -0.000001989 -0.000000866 +14070 13.77823 -0.000001988 -0.000000866 +14071 13.77919 -0.000001988 -0.000000865 +14072 13.78016 -0.000001987 -0.000000865 +14073 13.78112 -0.000001986 -0.000000865 +14074 13.78209 -0.000001985 -0.000000864 +14075 13.78305 -0.000001984 -0.000000864 +14076 13.78402 -0.000001983 -0.000000863 +14077 13.78498 -0.000001983 -0.000000863 +14078 13.78595 -0.000001982 -0.000000863 +14079 13.78691 -0.000001981 -0.000000862 +14080 13.78788 -0.000001980 -0.000000862 +14081 13.78884 -0.000001979 -0.000000861 +14082 13.78981 -0.000001978 -0.000000861 +14083 13.79077 -0.000001978 -0.000000860 +14084 13.79174 -0.000001977 -0.000000860 +14085 13.79270 -0.000001976 -0.000000860 +14086 13.79367 -0.000001975 -0.000000859 +14087 13.79463 -0.000001974 -0.000000859 +14088 13.79560 -0.000001974 -0.000000858 +14089 13.79656 -0.000001973 -0.000000858 +14090 13.79753 -0.000001972 -0.000000857 +14091 13.79849 -0.000001971 -0.000000857 +14092 13.79946 -0.000001970 -0.000000857 +14093 13.80043 -0.000001969 -0.000000856 +14094 13.80139 -0.000001969 -0.000000856 +14095 13.80236 -0.000001968 -0.000000855 +14096 13.80332 -0.000001967 -0.000000855 +14097 13.80429 -0.000001966 -0.000000855 +14098 13.80525 -0.000001965 -0.000000854 +14099 13.80622 -0.000001964 -0.000000854 +14100 13.80718 -0.000001964 -0.000000853 +14101 13.80815 -0.000001963 -0.000000853 +14102 13.80911 -0.000001962 -0.000000852 +14103 13.81008 -0.000001961 -0.000000852 +14104 13.81104 -0.000001960 -0.000000852 +14105 13.81201 -0.000001959 -0.000000851 +14106 13.81297 -0.000001959 -0.000000851 +14107 13.81394 -0.000001958 -0.000000850 +14108 13.81490 -0.000001957 -0.000000850 +14109 13.81587 -0.000001956 -0.000000850 +14110 13.81683 -0.000001955 -0.000000849 +14111 13.81780 -0.000001955 -0.000000849 +14112 13.81876 -0.000001954 -0.000000848 +14113 13.81973 -0.000001953 -0.000000848 +14114 13.82069 -0.000001952 -0.000000847 +14115 13.82166 -0.000001951 -0.000000847 +14116 13.82262 -0.000001950 -0.000000847 +14117 13.82359 -0.000001950 -0.000000846 +14118 13.82455 -0.000001949 -0.000000846 +14119 13.82552 -0.000001948 -0.000000845 +14120 13.82648 -0.000001947 -0.000000845 +14121 13.82745 -0.000001946 -0.000000845 +14122 13.82841 -0.000001946 -0.000000844 +14123 13.82938 -0.000001945 -0.000000844 +14124 13.83034 -0.000001944 -0.000000843 +14125 13.83131 -0.000001943 -0.000000843 +14126 13.83227 -0.000001942 -0.000000843 +14127 13.83324 -0.000001942 -0.000000842 +14128 13.83420 -0.000001941 -0.000000842 +14129 13.83517 -0.000001940 -0.000000841 +14130 13.83613 -0.000001939 -0.000000841 +14131 13.83710 -0.000001938 -0.000000840 +14132 13.83806 -0.000001937 -0.000000840 +14133 13.83903 -0.000001937 -0.000000840 +14134 13.83999 -0.000001936 -0.000000839 +14135 13.84096 -0.000001935 -0.000000839 +14136 13.84192 -0.000001934 -0.000000838 +14137 13.84289 -0.000001933 -0.000000838 +14138 13.84385 -0.000001933 -0.000000838 +14139 13.84482 -0.000001932 -0.000000837 +14140 13.84578 -0.000001931 -0.000000837 +14141 13.84675 -0.000001930 -0.000000836 +14142 13.84771 -0.000001929 -0.000000836 +14143 13.84868 -0.000001929 -0.000000836 +14144 13.84964 -0.000001928 -0.000000835 +14145 13.85061 -0.000001927 -0.000000835 +14146 13.85157 -0.000001926 -0.000000834 +14147 13.85254 -0.000001925 -0.000000834 +14148 13.85350 -0.000001925 -0.000000834 +14149 13.85447 -0.000001924 -0.000000833 +14150 13.85543 -0.000001923 -0.000000833 +14151 13.85640 -0.000001922 -0.000000832 +14152 13.85736 -0.000001921 -0.000000832 +14153 13.85833 -0.000001921 -0.000000831 +14154 13.85929 -0.000001920 -0.000000831 +14155 13.86026 -0.000001919 -0.000000831 +14156 13.86122 -0.000001918 -0.000000830 +14157 13.86219 -0.000001917 -0.000000830 +14158 13.86315 -0.000001917 -0.000000829 +14159 13.86412 -0.000001916 -0.000000829 +14160 13.86508 -0.000001915 -0.000000829 +14161 13.86605 -0.000001914 -0.000000828 +14162 13.86701 -0.000001913 -0.000000828 +14163 13.86798 -0.000001913 -0.000000827 +14164 13.86894 -0.000001912 -0.000000827 +14165 13.86991 -0.000001911 -0.000000827 +14166 13.87087 -0.000001910 -0.000000826 +14167 13.87184 -0.000001909 -0.000000826 +14168 13.87280 -0.000001909 -0.000000825 +14169 13.87377 -0.000001908 -0.000000825 +14170 13.87473 -0.000001907 -0.000000825 +14171 13.87570 -0.000001906 -0.000000824 +14172 13.87666 -0.000001905 -0.000000824 +14173 13.87763 -0.000001905 -0.000000823 +14174 13.87859 -0.000001904 -0.000000823 +14175 13.87956 -0.000001903 -0.000000823 +14176 13.88052 -0.000001902 -0.000000822 +14177 13.88149 -0.000001901 -0.000000822 +14178 13.88245 -0.000001901 -0.000000821 +14179 13.88342 -0.000001900 -0.000000821 +14180 13.88438 -0.000001899 -0.000000821 +14181 13.88535 -0.000001898 -0.000000820 +14182 13.88631 -0.000001897 -0.000000820 +14183 13.88728 -0.000001897 -0.000000819 +14184 13.88824 -0.000001896 -0.000000819 +14185 13.88921 -0.000001895 -0.000000819 +14186 13.89017 -0.000001894 -0.000000818 +14187 13.89114 -0.000001893 -0.000000818 +14188 13.89210 -0.000001893 -0.000000817 +14189 13.89307 -0.000001892 -0.000000817 +14190 13.89403 -0.000001891 -0.000000817 +14191 13.89500 -0.000001890 -0.000000816 +14192 13.89596 -0.000001890 -0.000000816 +14193 13.89693 -0.000001889 -0.000000815 +14194 13.89789 -0.000001888 -0.000000815 +14195 13.89886 -0.000001887 -0.000000815 +14196 13.89982 -0.000001886 -0.000000814 +14197 13.90079 -0.000001886 -0.000000814 +14198 13.90176 -0.000001885 -0.000000813 +14199 13.90272 -0.000001884 -0.000000813 +14200 13.90369 -0.000001883 -0.000000813 +14201 13.90465 -0.000001882 -0.000000812 +14202 13.90562 -0.000001882 -0.000000812 +14203 13.90658 -0.000001881 -0.000000812 +14204 13.90755 -0.000001880 -0.000000811 +14205 13.90851 -0.000001879 -0.000000811 +14206 13.90948 -0.000001879 -0.000000810 +14207 13.91044 -0.000001878 -0.000000810 +14208 13.91141 -0.000001877 -0.000000810 +14209 13.91237 -0.000001876 -0.000000809 +14210 13.91334 -0.000001875 -0.000000809 +14211 13.91430 -0.000001875 -0.000000808 +14212 13.91527 -0.000001874 -0.000000808 +14213 13.91623 -0.000001873 -0.000000808 +14214 13.91720 -0.000001872 -0.000000807 +14215 13.91816 -0.000001872 -0.000000807 +14216 13.91913 -0.000001871 -0.000000806 +14217 13.92009 -0.000001870 -0.000000806 +14218 13.92106 -0.000001869 -0.000000806 +14219 13.92202 -0.000001868 -0.000000805 +14220 13.92299 -0.000001868 -0.000000805 +14221 13.92395 -0.000001867 -0.000000804 +14222 13.92492 -0.000001866 -0.000000804 +14223 13.92588 -0.000001865 -0.000000804 +14224 13.92685 -0.000001865 -0.000000803 +14225 13.92781 -0.000001864 -0.000000803 +14226 13.92878 -0.000001863 -0.000000802 +14227 13.92974 -0.000001862 -0.000000802 +14228 13.93071 -0.000001861 -0.000000802 +14229 13.93167 -0.000001861 -0.000000801 +14230 13.93264 -0.000001860 -0.000000801 +14231 13.93360 -0.000001859 -0.000000801 +14232 13.93457 -0.000001858 -0.000000800 +14233 13.93553 -0.000001858 -0.000000800 +14234 13.93650 -0.000001857 -0.000000799 +14235 13.93746 -0.000001856 -0.000000799 +14236 13.93843 -0.000001855 -0.000000799 +14237 13.93939 -0.000001854 -0.000000798 +14238 13.94036 -0.000001854 -0.000000798 +14239 13.94132 -0.000001853 -0.000000797 +14240 13.94229 -0.000001852 -0.000000797 +14241 13.94325 -0.000001851 -0.000000797 +14242 13.94422 -0.000001851 -0.000000796 +14243 13.94518 -0.000001850 -0.000000796 +14244 13.94615 -0.000001849 -0.000000796 +14245 13.94711 -0.000001848 -0.000000795 +14246 13.94808 -0.000001848 -0.000000795 +14247 13.94904 -0.000001847 -0.000000794 +14248 13.95001 -0.000001846 -0.000000794 +14249 13.95097 -0.000001845 -0.000000794 +14250 13.95194 -0.000001844 -0.000000793 +14251 13.95290 -0.000001844 -0.000000793 +14252 13.95387 -0.000001843 -0.000000792 +14253 13.95483 -0.000001842 -0.000000792 +14254 13.95580 -0.000001841 -0.000000792 +14255 13.95676 -0.000001841 -0.000000791 +14256 13.95773 -0.000001840 -0.000000791 +14257 13.95869 -0.000001839 -0.000000791 +14258 13.95966 -0.000001838 -0.000000790 +14259 13.96062 -0.000001838 -0.000000790 +14260 13.96159 -0.000001837 -0.000000789 +14261 13.96255 -0.000001836 -0.000000789 +14262 13.96352 -0.000001835 -0.000000789 +14263 13.96448 -0.000001835 -0.000000788 +14264 13.96545 -0.000001834 -0.000000788 +14265 13.96641 -0.000001833 -0.000000787 +14266 13.96738 -0.000001832 -0.000000787 +14267 13.96834 -0.000001832 -0.000000787 +14268 13.96931 -0.000001831 -0.000000786 +14269 13.97027 -0.000001830 -0.000000786 +14270 13.97124 -0.000001829 -0.000000786 +14271 13.97220 -0.000001828 -0.000000785 +14272 13.97317 -0.000001828 -0.000000785 +14273 13.97413 -0.000001827 -0.000000784 +14274 13.97510 -0.000001826 -0.000000784 +14275 13.97606 -0.000001825 -0.000000784 +14276 13.97703 -0.000001825 -0.000000783 +14277 13.97799 -0.000001824 -0.000000783 +14278 13.97896 -0.000001823 -0.000000783 +14279 13.97992 -0.000001822 -0.000000782 +14280 13.98089 -0.000001822 -0.000000782 +14281 13.98185 -0.000001821 -0.000000781 +14282 13.98282 -0.000001820 -0.000000781 +14283 13.98378 -0.000001819 -0.000000781 +14284 13.98475 -0.000001819 -0.000000780 +14285 13.98571 -0.000001818 -0.000000780 +14286 13.98668 -0.000001817 -0.000000780 +14287 13.98764 -0.000001816 -0.000000779 +14288 13.98861 -0.000001816 -0.000000779 +14289 13.98957 -0.000001815 -0.000000778 +14290 13.99054 -0.000001814 -0.000000778 +14291 13.99150 -0.000001813 -0.000000778 +14292 13.99247 -0.000001813 -0.000000777 +14293 13.99343 -0.000001812 -0.000000777 +14294 13.99440 -0.000001811 -0.000000777 +14295 13.99536 -0.000001810 -0.000000776 +14296 13.99633 -0.000001810 -0.000000776 +14297 13.99729 -0.000001809 -0.000000775 +14298 13.99826 -0.000001808 -0.000000775 +14299 13.99922 -0.000001807 -0.000000775 +14300 14.00019 -0.000001807 -0.000000774 +14301 14.00116 -0.000001806 -0.000000774 +14302 14.00212 -0.000001805 -0.000000774 +14303 14.00309 -0.000001804 -0.000000773 +14304 14.00405 -0.000001804 -0.000000773 +14305 14.00502 -0.000001803 -0.000000772 +14306 14.00598 -0.000001802 -0.000000772 +14307 14.00695 -0.000001801 -0.000000772 +14308 14.00791 -0.000001801 -0.000000771 +14309 14.00888 -0.000001800 -0.000000771 +14310 14.00984 -0.000001799 -0.000000771 +14311 14.01081 -0.000001798 -0.000000770 +14312 14.01177 -0.000001798 -0.000000770 +14313 14.01274 -0.000001797 -0.000000769 +14314 14.01370 -0.000001796 -0.000000769 +14315 14.01467 -0.000001796 -0.000000769 +14316 14.01563 -0.000001795 -0.000000768 +14317 14.01660 -0.000001794 -0.000000768 +14318 14.01756 -0.000001793 -0.000000768 +14319 14.01853 -0.000001793 -0.000000767 +14320 14.01949 -0.000001792 -0.000000767 +14321 14.02046 -0.000001791 -0.000000766 +14322 14.02142 -0.000001790 -0.000000766 +14323 14.02239 -0.000001790 -0.000000766 +14324 14.02335 -0.000001789 -0.000000765 +14325 14.02432 -0.000001788 -0.000000765 +14326 14.02528 -0.000001787 -0.000000765 +14327 14.02625 -0.000001787 -0.000000764 +14328 14.02721 -0.000001786 -0.000000764 +14329 14.02818 -0.000001785 -0.000000764 +14330 14.02914 -0.000001784 -0.000000763 +14331 14.03011 -0.000001784 -0.000000763 +14332 14.03107 -0.000001783 -0.000000762 +14333 14.03204 -0.000001782 -0.000000762 +14334 14.03300 -0.000001781 -0.000000762 +14335 14.03397 -0.000001781 -0.000000761 +14336 14.03493 -0.000001780 -0.000000761 +14337 14.03590 -0.000001779 -0.000000761 +14338 14.03686 -0.000001779 -0.000000760 +14339 14.03783 -0.000001778 -0.000000760 +14340 14.03879 -0.000001777 -0.000000759 +14341 14.03976 -0.000001776 -0.000000759 +14342 14.04072 -0.000001776 -0.000000759 +14343 14.04169 -0.000001775 -0.000000758 +14344 14.04265 -0.000001774 -0.000000758 +14345 14.04362 -0.000001773 -0.000000758 +14346 14.04458 -0.000001773 -0.000000757 +14347 14.04555 -0.000001772 -0.000000757 +14348 14.04651 -0.000001771 -0.000000757 +14349 14.04748 -0.000001770 -0.000000756 +14350 14.04844 -0.000001770 -0.000000756 +14351 14.04941 -0.000001769 -0.000000755 +14352 14.05037 -0.000001768 -0.000000755 +14353 14.05134 -0.000001768 -0.000000755 +14354 14.05230 -0.000001767 -0.000000754 +14355 14.05327 -0.000001766 -0.000000754 +14356 14.05423 -0.000001765 -0.000000754 +14357 14.05520 -0.000001765 -0.000000753 +14358 14.05616 -0.000001764 -0.000000753 +14359 14.05713 -0.000001763 -0.000000753 +14360 14.05809 -0.000001762 -0.000000752 +14361 14.05906 -0.000001762 -0.000000752 +14362 14.06002 -0.000001761 -0.000000752 +14363 14.06099 -0.000001760 -0.000000751 +14364 14.06195 -0.000001760 -0.000000751 +14365 14.06292 -0.000001759 -0.000000750 +14366 14.06388 -0.000001758 -0.000000750 +14367 14.06485 -0.000001757 -0.000000750 +14368 14.06581 -0.000001757 -0.000000749 +14369 14.06678 -0.000001756 -0.000000749 +14370 14.06774 -0.000001755 -0.000000749 +14371 14.06871 -0.000001755 -0.000000748 +14372 14.06967 -0.000001754 -0.000000748 +14373 14.07064 -0.000001753 -0.000000748 +14374 14.07160 -0.000001752 -0.000000747 +14375 14.07257 -0.000001752 -0.000000747 +14376 14.07353 -0.000001751 -0.000000746 +14377 14.07450 -0.000001750 -0.000000746 +14378 14.07546 -0.000001749 -0.000000746 +14379 14.07643 -0.000001749 -0.000000745 +14380 14.07739 -0.000001748 -0.000000745 +14381 14.07836 -0.000001747 -0.000000745 +14382 14.07932 -0.000001747 -0.000000744 +14383 14.08029 -0.000001746 -0.000000744 +14384 14.08125 -0.000001745 -0.000000744 +14385 14.08222 -0.000001744 -0.000000743 +14386 14.08318 -0.000001744 -0.000000743 +14387 14.08415 -0.000001743 -0.000000743 +14388 14.08511 -0.000001742 -0.000000742 +14389 14.08608 -0.000001742 -0.000000742 +14390 14.08704 -0.000001741 -0.000000741 +14391 14.08801 -0.000001740 -0.000000741 +14392 14.08897 -0.000001739 -0.000000741 +14393 14.08994 -0.000001739 -0.000000740 +14394 14.09090 -0.000001738 -0.000000740 +14395 14.09187 -0.000001737 -0.000000740 +14396 14.09283 -0.000001737 -0.000000739 +14397 14.09380 -0.000001736 -0.000000739 +14398 14.09476 -0.000001735 -0.000000739 +14399 14.09573 -0.000001734 -0.000000738 +14400 14.09669 -0.000001734 -0.000000738 +14401 14.09766 -0.000001733 -0.000000738 +14402 14.09862 -0.000001732 -0.000000737 +14403 14.09959 -0.000001732 -0.000000737 +14404 14.10056 -0.000001731 -0.000000737 +14405 14.10152 -0.000001730 -0.000000736 +14406 14.10249 -0.000001729 -0.000000736 +14407 14.10345 -0.000001729 -0.000000735 +14408 14.10442 -0.000001728 -0.000000735 +14409 14.10538 -0.000001727 -0.000000735 +14410 14.10635 -0.000001727 -0.000000734 +14411 14.10731 -0.000001726 -0.000000734 +14412 14.10828 -0.000001725 -0.000000734 +14413 14.10924 -0.000001724 -0.000000733 +14414 14.11021 -0.000001724 -0.000000733 +14415 14.11117 -0.000001723 -0.000000733 +14416 14.11214 -0.000001722 -0.000000732 +14417 14.11310 -0.000001722 -0.000000732 +14418 14.11407 -0.000001721 -0.000000732 +14419 14.11503 -0.000001720 -0.000000731 +14420 14.11600 -0.000001720 -0.000000731 +14421 14.11696 -0.000001719 -0.000000731 +14422 14.11793 -0.000001718 -0.000000730 +14423 14.11889 -0.000001717 -0.000000730 +14424 14.11986 -0.000001717 -0.000000729 +14425 14.12082 -0.000001716 -0.000000729 +14426 14.12179 -0.000001715 -0.000000729 +14427 14.12275 -0.000001715 -0.000000728 +14428 14.12372 -0.000001714 -0.000000728 +14429 14.12468 -0.000001713 -0.000000728 +14430 14.12565 -0.000001713 -0.000000727 +14431 14.12661 -0.000001712 -0.000000727 +14432 14.12758 -0.000001711 -0.000000727 +14433 14.12854 -0.000001710 -0.000000726 +14434 14.12951 -0.000001710 -0.000000726 +14435 14.13047 -0.000001709 -0.000000726 +14436 14.13144 -0.000001708 -0.000000725 +14437 14.13240 -0.000001708 -0.000000725 +14438 14.13337 -0.000001707 -0.000000725 +14439 14.13433 -0.000001706 -0.000000724 +14440 14.13530 -0.000001706 -0.000000724 +14441 14.13626 -0.000001705 -0.000000724 +14442 14.13723 -0.000001704 -0.000000723 +14443 14.13819 -0.000001703 -0.000000723 +14444 14.13916 -0.000001703 -0.000000723 +14445 14.14012 -0.000001702 -0.000000722 +14446 14.14109 -0.000001701 -0.000000722 +14447 14.14205 -0.000001701 -0.000000722 +14448 14.14302 -0.000001700 -0.000000721 +14449 14.14398 -0.000001699 -0.000000721 +14450 14.14495 -0.000001699 -0.000000720 +14451 14.14591 -0.000001698 -0.000000720 +14452 14.14688 -0.000001697 -0.000000720 +14453 14.14784 -0.000001696 -0.000000719 +14454 14.14881 -0.000001696 -0.000000719 +14455 14.14977 -0.000001695 -0.000000719 +14456 14.15074 -0.000001694 -0.000000718 +14457 14.15170 -0.000001694 -0.000000718 +14458 14.15267 -0.000001693 -0.000000718 +14459 14.15363 -0.000001692 -0.000000717 +14460 14.15460 -0.000001692 -0.000000717 +14461 14.15556 -0.000001691 -0.000000717 +14462 14.15653 -0.000001690 -0.000000716 +14463 14.15749 -0.000001690 -0.000000716 +14464 14.15846 -0.000001689 -0.000000716 +14465 14.15942 -0.000001688 -0.000000715 +14466 14.16039 -0.000001687 -0.000000715 +14467 14.16135 -0.000001687 -0.000000715 +14468 14.16232 -0.000001686 -0.000000714 +14469 14.16328 -0.000001685 -0.000000714 +14470 14.16425 -0.000001685 -0.000000714 +14471 14.16521 -0.000001684 -0.000000713 +14472 14.16618 -0.000001683 -0.000000713 +14473 14.16714 -0.000001683 -0.000000713 +14474 14.16811 -0.000001682 -0.000000712 +14475 14.16907 -0.000001681 -0.000000712 +14476 14.17004 -0.000001681 -0.000000712 +14477 14.17100 -0.000001680 -0.000000711 +14478 14.17197 -0.000001679 -0.000000711 +14479 14.17293 -0.000001679 -0.000000711 +14480 14.17390 -0.000001678 -0.000000710 +14481 14.17486 -0.000001677 -0.000000710 +14482 14.17583 -0.000001676 -0.000000710 +14483 14.17679 -0.000001676 -0.000000709 +14484 14.17776 -0.000001675 -0.000000709 +14485 14.17872 -0.000001674 -0.000000709 +14486 14.17969 -0.000001674 -0.000000708 +14487 14.18065 -0.000001673 -0.000000708 +14488 14.18162 -0.000001672 -0.000000708 +14489 14.18258 -0.000001672 -0.000000707 +14490 14.18355 -0.000001671 -0.000000707 +14491 14.18451 -0.000001670 -0.000000707 +14492 14.18548 -0.000001670 -0.000000706 +14493 14.18644 -0.000001669 -0.000000706 +14494 14.18741 -0.000001668 -0.000000706 +14495 14.18837 -0.000001668 -0.000000705 +14496 14.18934 -0.000001667 -0.000000705 +14497 14.19030 -0.000001666 -0.000000705 +14498 14.19127 -0.000001666 -0.000000704 +14499 14.19223 -0.000001665 -0.000000704 +14500 14.19320 -0.000001664 -0.000000704 +14501 14.19416 -0.000001664 -0.000000703 +14502 14.19513 -0.000001663 -0.000000703 +14503 14.19609 -0.000001662 -0.000000703 +14504 14.19706 -0.000001661 -0.000000702 +14505 14.19802 -0.000001661 -0.000000702 +14506 14.19899 -0.000001660 -0.000000702 +14507 14.19995 -0.000001659 -0.000000701 +14508 14.20092 -0.000001659 -0.000000701 +14509 14.20189 -0.000001658 -0.000000701 +14510 14.20285 -0.000001657 -0.000000700 +14511 14.20382 -0.000001657 -0.000000700 +14512 14.20478 -0.000001656 -0.000000700 +14513 14.20575 -0.000001655 -0.000000699 +14514 14.20671 -0.000001655 -0.000000699 +14515 14.20768 -0.000001654 -0.000000699 +14516 14.20864 -0.000001653 -0.000000698 +14517 14.20961 -0.000001653 -0.000000698 +14518 14.21057 -0.000001652 -0.000000698 +14519 14.21154 -0.000001651 -0.000000697 +14520 14.21250 -0.000001651 -0.000000697 +14521 14.21347 -0.000001650 -0.000000697 +14522 14.21443 -0.000001649 -0.000000696 +14523 14.21540 -0.000001649 -0.000000696 +14524 14.21636 -0.000001648 -0.000000696 +14525 14.21733 -0.000001647 -0.000000695 +14526 14.21829 -0.000001647 -0.000000695 +14527 14.21926 -0.000001646 -0.000000695 +14528 14.22022 -0.000001645 -0.000000694 +14529 14.22119 -0.000001645 -0.000000694 +14530 14.22215 -0.000001644 -0.000000694 +14531 14.22312 -0.000001643 -0.000000693 +14532 14.22408 -0.000001643 -0.000000693 +14533 14.22505 -0.000001642 -0.000000693 +14534 14.22601 -0.000001641 -0.000000692 +14535 14.22698 -0.000001641 -0.000000692 +14536 14.22794 -0.000001640 -0.000000692 +14537 14.22891 -0.000001639 -0.000000691 +14538 14.22987 -0.000001639 -0.000000691 +14539 14.23084 -0.000001638 -0.000000691 +14540 14.23180 -0.000001637 -0.000000690 +14541 14.23277 -0.000001637 -0.000000690 +14542 14.23373 -0.000001636 -0.000000690 +14543 14.23470 -0.000001635 -0.000000689 +14544 14.23566 -0.000001635 -0.000000689 +14545 14.23663 -0.000001634 -0.000000689 +14546 14.23759 -0.000001633 -0.000000688 +14547 14.23856 -0.000001633 -0.000000688 +14548 14.23952 -0.000001632 -0.000000688 +14549 14.24049 -0.000001631 -0.000000687 +14550 14.24145 -0.000001631 -0.000000687 +14551 14.24242 -0.000001630 -0.000000687 +14552 14.24338 -0.000001629 -0.000000686 +14553 14.24435 -0.000001629 -0.000000686 +14554 14.24531 -0.000001628 -0.000000686 +14555 14.24628 -0.000001627 -0.000000685 +14556 14.24724 -0.000001627 -0.000000685 +14557 14.24821 -0.000001626 -0.000000685 +14558 14.24917 -0.000001625 -0.000000684 +14559 14.25014 -0.000001625 -0.000000684 +14560 14.25110 -0.000001624 -0.000000684 +14561 14.25207 -0.000001623 -0.000000683 +14562 14.25303 -0.000001623 -0.000000683 +14563 14.25400 -0.000001622 -0.000000683 +14564 14.25496 -0.000001621 -0.000000682 +14565 14.25593 -0.000001621 -0.000000682 +14566 14.25689 -0.000001620 -0.000000682 +14567 14.25786 -0.000001619 -0.000000681 +14568 14.25882 -0.000001619 -0.000000681 +14569 14.25979 -0.000001618 -0.000000681 +14570 14.26075 -0.000001617 -0.000000681 +14571 14.26172 -0.000001617 -0.000000680 +14572 14.26268 -0.000001616 -0.000000680 +14573 14.26365 -0.000001615 -0.000000680 +14574 14.26461 -0.000001615 -0.000000679 +14575 14.26558 -0.000001614 -0.000000679 +14576 14.26654 -0.000001614 -0.000000679 +14577 14.26751 -0.000001613 -0.000000678 +14578 14.26847 -0.000001612 -0.000000678 +14579 14.26944 -0.000001612 -0.000000678 +14580 14.27040 -0.000001611 -0.000000677 +14581 14.27137 -0.000001610 -0.000000677 +14582 14.27233 -0.000001610 -0.000000677 +14583 14.27330 -0.000001609 -0.000000676 +14584 14.27426 -0.000001608 -0.000000676 +14585 14.27523 -0.000001608 -0.000000676 +14586 14.27619 -0.000001607 -0.000000675 +14587 14.27716 -0.000001606 -0.000000675 +14588 14.27812 -0.000001606 -0.000000675 +14589 14.27909 -0.000001605 -0.000000674 +14590 14.28005 -0.000001604 -0.000000674 +14591 14.28102 -0.000001604 -0.000000674 +14592 14.28198 -0.000001603 -0.000000673 +14593 14.28295 -0.000001602 -0.000000673 +14594 14.28391 -0.000001602 -0.000000673 +14595 14.28488 -0.000001601 -0.000000673 +14596 14.28584 -0.000001600 -0.000000672 +14597 14.28681 -0.000001600 -0.000000672 +14598 14.28777 -0.000001599 -0.000000672 +14599 14.28874 -0.000001599 -0.000000671 +14600 14.28970 -0.000001598 -0.000000671 +14601 14.29067 -0.000001597 -0.000000671 +14602 14.29163 -0.000001597 -0.000000670 +14603 14.29260 -0.000001596 -0.000000670 +14604 14.29356 -0.000001595 -0.000000670 +14605 14.29453 -0.000001595 -0.000000669 +14606 14.29549 -0.000001594 -0.000000669 +14607 14.29646 -0.000001593 -0.000000669 +14608 14.29742 -0.000001593 -0.000000668 +14609 14.29839 -0.000001592 -0.000000668 +14610 14.29935 -0.000001591 -0.000000668 +14611 14.30032 -0.000001591 -0.000000667 +14612 14.30129 -0.000001590 -0.000000667 +14613 14.30225 -0.000001589 -0.000000667 +14614 14.30322 -0.000001589 -0.000000666 +14615 14.30418 -0.000001588 -0.000000666 +14616 14.30515 -0.000001588 -0.000000666 +14617 14.30611 -0.000001587 -0.000000666 +14618 14.30708 -0.000001586 -0.000000665 +14619 14.30804 -0.000001586 -0.000000665 +14620 14.30901 -0.000001585 -0.000000665 +14621 14.30997 -0.000001584 -0.000000664 +14622 14.31094 -0.000001584 -0.000000664 +14623 14.31190 -0.000001583 -0.000000664 +14624 14.31287 -0.000001582 -0.000000663 +14625 14.31383 -0.000001582 -0.000000663 +14626 14.31480 -0.000001581 -0.000000663 +14627 14.31576 -0.000001581 -0.000000662 +14628 14.31673 -0.000001580 -0.000000662 +14629 14.31769 -0.000001579 -0.000000662 +14630 14.31866 -0.000001579 -0.000000661 +14631 14.31962 -0.000001578 -0.000000661 +14632 14.32059 -0.000001577 -0.000000661 +14633 14.32155 -0.000001577 -0.000000661 +14634 14.32252 -0.000001576 -0.000000660 +14635 14.32348 -0.000001575 -0.000000660 +14636 14.32445 -0.000001575 -0.000000660 +14637 14.32541 -0.000001574 -0.000000659 +14638 14.32638 -0.000001573 -0.000000659 +14639 14.32734 -0.000001573 -0.000000659 +14640 14.32831 -0.000001572 -0.000000658 +14641 14.32927 -0.000001572 -0.000000658 +14642 14.33024 -0.000001571 -0.000000658 +14643 14.33120 -0.000001570 -0.000000657 +14644 14.33217 -0.000001570 -0.000000657 +14645 14.33313 -0.000001569 -0.000000657 +14646 14.33410 -0.000001568 -0.000000657 +14647 14.33506 -0.000001568 -0.000000656 +14648 14.33603 -0.000001567 -0.000000656 +14649 14.33699 -0.000001567 -0.000000656 +14650 14.33796 -0.000001566 -0.000000655 +14651 14.33892 -0.000001565 -0.000000655 +14652 14.33989 -0.000001565 -0.000000655 +14653 14.34085 -0.000001564 -0.000000654 +14654 14.34182 -0.000001563 -0.000000654 +14655 14.34278 -0.000001563 -0.000000654 +14656 14.34375 -0.000001562 -0.000000653 +14657 14.34471 -0.000001561 -0.000000653 +14658 14.34568 -0.000001561 -0.000000653 +14659 14.34664 -0.000001560 -0.000000653 +14660 14.34761 -0.000001560 -0.000000652 +14661 14.34857 -0.000001559 -0.000000652 +14662 14.34954 -0.000001558 -0.000000652 +14663 14.35050 -0.000001558 -0.000000651 +14664 14.35147 -0.000001557 -0.000000651 +14665 14.35243 -0.000001556 -0.000000651 +14666 14.35340 -0.000001556 -0.000000650 +14667 14.35436 -0.000001555 -0.000000650 +14668 14.35533 -0.000001555 -0.000000650 +14669 14.35629 -0.000001554 -0.000000649 +14670 14.35726 -0.000001553 -0.000000649 +14671 14.35822 -0.000001553 -0.000000649 +14672 14.35919 -0.000001552 -0.000000649 +14673 14.36015 -0.000001551 -0.000000648 +14674 14.36112 -0.000001551 -0.000000648 +14675 14.36208 -0.000001550 -0.000000648 +14676 14.36305 -0.000001550 -0.000000647 +14677 14.36401 -0.000001549 -0.000000647 +14678 14.36498 -0.000001548 -0.000000647 +14679 14.36594 -0.000001548 -0.000000646 +14680 14.36691 -0.000001547 -0.000000646 +14681 14.36787 -0.000001546 -0.000000646 +14682 14.36884 -0.000001546 -0.000000645 +14683 14.36980 -0.000001545 -0.000000645 +14684 14.37077 -0.000001545 -0.000000645 +14685 14.37173 -0.000001544 -0.000000645 +14686 14.37270 -0.000001543 -0.000000644 +14687 14.37366 -0.000001543 -0.000000644 +14688 14.37463 -0.000001542 -0.000000644 +14689 14.37559 -0.000001541 -0.000000643 +14690 14.37656 -0.000001541 -0.000000643 +14691 14.37752 -0.000001540 -0.000000643 +14692 14.37849 -0.000001540 -0.000000642 +14693 14.37945 -0.000001539 -0.000000642 +14694 14.38042 -0.000001538 -0.000000642 +14695 14.38138 -0.000001538 -0.000000642 +14696 14.38235 -0.000001537 -0.000000641 +14697 14.38331 -0.000001536 -0.000000641 +14698 14.38428 -0.000001536 -0.000000641 +14699 14.38524 -0.000001535 -0.000000640 +14700 14.38621 -0.000001535 -0.000000640 +14701 14.38717 -0.000001534 -0.000000640 +14702 14.38814 -0.000001533 -0.000000639 +14703 14.38910 -0.000001533 -0.000000639 +14704 14.39007 -0.000001532 -0.000000639 +14705 14.39103 -0.000001532 -0.000000639 +14706 14.39200 -0.000001531 -0.000000638 +14707 14.39296 -0.000001530 -0.000000638 +14708 14.39393 -0.000001530 -0.000000638 +14709 14.39489 -0.000001529 -0.000000637 +14710 14.39586 -0.000001528 -0.000000637 +14711 14.39682 -0.000001528 -0.000000637 +14712 14.39779 -0.000001527 -0.000000636 +14713 14.39875 -0.000001527 -0.000000636 +14714 14.39972 -0.000001526 -0.000000636 +14715 14.40069 -0.000001525 -0.000000636 +14716 14.40165 -0.000001525 -0.000000635 +14717 14.40262 -0.000001524 -0.000000635 +14718 14.40358 -0.000001524 -0.000000635 +14719 14.40455 -0.000001523 -0.000000634 +14720 14.40551 -0.000001522 -0.000000634 +14721 14.40648 -0.000001522 -0.000000634 +14722 14.40744 -0.000001521 -0.000000633 +14723 14.40841 -0.000001521 -0.000000633 +14724 14.40937 -0.000001520 -0.000000633 +14725 14.41034 -0.000001519 -0.000000633 +14726 14.41130 -0.000001519 -0.000000632 +14727 14.41227 -0.000001518 -0.000000632 +14728 14.41323 -0.000001517 -0.000000632 +14729 14.41420 -0.000001517 -0.000000631 +14730 14.41516 -0.000001516 -0.000000631 +14731 14.41613 -0.000001516 -0.000000631 +14732 14.41709 -0.000001515 -0.000000631 +14733 14.41806 -0.000001514 -0.000000630 +14734 14.41902 -0.000001514 -0.000000630 +14735 14.41999 -0.000001513 -0.000000630 +14736 14.42095 -0.000001513 -0.000000629 +14737 14.42192 -0.000001512 -0.000000629 +14738 14.42288 -0.000001511 -0.000000629 +14739 14.42385 -0.000001511 -0.000000628 +14740 14.42481 -0.000001510 -0.000000628 +14741 14.42578 -0.000001510 -0.000000628 +14742 14.42674 -0.000001509 -0.000000628 +14743 14.42771 -0.000001508 -0.000000627 +14744 14.42867 -0.000001508 -0.000000627 +14745 14.42964 -0.000001507 -0.000000627 +14746 14.43060 -0.000001507 -0.000000626 +14747 14.43157 -0.000001506 -0.000000626 +14748 14.43253 -0.000001505 -0.000000626 +14749 14.43350 -0.000001505 -0.000000626 +14750 14.43446 -0.000001504 -0.000000625 +14751 14.43543 -0.000001504 -0.000000625 +14752 14.43639 -0.000001503 -0.000000625 +14753 14.43736 -0.000001502 -0.000000624 +14754 14.43832 -0.000001502 -0.000000624 +14755 14.43929 -0.000001501 -0.000000624 +14756 14.44025 -0.000001500 -0.000000623 +14757 14.44122 -0.000001500 -0.000000623 +14758 14.44218 -0.000001499 -0.000000623 +14759 14.44315 -0.000001499 -0.000000623 +14760 14.44411 -0.000001498 -0.000000622 +14761 14.44508 -0.000001497 -0.000000622 +14762 14.44604 -0.000001497 -0.000000622 +14763 14.44701 -0.000001496 -0.000000621 +14764 14.44797 -0.000001496 -0.000000621 +14765 14.44894 -0.000001495 -0.000000621 +14766 14.44990 -0.000001494 -0.000000621 +14767 14.45087 -0.000001494 -0.000000620 +14768 14.45183 -0.000001493 -0.000000620 +14769 14.45280 -0.000001493 -0.000000620 +14770 14.45376 -0.000001492 -0.000000619 +14771 14.45473 -0.000001491 -0.000000619 +14772 14.45569 -0.000001491 -0.000000619 +14773 14.45666 -0.000001490 -0.000000619 +14774 14.45762 -0.000001490 -0.000000618 +14775 14.45859 -0.000001489 -0.000000618 +14776 14.45955 -0.000001489 -0.000000618 +14777 14.46052 -0.000001488 -0.000000617 +14778 14.46148 -0.000001487 -0.000000617 +14779 14.46245 -0.000001487 -0.000000617 +14780 14.46341 -0.000001486 -0.000000617 +14781 14.46438 -0.000001486 -0.000000616 +14782 14.46534 -0.000001485 -0.000000616 +14783 14.46631 -0.000001484 -0.000000616 +14784 14.46727 -0.000001484 -0.000000615 +14785 14.46824 -0.000001483 -0.000000615 +14786 14.46920 -0.000001483 -0.000000615 +14787 14.47017 -0.000001482 -0.000000614 +14788 14.47113 -0.000001481 -0.000000614 +14789 14.47210 -0.000001481 -0.000000614 +14790 14.47306 -0.000001480 -0.000000614 +14791 14.47403 -0.000001480 -0.000000613 +14792 14.47499 -0.000001479 -0.000000613 +14793 14.47596 -0.000001478 -0.000000613 +14794 14.47692 -0.000001478 -0.000000612 +14795 14.47789 -0.000001477 -0.000000612 +14796 14.47885 -0.000001477 -0.000000612 +14797 14.47982 -0.000001476 -0.000000612 +14798 14.48078 -0.000001475 -0.000000611 +14799 14.48175 -0.000001475 -0.000000611 +14800 14.48271 -0.000001474 -0.000000611 +14801 14.48368 -0.000001474 -0.000000610 +14802 14.48464 -0.000001473 -0.000000610 +14803 14.48561 -0.000001473 -0.000000610 +14804 14.48657 -0.000001472 -0.000000610 +14805 14.48754 -0.000001471 -0.000000609 +14806 14.48850 -0.000001471 -0.000000609 +14807 14.48947 -0.000001470 -0.000000609 +14808 14.49043 -0.000001470 -0.000000609 +14809 14.49140 -0.000001469 -0.000000608 +14810 14.49236 -0.000001468 -0.000000608 +14811 14.49333 -0.000001468 -0.000000608 +14812 14.49429 -0.000001467 -0.000000607 +14813 14.49526 -0.000001467 -0.000000607 +14814 14.49622 -0.000001466 -0.000000607 +14815 14.49719 -0.000001465 -0.000000607 +14816 14.49815 -0.000001465 -0.000000606 +14817 14.49912 -0.000001464 -0.000000606 +14818 14.50009 -0.000001464 -0.000000606 +14819 14.50105 -0.000001463 -0.000000605 +14820 14.50202 -0.000001463 -0.000000605 +14821 14.50298 -0.000001462 -0.000000605 +14822 14.50395 -0.000001461 -0.000000605 +14823 14.50491 -0.000001461 -0.000000604 +14824 14.50588 -0.000001460 -0.000000604 +14825 14.50684 -0.000001460 -0.000000604 +14826 14.50781 -0.000001459 -0.000000603 +14827 14.50877 -0.000001458 -0.000000603 +14828 14.50974 -0.000001458 -0.000000603 +14829 14.51070 -0.000001457 -0.000000603 +14830 14.51167 -0.000001457 -0.000000602 +14831 14.51263 -0.000001456 -0.000000602 +14832 14.51360 -0.000001456 -0.000000602 +14833 14.51456 -0.000001455 -0.000000601 +14834 14.51553 -0.000001454 -0.000000601 +14835 14.51649 -0.000001454 -0.000000601 +14836 14.51746 -0.000001453 -0.000000601 +14837 14.51842 -0.000001453 -0.000000600 +14838 14.51939 -0.000001452 -0.000000600 +14839 14.52035 -0.000001452 -0.000000600 +14840 14.52132 -0.000001451 -0.000000600 +14841 14.52228 -0.000001450 -0.000000599 +14842 14.52325 -0.000001450 -0.000000599 +14843 14.52421 -0.000001449 -0.000000599 +14844 14.52518 -0.000001449 -0.000000598 +14845 14.52614 -0.000001448 -0.000000598 +14846 14.52711 -0.000001447 -0.000000598 +14847 14.52807 -0.000001447 -0.000000598 +14848 14.52904 -0.000001446 -0.000000597 +14849 14.53000 -0.000001446 -0.000000597 +14850 14.53097 -0.000001445 -0.000000597 +14851 14.53193 -0.000001445 -0.000000596 +14852 14.53290 -0.000001444 -0.000000596 +14853 14.53386 -0.000001443 -0.000000596 +14854 14.53483 -0.000001443 -0.000000596 +14855 14.53579 -0.000001442 -0.000000595 +14856 14.53676 -0.000001442 -0.000000595 +14857 14.53772 -0.000001441 -0.000000595 +14858 14.53869 -0.000001441 -0.000000595 +14859 14.53965 -0.000001440 -0.000000594 +14860 14.54062 -0.000001439 -0.000000594 +14861 14.54158 -0.000001439 -0.000000594 +14862 14.54255 -0.000001438 -0.000000593 +14863 14.54351 -0.000001438 -0.000000593 +14864 14.54448 -0.000001437 -0.000000593 +14865 14.54544 -0.000001437 -0.000000593 +14866 14.54641 -0.000001436 -0.000000592 +14867 14.54737 -0.000001435 -0.000000592 +14868 14.54834 -0.000001435 -0.000000592 +14869 14.54930 -0.000001434 -0.000000591 +14870 14.55027 -0.000001434 -0.000000591 +14871 14.55123 -0.000001433 -0.000000591 +14872 14.55220 -0.000001433 -0.000000591 +14873 14.55316 -0.000001432 -0.000000590 +14874 14.55413 -0.000001431 -0.000000590 +14875 14.55509 -0.000001431 -0.000000590 +14876 14.55606 -0.000001430 -0.000000590 +14877 14.55702 -0.000001430 -0.000000589 +14878 14.55799 -0.000001429 -0.000000589 +14879 14.55895 -0.000001429 -0.000000589 +14880 14.55992 -0.000001428 -0.000000588 +14881 14.56088 -0.000001427 -0.000000588 +14882 14.56185 -0.000001427 -0.000000588 +14883 14.56281 -0.000001426 -0.000000588 +14884 14.56378 -0.000001426 -0.000000587 +14885 14.56474 -0.000001425 -0.000000587 +14886 14.56571 -0.000001425 -0.000000587 +14887 14.56667 -0.000001424 -0.000000587 +14888 14.56764 -0.000001423 -0.000000586 +14889 14.56860 -0.000001423 -0.000000586 +14890 14.56957 -0.000001422 -0.000000586 +14891 14.57053 -0.000001422 -0.000000585 +14892 14.57150 -0.000001421 -0.000000585 +14893 14.57246 -0.000001421 -0.000000585 +14894 14.57343 -0.000001420 -0.000000585 +14895 14.57439 -0.000001420 -0.000000584 +14896 14.57536 -0.000001419 -0.000000584 +14897 14.57632 -0.000001418 -0.000000584 +14898 14.57729 -0.000001418 -0.000000584 +14899 14.57825 -0.000001417 -0.000000583 +14900 14.57922 -0.000001417 -0.000000583 +14901 14.58018 -0.000001416 -0.000000583 +14902 14.58115 -0.000001416 -0.000000582 +14903 14.58211 -0.000001415 -0.000000582 +14904 14.58308 -0.000001414 -0.000000582 +14905 14.58404 -0.000001414 -0.000000582 +14906 14.58501 -0.000001413 -0.000000581 +14907 14.58597 -0.000001413 -0.000000581 +14908 14.58694 -0.000001412 -0.000000581 +14909 14.58790 -0.000001412 -0.000000581 +14910 14.58887 -0.000001411 -0.000000580 +14911 14.58983 -0.000001411 -0.000000580 +14912 14.59080 -0.000001410 -0.000000580 +14913 14.59176 -0.000001409 -0.000000580 +14914 14.59273 -0.000001409 -0.000000579 +14915 14.59369 -0.000001408 -0.000000579 +14916 14.59466 -0.000001408 -0.000000579 +14917 14.59562 -0.000001407 -0.000000578 +14918 14.59659 -0.000001407 -0.000000578 +14919 14.59755 -0.000001406 -0.000000578 +14920 14.59852 -0.000001405 -0.000000578 +14921 14.59948 -0.000001405 -0.000000577 +14922 14.60045 -0.000001404 -0.000000577 +14923 14.60142 -0.000001404 -0.000000577 +14924 14.60238 -0.000001403 -0.000000577 +14925 14.60335 -0.000001403 -0.000000576 +14926 14.60431 -0.000001402 -0.000000576 +14927 14.60528 -0.000001402 -0.000000576 +14928 14.60624 -0.000001401 -0.000000576 +14929 14.60721 -0.000001400 -0.000000575 +14930 14.60817 -0.000001400 -0.000000575 +14931 14.60914 -0.000001399 -0.000000575 +14932 14.61010 -0.000001399 -0.000000574 +14933 14.61107 -0.000001398 -0.000000574 +14934 14.61203 -0.000001398 -0.000000574 +14935 14.61300 -0.000001397 -0.000000574 +14936 14.61396 -0.000001397 -0.000000573 +14937 14.61493 -0.000001396 -0.000000573 +14938 14.61589 -0.000001396 -0.000000573 +14939 14.61686 -0.000001395 -0.000000573 +14940 14.61782 -0.000001394 -0.000000572 +14941 14.61879 -0.000001394 -0.000000572 +14942 14.61975 -0.000001393 -0.000000572 +14943 14.62072 -0.000001393 -0.000000572 +14944 14.62168 -0.000001392 -0.000000571 +14945 14.62265 -0.000001392 -0.000000571 +14946 14.62361 -0.000001391 -0.000000571 +14947 14.62458 -0.000001391 -0.000000570 +14948 14.62554 -0.000001390 -0.000000570 +14949 14.62651 -0.000001389 -0.000000570 +14950 14.62747 -0.000001389 -0.000000570 +14951 14.62844 -0.000001388 -0.000000569 +14952 14.62940 -0.000001388 -0.000000569 +14953 14.63037 -0.000001387 -0.000000569 +14954 14.63133 -0.000001387 -0.000000569 +14955 14.63230 -0.000001386 -0.000000568 +14956 14.63326 -0.000001386 -0.000000568 +14957 14.63423 -0.000001385 -0.000000568 +14958 14.63519 -0.000001384 -0.000000568 +14959 14.63616 -0.000001384 -0.000000567 +14960 14.63712 -0.000001383 -0.000000567 +14961 14.63809 -0.000001383 -0.000000567 +14962 14.63905 -0.000001382 -0.000000567 +14963 14.64002 -0.000001382 -0.000000566 +14964 14.64098 -0.000001381 -0.000000566 +14965 14.64195 -0.000001381 -0.000000566 +14966 14.64291 -0.000001380 -0.000000566 +14967 14.64388 -0.000001380 -0.000000565 +14968 14.64484 -0.000001379 -0.000000565 +14969 14.64581 -0.000001378 -0.000000565 +14970 14.64677 -0.000001378 -0.000000564 +14971 14.64774 -0.000001377 -0.000000564 +14972 14.64870 -0.000001377 -0.000000564 +14973 14.64967 -0.000001376 -0.000000564 +14974 14.65063 -0.000001376 -0.000000563 +14975 14.65160 -0.000001375 -0.000000563 +14976 14.65256 -0.000001375 -0.000000563 +14977 14.65353 -0.000001374 -0.000000563 +14978 14.65449 -0.000001374 -0.000000562 +14979 14.65546 -0.000001373 -0.000000562 +14980 14.65642 -0.000001373 -0.000000562 +14981 14.65739 -0.000001372 -0.000000562 +14982 14.65835 -0.000001371 -0.000000561 +14983 14.65932 -0.000001371 -0.000000561 +14984 14.66028 -0.000001370 -0.000000561 +14985 14.66125 -0.000001370 -0.000000561 +14986 14.66221 -0.000001369 -0.000000560 +14987 14.66318 -0.000001369 -0.000000560 +14988 14.66414 -0.000001368 -0.000000560 +14989 14.66511 -0.000001368 -0.000000560 +14990 14.66607 -0.000001367 -0.000000559 +14991 14.66704 -0.000001367 -0.000000559 +14992 14.66800 -0.000001366 -0.000000559 +14993 14.66897 -0.000001365 -0.000000559 +14994 14.66993 -0.000001365 -0.000000558 +14995 14.67090 -0.000001364 -0.000000558 +14996 14.67186 -0.000001364 -0.000000558 +14997 14.67283 -0.000001363 -0.000000557 +14998 14.67379 -0.000001363 -0.000000557 +14999 14.67476 -0.000001362 -0.000000557 +15000 14.67572 -0.000001362 -0.000000557 +15001 14.67669 -0.000001361 -0.000000556 +15002 14.67765 -0.000001361 -0.000000556 +15003 14.67862 -0.000001360 -0.000000556 +15004 14.67958 -0.000001360 -0.000000556 +15005 14.68055 -0.000001359 -0.000000555 +15006 14.68151 -0.000001358 -0.000000555 +15007 14.68248 -0.000001358 -0.000000555 +15008 14.68344 -0.000001357 -0.000000555 +15009 14.68441 -0.000001357 -0.000000554 +15010 14.68537 -0.000001356 -0.000000554 +15011 14.68634 -0.000001356 -0.000000554 +15012 14.68730 -0.000001355 -0.000000554 +15013 14.68827 -0.000001355 -0.000000553 +15014 14.68923 -0.000001354 -0.000000553 +15015 14.69020 -0.000001354 -0.000000553 +15016 14.69116 -0.000001353 -0.000000553 +15017 14.69213 -0.000001353 -0.000000552 +15018 14.69309 -0.000001352 -0.000000552 +15019 14.69406 -0.000001352 -0.000000552 +15020 14.69502 -0.000001351 -0.000000552 +15021 14.69599 -0.000001350 -0.000000551 +15022 14.69695 -0.000001350 -0.000000551 +15023 14.69792 -0.000001349 -0.000000551 +15024 14.69888 -0.000001349 -0.000000551 +15025 14.69985 -0.000001348 -0.000000550 +15026 14.70082 -0.000001348 -0.000000550 +15027 14.70178 -0.000001347 -0.000000550 +15028 14.70275 -0.000001347 -0.000000550 +15029 14.70371 -0.000001346 -0.000000549 +15030 14.70468 -0.000001346 -0.000000549 +15031 14.70564 -0.000001345 -0.000000549 +15032 14.70661 -0.000001345 -0.000000549 +15033 14.70757 -0.000001344 -0.000000548 +15034 14.70854 -0.000001344 -0.000000548 +15035 14.70950 -0.000001343 -0.000000548 +15036 14.71047 -0.000001343 -0.000000548 +15037 14.71143 -0.000001342 -0.000000547 +15038 14.71240 -0.000001341 -0.000000547 +15039 14.71336 -0.000001341 -0.000000547 +15040 14.71433 -0.000001340 -0.000000547 +15041 14.71529 -0.000001340 -0.000000546 +15042 14.71626 -0.000001339 -0.000000546 +15043 14.71722 -0.000001339 -0.000000546 +15044 14.71819 -0.000001338 -0.000000546 +15045 14.71915 -0.000001338 -0.000000545 +15046 14.72012 -0.000001337 -0.000000545 +15047 14.72108 -0.000001337 -0.000000545 +15048 14.72205 -0.000001336 -0.000000545 +15049 14.72301 -0.000001336 -0.000000544 +15050 14.72398 -0.000001335 -0.000000544 +15051 14.72494 -0.000001335 -0.000000544 +15052 14.72591 -0.000001334 -0.000000544 +15053 14.72687 -0.000001334 -0.000000543 +15054 14.72784 -0.000001333 -0.000000543 +15055 14.72880 -0.000001333 -0.000000543 +15056 14.72977 -0.000001332 -0.000000543 +15057 14.73073 -0.000001331 -0.000000542 +15058 14.73170 -0.000001331 -0.000000542 +15059 14.73266 -0.000001330 -0.000000542 +15060 14.73363 -0.000001330 -0.000000542 +15061 14.73459 -0.000001329 -0.000000541 +15062 14.73556 -0.000001329 -0.000000541 +15063 14.73652 -0.000001328 -0.000000541 +15064 14.73749 -0.000001328 -0.000000541 +15065 14.73845 -0.000001327 -0.000000540 +15066 14.73942 -0.000001327 -0.000000540 +15067 14.74038 -0.000001326 -0.000000540 +15068 14.74135 -0.000001326 -0.000000540 +15069 14.74231 -0.000001325 -0.000000539 +15070 14.74328 -0.000001325 -0.000000539 +15071 14.74424 -0.000001324 -0.000000539 +15072 14.74521 -0.000001324 -0.000000539 +15073 14.74617 -0.000001323 -0.000000538 +15074 14.74714 -0.000001323 -0.000000538 +15075 14.74810 -0.000001322 -0.000000538 +15076 14.74907 -0.000001322 -0.000000538 +15077 14.75003 -0.000001321 -0.000000537 +15078 14.75100 -0.000001321 -0.000000537 +15079 14.75196 -0.000001320 -0.000000537 +15080 14.75293 -0.000001320 -0.000000537 +15081 14.75389 -0.000001319 -0.000000536 +15082 14.75486 -0.000001318 -0.000000536 +15083 14.75582 -0.000001318 -0.000000536 +15084 14.75679 -0.000001317 -0.000000536 +15085 14.75775 -0.000001317 -0.000000535 +15086 14.75872 -0.000001316 -0.000000535 +15087 14.75968 -0.000001316 -0.000000535 +15088 14.76065 -0.000001315 -0.000000535 +15089 14.76161 -0.000001315 -0.000000534 +15090 14.76258 -0.000001314 -0.000000534 +15091 14.76354 -0.000001314 -0.000000534 +15092 14.76451 -0.000001313 -0.000000534 +15093 14.76547 -0.000001313 -0.000000533 +15094 14.76644 -0.000001312 -0.000000533 +15095 14.76740 -0.000001312 -0.000000533 +15096 14.76837 -0.000001311 -0.000000533 +15097 14.76933 -0.000001311 -0.000000532 +15098 14.77030 -0.000001310 -0.000000532 +15099 14.77126 -0.000001310 -0.000000532 +15100 14.77223 -0.000001309 -0.000000532 +15101 14.77319 -0.000001309 -0.000000532 +15102 14.77416 -0.000001308 -0.000000531 +15103 14.77512 -0.000001308 -0.000000531 +15104 14.77609 -0.000001307 -0.000000531 +15105 14.77705 -0.000001307 -0.000000531 +15106 14.77802 -0.000001306 -0.000000530 +15107 14.77898 -0.000001306 -0.000000530 +15108 14.77995 -0.000001305 -0.000000530 +15109 14.78091 -0.000001305 -0.000000530 +15110 14.78188 -0.000001304 -0.000000529 +15111 14.78284 -0.000001304 -0.000000529 +15112 14.78381 -0.000001303 -0.000000529 +15113 14.78477 -0.000001303 -0.000000529 +15114 14.78574 -0.000001302 -0.000000528 +15115 14.78670 -0.000001302 -0.000000528 +15116 14.78767 -0.000001301 -0.000000528 +15117 14.78863 -0.000001301 -0.000000528 +15118 14.78960 -0.000001300 -0.000000527 +15119 14.79056 -0.000001299 -0.000000527 +15120 14.79153 -0.000001299 -0.000000527 +15121 14.79249 -0.000001298 -0.000000527 +15122 14.79346 -0.000001298 -0.000000526 +15123 14.79442 -0.000001297 -0.000000526 +15124 14.79539 -0.000001297 -0.000000526 +15125 14.79635 -0.000001296 -0.000000526 +15126 14.79732 -0.000001296 -0.000000525 +15127 14.79828 -0.000001295 -0.000000525 +15128 14.79925 -0.000001295 -0.000000525 +15129 14.80022 -0.000001294 -0.000000525 +15130 14.80118 -0.000001294 -0.000000525 +15131 14.80215 -0.000001293 -0.000000524 +15132 14.80311 -0.000001293 -0.000000524 +15133 14.80408 -0.000001292 -0.000000524 +15134 14.80504 -0.000001292 -0.000000524 +15135 14.80601 -0.000001291 -0.000000523 +15136 14.80697 -0.000001291 -0.000000523 +15137 14.80794 -0.000001290 -0.000000523 +15138 14.80890 -0.000001290 -0.000000523 +15139 14.80987 -0.000001289 -0.000000522 +15140 14.81083 -0.000001289 -0.000000522 +15141 14.81180 -0.000001288 -0.000000522 +15142 14.81276 -0.000001288 -0.000000522 +15143 14.81373 -0.000001287 -0.000000521 +15144 14.81469 -0.000001287 -0.000000521 +15145 14.81566 -0.000001286 -0.000000521 +15146 14.81662 -0.000001286 -0.000000521 +15147 14.81759 -0.000001285 -0.000000520 +15148 14.81855 -0.000001285 -0.000000520 +15149 14.81952 -0.000001284 -0.000000520 +15150 14.82048 -0.000001284 -0.000000520 +15151 14.82145 -0.000001283 -0.000000520 +15152 14.82241 -0.000001283 -0.000000519 +15153 14.82338 -0.000001282 -0.000000519 +15154 14.82434 -0.000001282 -0.000000519 +15155 14.82531 -0.000001281 -0.000000519 +15156 14.82627 -0.000001281 -0.000000518 +15157 14.82724 -0.000001280 -0.000000518 +15158 14.82820 -0.000001280 -0.000000518 +15159 14.82917 -0.000001279 -0.000000518 +15160 14.83013 -0.000001279 -0.000000517 +15161 14.83110 -0.000001278 -0.000000517 +15162 14.83206 -0.000001278 -0.000000517 +15163 14.83303 -0.000001277 -0.000000517 +15164 14.83399 -0.000001277 -0.000000516 +15165 14.83496 -0.000001276 -0.000000516 +15166 14.83592 -0.000001276 -0.000000516 +15167 14.83689 -0.000001275 -0.000000516 +15168 14.83785 -0.000001275 -0.000000516 +15169 14.83882 -0.000001274 -0.000000515 +15170 14.83978 -0.000001274 -0.000000515 +15171 14.84075 -0.000001273 -0.000000515 +15172 14.84171 -0.000001273 -0.000000515 +15173 14.84268 -0.000001272 -0.000000514 +15174 14.84364 -0.000001272 -0.000000514 +15175 14.84461 -0.000001271 -0.000000514 +15176 14.84557 -0.000001271 -0.000000514 +15177 14.84654 -0.000001270 -0.000000513 +15178 14.84750 -0.000001270 -0.000000513 +15179 14.84847 -0.000001269 -0.000000513 +15180 14.84943 -0.000001269 -0.000000513 +15181 14.85040 -0.000001268 -0.000000512 +15182 14.85136 -0.000001268 -0.000000512 +15183 14.85233 -0.000001267 -0.000000512 +15184 14.85329 -0.000001267 -0.000000512 +15185 14.85426 -0.000001266 -0.000000512 +15186 14.85522 -0.000001266 -0.000000511 +15187 14.85619 -0.000001265 -0.000000511 +15188 14.85715 -0.000001265 -0.000000511 +15189 14.85812 -0.000001264 -0.000000511 +15190 14.85908 -0.000001264 -0.000000510 +15191 14.86005 -0.000001263 -0.000000510 +15192 14.86101 -0.000001263 -0.000000510 +15193 14.86198 -0.000001262 -0.000000510 +15194 14.86294 -0.000001262 -0.000000509 +15195 14.86391 -0.000001261 -0.000000509 +15196 14.86487 -0.000001261 -0.000000509 +15197 14.86584 -0.000001261 -0.000000509 +15198 14.86680 -0.000001260 -0.000000509 +15199 14.86777 -0.000001260 -0.000000508 +15200 14.86873 -0.000001259 -0.000000508 +15201 14.86970 -0.000001259 -0.000000508 +15202 14.87066 -0.000001258 -0.000000508 +15203 14.87163 -0.000001258 -0.000000507 +15204 14.87259 -0.000001257 -0.000000507 +15205 14.87356 -0.000001257 -0.000000507 +15206 14.87452 -0.000001256 -0.000000507 +15207 14.87549 -0.000001256 -0.000000506 +15208 14.87645 -0.000001255 -0.000000506 +15209 14.87742 -0.000001255 -0.000000506 +15210 14.87838 -0.000001254 -0.000000506 +15211 14.87935 -0.000001254 -0.000000506 +15212 14.88031 -0.000001253 -0.000000505 +15213 14.88128 -0.000001253 -0.000000505 +15214 14.88224 -0.000001252 -0.000000505 +15215 14.88321 -0.000001252 -0.000000505 +15216 14.88417 -0.000001251 -0.000000504 +15217 14.88514 -0.000001251 -0.000000504 +15218 14.88610 -0.000001250 -0.000000504 +15219 14.88707 -0.000001250 -0.000000504 +15220 14.88803 -0.000001249 -0.000000503 +15221 14.88900 -0.000001249 -0.000000503 +15222 14.88996 -0.000001248 -0.000000503 +15223 14.89093 -0.000001248 -0.000000503 +15224 14.89189 -0.000001247 -0.000000503 +15225 14.89286 -0.000001247 -0.000000502 +15226 14.89382 -0.000001246 -0.000000502 +15227 14.89479 -0.000001246 -0.000000502 +15228 14.89575 -0.000001245 -0.000000502 +15229 14.89672 -0.000001245 -0.000000501 +15230 14.89768 -0.000001244 -0.000000501 +15231 14.89865 -0.000001244 -0.000000501 +15232 14.89961 -0.000001243 -0.000000501 +15233 14.90058 -0.000001243 -0.000000501 +15234 14.90155 -0.000001242 -0.000000500 +15235 14.90251 -0.000001242 -0.000000500 +15236 14.90348 -0.000001242 -0.000000500 +15237 14.90444 -0.000001241 -0.000000500 +15238 14.90541 -0.000001241 -0.000000499 +15239 14.90637 -0.000001240 -0.000000499 +15240 14.90734 -0.000001240 -0.000000499 +15241 14.90830 -0.000001239 -0.000000499 +15242 14.90927 -0.000001239 -0.000000498 +15243 14.91023 -0.000001238 -0.000000498 +15244 14.91120 -0.000001238 -0.000000498 +15245 14.91216 -0.000001237 -0.000000498 +15246 14.91313 -0.000001237 -0.000000498 +15247 14.91409 -0.000001236 -0.000000497 +15248 14.91506 -0.000001236 -0.000000497 +15249 14.91602 -0.000001235 -0.000000497 +15250 14.91699 -0.000001235 -0.000000497 +15251 14.91795 -0.000001234 -0.000000496 +15252 14.91892 -0.000001234 -0.000000496 +15253 14.91988 -0.000001233 -0.000000496 +15254 14.92085 -0.000001233 -0.000000496 +15255 14.92181 -0.000001232 -0.000000496 +15256 14.92278 -0.000001232 -0.000000495 +15257 14.92374 -0.000001231 -0.000000495 +15258 14.92471 -0.000001231 -0.000000495 +15259 14.92567 -0.000001230 -0.000000495 +15260 14.92664 -0.000001230 -0.000000494 +15261 14.92760 -0.000001230 -0.000000494 +15262 14.92857 -0.000001229 -0.000000494 +15263 14.92953 -0.000001229 -0.000000494 +15264 14.93050 -0.000001228 -0.000000494 +15265 14.93146 -0.000001228 -0.000000493 +15266 14.93243 -0.000001227 -0.000000493 +15267 14.93339 -0.000001227 -0.000000493 +15268 14.93436 -0.000001226 -0.000000493 +15269 14.93532 -0.000001226 -0.000000492 +15270 14.93629 -0.000001225 -0.000000492 +15271 14.93725 -0.000001225 -0.000000492 +15272 14.93822 -0.000001224 -0.000000492 +15273 14.93918 -0.000001224 -0.000000492 +15274 14.94015 -0.000001223 -0.000000491 +15275 14.94111 -0.000001223 -0.000000491 +15276 14.94208 -0.000001222 -0.000000491 +15277 14.94304 -0.000001222 -0.000000491 +15278 14.94401 -0.000001221 -0.000000490 +15279 14.94497 -0.000001221 -0.000000490 +15280 14.94594 -0.000001221 -0.000000490 +15281 14.94690 -0.000001220 -0.000000490 +15282 14.94787 -0.000001220 -0.000000490 +15283 14.94883 -0.000001219 -0.000000489 +15284 14.94980 -0.000001219 -0.000000489 +15285 14.95076 -0.000001218 -0.000000489 +15286 14.95173 -0.000001218 -0.000000489 +15287 14.95269 -0.000001217 -0.000000488 +15288 14.95366 -0.000001217 -0.000000488 +15289 14.95462 -0.000001216 -0.000000488 +15290 14.95559 -0.000001216 -0.000000488 +15291 14.95655 -0.000001215 -0.000000488 +15292 14.95752 -0.000001215 -0.000000487 +15293 14.95848 -0.000001214 -0.000000487 +15294 14.95945 -0.000001214 -0.000000487 +15295 14.96041 -0.000001213 -0.000000487 +15296 14.96138 -0.000001213 -0.000000486 +15297 14.96234 -0.000001213 -0.000000486 +15298 14.96331 -0.000001212 -0.000000486 +15299 14.96427 -0.000001212 -0.000000486 +15300 14.96524 -0.000001211 -0.000000486 +15301 14.96620 -0.000001211 -0.000000485 +15302 14.96717 -0.000001210 -0.000000485 +15303 14.96813 -0.000001210 -0.000000485 +15304 14.96910 -0.000001209 -0.000000485 +15305 14.97006 -0.000001209 -0.000000484 +15306 14.97103 -0.000001208 -0.000000484 +15307 14.97199 -0.000001208 -0.000000484 +15308 14.97296 -0.000001207 -0.000000484 +15309 14.97392 -0.000001207 -0.000000484 +15310 14.97489 -0.000001206 -0.000000483 +15311 14.97585 -0.000001206 -0.000000483 +15312 14.97682 -0.000001205 -0.000000483 +15313 14.97778 -0.000001205 -0.000000483 +15314 14.97875 -0.000001205 -0.000000483 +15315 14.97971 -0.000001204 -0.000000482 +15316 14.98068 -0.000001204 -0.000000482 +15317 14.98164 -0.000001203 -0.000000482 +15318 14.98261 -0.000001203 -0.000000482 +15319 14.98357 -0.000001202 -0.000000481 +15320 14.98454 -0.000001202 -0.000000481 +15321 14.98550 -0.000001201 -0.000000481 +15322 14.98647 -0.000001201 -0.000000481 +15323 14.98743 -0.000001200 -0.000000481 +15324 14.98840 -0.000001200 -0.000000480 +15325 14.98936 -0.000001199 -0.000000480 +15326 14.99033 -0.000001199 -0.000000480 +15327 14.99129 -0.000001199 -0.000000480 +15328 14.99226 -0.000001198 -0.000000479 +15329 14.99322 -0.000001198 -0.000000479 +15330 14.99419 -0.000001197 -0.000000479 +15331 14.99515 -0.000001197 -0.000000479 +15332 14.99612 -0.000001196 -0.000000479 +15333 14.99708 -0.000001196 -0.000000478 +15334 14.99805 -0.000001195 -0.000000478 +15335 14.99901 -0.000001195 -0.000000478 +15336 14.99998 -0.000001194 -0.000000478 +15337 15.00095 -0.000001194 -0.000000478 +15338 15.00191 -0.000001193 -0.000000477 +15339 15.00288 -0.000001193 -0.000000477 +15340 15.00384 -0.000001193 -0.000000477 +15341 15.00481 -0.000001192 -0.000000477 +15342 15.00577 -0.000001192 -0.000000476 +15343 15.00674 -0.000001191 -0.000000476 +15344 15.00770 -0.000001191 -0.000000476 +15345 15.00867 -0.000001190 -0.000000476 +15346 15.00963 -0.000001190 -0.000000476 +15347 15.01060 -0.000001189 -0.000000475 +15348 15.01156 -0.000001189 -0.000000475 +15349 15.01253 -0.000001188 -0.000000475 +15350 15.01349 -0.000001188 -0.000000475 +15351 15.01446 -0.000001187 -0.000000475 +15352 15.01542 -0.000001187 -0.000000474 +15353 15.01639 -0.000001187 -0.000000474 +15354 15.01735 -0.000001186 -0.000000474 +15355 15.01832 -0.000001186 -0.000000474 +15356 15.01928 -0.000001185 -0.000000473 +15357 15.02025 -0.000001185 -0.000000473 +15358 15.02121 -0.000001184 -0.000000473 +15359 15.02218 -0.000001184 -0.000000473 +15360 15.02314 -0.000001183 -0.000000473 +15361 15.02411 -0.000001183 -0.000000472 +15362 15.02507 -0.000001182 -0.000000472 +15363 15.02604 -0.000001182 -0.000000472 +15364 15.02700 -0.000001182 -0.000000472 +15365 15.02797 -0.000001181 -0.000000472 +15366 15.02893 -0.000001181 -0.000000471 +15367 15.02990 -0.000001180 -0.000000471 +15368 15.03086 -0.000001180 -0.000000471 +15369 15.03183 -0.000001179 -0.000000471 +15370 15.03279 -0.000001179 -0.000000470 +15371 15.03376 -0.000001178 -0.000000470 +15372 15.03472 -0.000001178 -0.000000470 +15373 15.03569 -0.000001177 -0.000000470 +15374 15.03665 -0.000001177 -0.000000470 +15375 15.03762 -0.000001177 -0.000000469 +15376 15.03858 -0.000001176 -0.000000469 +15377 15.03955 -0.000001176 -0.000000469 +15378 15.04051 -0.000001175 -0.000000469 +15379 15.04148 -0.000001175 -0.000000469 +15380 15.04244 -0.000001174 -0.000000468 +15381 15.04341 -0.000001174 -0.000000468 +15382 15.04437 -0.000001173 -0.000000468 +15383 15.04534 -0.000001173 -0.000000468 +15384 15.04630 -0.000001172 -0.000000468 +15385 15.04727 -0.000001172 -0.000000467 +15386 15.04823 -0.000001172 -0.000000467 +15387 15.04920 -0.000001171 -0.000000467 +15388 15.05016 -0.000001171 -0.000000467 +15389 15.05113 -0.000001170 -0.000000466 +15390 15.05209 -0.000001170 -0.000000466 +15391 15.05306 -0.000001169 -0.000000466 +15392 15.05402 -0.000001169 -0.000000466 +15393 15.05499 -0.000001168 -0.000000466 +15394 15.05595 -0.000001168 -0.000000465 +15395 15.05692 -0.000001168 -0.000000465 +15396 15.05788 -0.000001167 -0.000000465 +15397 15.05885 -0.000001167 -0.000000465 +15398 15.05981 -0.000001166 -0.000000465 +15399 15.06078 -0.000001166 -0.000000464 +15400 15.06174 -0.000001165 -0.000000464 +15401 15.06271 -0.000001165 -0.000000464 +15402 15.06367 -0.000001164 -0.000000464 +15403 15.06464 -0.000001164 -0.000000464 +15404 15.06560 -0.000001163 -0.000000463 +15405 15.06657 -0.000001163 -0.000000463 +15406 15.06753 -0.000001163 -0.000000463 +15407 15.06850 -0.000001162 -0.000000463 +15408 15.06946 -0.000001162 -0.000000463 +15409 15.07043 -0.000001161 -0.000000462 +15410 15.07139 -0.000001161 -0.000000462 +15411 15.07236 -0.000001160 -0.000000462 +15412 15.07332 -0.000001160 -0.000000462 +15413 15.07429 -0.000001159 -0.000000462 +15414 15.07525 -0.000001159 -0.000000461 +15415 15.07622 -0.000001159 -0.000000461 +15416 15.07718 -0.000001158 -0.000000461 +15417 15.07815 -0.000001158 -0.000000461 +15418 15.07911 -0.000001157 -0.000000460 +15419 15.08008 -0.000001157 -0.000000460 +15420 15.08104 -0.000001156 -0.000000460 +15421 15.08201 -0.000001156 -0.000000460 +15422 15.08297 -0.000001155 -0.000000460 +15423 15.08394 -0.000001155 -0.000000459 +15424 15.08490 -0.000001155 -0.000000459 +15425 15.08587 -0.000001154 -0.000000459 +15426 15.08683 -0.000001154 -0.000000459 +15427 15.08780 -0.000001153 -0.000000459 +15428 15.08876 -0.000001153 -0.000000458 +15429 15.08973 -0.000001152 -0.000000458 +15430 15.09069 -0.000001152 -0.000000458 +15431 15.09166 -0.000001151 -0.000000458 +15432 15.09262 -0.000001151 -0.000000458 +15433 15.09359 -0.000001151 -0.000000457 +15434 15.09455 -0.000001150 -0.000000457 +15435 15.09552 -0.000001150 -0.000000457 +15436 15.09648 -0.000001149 -0.000000457 +15437 15.09745 -0.000001149 -0.000000457 +15438 15.09841 -0.000001148 -0.000000456 +15439 15.09938 -0.000001148 -0.000000456 +15440 15.10035 -0.000001148 -0.000000456 +15441 15.10131 -0.000001147 -0.000000456 +15442 15.10228 -0.000001147 -0.000000456 +15443 15.10324 -0.000001146 -0.000000455 +15444 15.10421 -0.000001146 -0.000000455 +15445 15.10517 -0.000001145 -0.000000455 +15446 15.10614 -0.000001145 -0.000000455 +15447 15.10710 -0.000001144 -0.000000455 +15448 15.10807 -0.000001144 -0.000000454 +15449 15.10903 -0.000001144 -0.000000454 +15450 15.11000 -0.000001143 -0.000000454 +15451 15.11096 -0.000001143 -0.000000454 +15452 15.11193 -0.000001142 -0.000000454 +15453 15.11289 -0.000001142 -0.000000453 +15454 15.11386 -0.000001141 -0.000000453 +15455 15.11482 -0.000001141 -0.000000453 +15456 15.11579 -0.000001141 -0.000000453 +15457 15.11675 -0.000001140 -0.000000453 +15458 15.11772 -0.000001140 -0.000000452 +15459 15.11868 -0.000001139 -0.000000452 +15460 15.11965 -0.000001139 -0.000000452 +15461 15.12061 -0.000001138 -0.000000452 +15462 15.12158 -0.000001138 -0.000000451 +15463 15.12254 -0.000001137 -0.000000451 +15464 15.12351 -0.000001137 -0.000000451 +15465 15.12447 -0.000001137 -0.000000451 +15466 15.12544 -0.000001136 -0.000000451 +15467 15.12640 -0.000001136 -0.000000450 +15468 15.12737 -0.000001135 -0.000000450 +15469 15.12833 -0.000001135 -0.000000450 +15470 15.12930 -0.000001134 -0.000000450 +15471 15.13026 -0.000001134 -0.000000450 +15472 15.13123 -0.000001134 -0.000000449 +15473 15.13219 -0.000001133 -0.000000449 +15474 15.13316 -0.000001133 -0.000000449 +15475 15.13412 -0.000001132 -0.000000449 +15476 15.13509 -0.000001132 -0.000000449 +15477 15.13605 -0.000001131 -0.000000448 +15478 15.13702 -0.000001131 -0.000000448 +15479 15.13798 -0.000001131 -0.000000448 +15480 15.13895 -0.000001130 -0.000000448 +15481 15.13991 -0.000001130 -0.000000448 +15482 15.14088 -0.000001129 -0.000000447 +15483 15.14184 -0.000001129 -0.000000447 +15484 15.14281 -0.000001128 -0.000000447 +15485 15.14377 -0.000001128 -0.000000447 +15486 15.14474 -0.000001127 -0.000000447 +15487 15.14570 -0.000001127 -0.000000446 +15488 15.14667 -0.000001127 -0.000000446 +15489 15.14763 -0.000001126 -0.000000446 +15490 15.14860 -0.000001126 -0.000000446 +15491 15.14956 -0.000001125 -0.000000446 +15492 15.15053 -0.000001125 -0.000000445 +15493 15.15149 -0.000001124 -0.000000445 +15494 15.15246 -0.000001124 -0.000000445 +15495 15.15342 -0.000001124 -0.000000445 +15496 15.15439 -0.000001123 -0.000000445 +15497 15.15535 -0.000001123 -0.000000444 +15498 15.15632 -0.000001122 -0.000000444 +15499 15.15728 -0.000001122 -0.000000444 +15500 15.15825 -0.000001121 -0.000000444 +15501 15.15921 -0.000001121 -0.000000444 +15502 15.16018 -0.000001121 -0.000000444 +15503 15.16114 -0.000001120 -0.000000443 +15504 15.16211 -0.000001120 -0.000000443 +15505 15.16307 -0.000001119 -0.000000443 +15506 15.16404 -0.000001119 -0.000000443 +15507 15.16500 -0.000001118 -0.000000443 +15508 15.16597 -0.000001118 -0.000000442 +15509 15.16693 -0.000001118 -0.000000442 +15510 15.16790 -0.000001117 -0.000000442 +15511 15.16886 -0.000001117 -0.000000442 +15512 15.16983 -0.000001116 -0.000000442 +15513 15.17079 -0.000001116 -0.000000441 +15514 15.17176 -0.000001115 -0.000000441 +15515 15.17272 -0.000001115 -0.000000441 +15516 15.17369 -0.000001115 -0.000000441 +15517 15.17465 -0.000001114 -0.000000441 +15518 15.17562 -0.000001114 -0.000000440 +15519 15.17658 -0.000001113 -0.000000440 +15520 15.17755 -0.000001113 -0.000000440 +15521 15.17851 -0.000001113 -0.000000440 +15522 15.17948 -0.000001112 -0.000000440 +15523 15.18044 -0.000001112 -0.000000439 +15524 15.18141 -0.000001111 -0.000000439 +15525 15.18237 -0.000001111 -0.000000439 +15526 15.18334 -0.000001110 -0.000000439 +15527 15.18430 -0.000001110 -0.000000439 +15528 15.18527 -0.000001110 -0.000000438 +15529 15.18623 -0.000001109 -0.000000438 +15530 15.18720 -0.000001109 -0.000000438 +15531 15.18816 -0.000001108 -0.000000438 +15532 15.18913 -0.000001108 -0.000000438 +15533 15.19009 -0.000001107 -0.000000437 +15534 15.19106 -0.000001107 -0.000000437 +15535 15.19202 -0.000001107 -0.000000437 +15536 15.19299 -0.000001106 -0.000000437 +15537 15.19395 -0.000001106 -0.000000437 +15538 15.19492 -0.000001105 -0.000000436 +15539 15.19588 -0.000001105 -0.000000436 +15540 15.19685 -0.000001104 -0.000000436 +15541 15.19781 -0.000001104 -0.000000436 +15542 15.19878 -0.000001104 -0.000000436 +15543 15.19974 -0.000001103 -0.000000435 +15544 15.20071 -0.000001103 -0.000000435 +15545 15.20168 -0.000001102 -0.000000435 +15546 15.20264 -0.000001102 -0.000000435 +15547 15.20361 -0.000001102 -0.000000435 +15548 15.20457 -0.000001101 -0.000000435 +15549 15.20554 -0.000001101 -0.000000434 +15550 15.20650 -0.000001100 -0.000000434 +15551 15.20747 -0.000001100 -0.000000434 +15552 15.20843 -0.000001099 -0.000000434 +15553 15.20940 -0.000001099 -0.000000434 +15554 15.21036 -0.000001099 -0.000000433 +15555 15.21133 -0.000001098 -0.000000433 +15556 15.21229 -0.000001098 -0.000000433 +15557 15.21326 -0.000001097 -0.000000433 +15558 15.21422 -0.000001097 -0.000000433 +15559 15.21519 -0.000001097 -0.000000432 +15560 15.21615 -0.000001096 -0.000000432 +15561 15.21712 -0.000001096 -0.000000432 +15562 15.21808 -0.000001095 -0.000000432 +15563 15.21905 -0.000001095 -0.000000432 +15564 15.22001 -0.000001094 -0.000000431 +15565 15.22098 -0.000001094 -0.000000431 +15566 15.22194 -0.000001094 -0.000000431 +15567 15.22291 -0.000001093 -0.000000431 +15568 15.22387 -0.000001093 -0.000000431 +15569 15.22484 -0.000001092 -0.000000430 +15570 15.22580 -0.000001092 -0.000000430 +15571 15.22677 -0.000001092 -0.000000430 +15572 15.22773 -0.000001091 -0.000000430 +15573 15.22870 -0.000001091 -0.000000430 +15574 15.22966 -0.000001090 -0.000000430 +15575 15.23063 -0.000001090 -0.000000429 +15576 15.23159 -0.000001089 -0.000000429 +15577 15.23256 -0.000001089 -0.000000429 +15578 15.23352 -0.000001089 -0.000000429 +15579 15.23449 -0.000001088 -0.000000429 +15580 15.23545 -0.000001088 -0.000000428 +15581 15.23642 -0.000001087 -0.000000428 +15582 15.23738 -0.000001087 -0.000000428 +15583 15.23835 -0.000001087 -0.000000428 +15584 15.23931 -0.000001086 -0.000000428 +15585 15.24028 -0.000001086 -0.000000427 +15586 15.24124 -0.000001085 -0.000000427 +15587 15.24221 -0.000001085 -0.000000427 +15588 15.24317 -0.000001085 -0.000000427 +15589 15.24414 -0.000001084 -0.000000427 +15590 15.24510 -0.000001084 -0.000000427 +15591 15.24607 -0.000001083 -0.000000426 +15592 15.24703 -0.000001083 -0.000000426 +15593 15.24800 -0.000001082 -0.000000426 +15594 15.24896 -0.000001082 -0.000000426 +15595 15.24993 -0.000001082 -0.000000426 +15596 15.25089 -0.000001081 -0.000000425 +15597 15.25186 -0.000001081 -0.000000425 +15598 15.25282 -0.000001080 -0.000000425 +15599 15.25379 -0.000001080 -0.000000425 +15600 15.25475 -0.000001080 -0.000000425 +15601 15.25572 -0.000001079 -0.000000424 +15602 15.25668 -0.000001079 -0.000000424 +15603 15.25765 -0.000001078 -0.000000424 +15604 15.25861 -0.000001078 -0.000000424 +15605 15.25958 -0.000001078 -0.000000424 +15606 15.26054 -0.000001077 -0.000000423 +15607 15.26151 -0.000001077 -0.000000423 +15608 15.26247 -0.000001076 -0.000000423 +15609 15.26344 -0.000001076 -0.000000423 +15610 15.26440 -0.000001075 -0.000000423 +15611 15.26537 -0.000001075 -0.000000423 +15612 15.26633 -0.000001075 -0.000000422 +15613 15.26730 -0.000001074 -0.000000422 +15614 15.26826 -0.000001074 -0.000000422 +15615 15.26923 -0.000001073 -0.000000422 +15616 15.27019 -0.000001073 -0.000000422 +15617 15.27116 -0.000001073 -0.000000421 +15618 15.27212 -0.000001072 -0.000000421 +15619 15.27309 -0.000001072 -0.000000421 +15620 15.27405 -0.000001071 -0.000000421 +15621 15.27502 -0.000001071 -0.000000421 +15622 15.27598 -0.000001071 -0.000000421 +15623 15.27695 -0.000001070 -0.000000420 +15624 15.27791 -0.000001070 -0.000000420 +15625 15.27888 -0.000001069 -0.000000420 +15626 15.27984 -0.000001069 -0.000000420 +15627 15.28081 -0.000001069 -0.000000420 +15628 15.28177 -0.000001068 -0.000000419 +15629 15.28274 -0.000001068 -0.000000419 +15630 15.28370 -0.000001067 -0.000000419 +15631 15.28467 -0.000001067 -0.000000419 +15632 15.28563 -0.000001067 -0.000000419 +15633 15.28660 -0.000001066 -0.000000418 +15634 15.28756 -0.000001066 -0.000000418 +15635 15.28853 -0.000001065 -0.000000418 +15636 15.28949 -0.000001065 -0.000000418 +15637 15.29046 -0.000001065 -0.000000418 +15638 15.29142 -0.000001064 -0.000000418 +15639 15.29239 -0.000001064 -0.000000417 +15640 15.29335 -0.000001063 -0.000000417 +15641 15.29432 -0.000001063 -0.000000417 +15642 15.29528 -0.000001063 -0.000000417 +15643 15.29625 -0.000001062 -0.000000417 +15644 15.29721 -0.000001062 -0.000000416 +15645 15.29818 -0.000001061 -0.000000416 +15646 15.29914 -0.000001061 -0.000000416 +15647 15.30011 -0.000001061 -0.000000416 +15648 15.30108 -0.000001060 -0.000000416 +15649 15.30204 -0.000001060 -0.000000416 +15650 15.30301 -0.000001059 -0.000000415 +15651 15.30397 -0.000001059 -0.000000415 +15652 15.30494 -0.000001059 -0.000000415 +15653 15.30590 -0.000001058 -0.000000415 +15654 15.30687 -0.000001058 -0.000000415 +15655 15.30783 -0.000001057 -0.000000414 +15656 15.30880 -0.000001057 -0.000000414 +15657 15.30976 -0.000001057 -0.000000414 +15658 15.31073 -0.000001056 -0.000000414 +15659 15.31169 -0.000001056 -0.000000414 +15660 15.31266 -0.000001055 -0.000000414 +15661 15.31362 -0.000001055 -0.000000413 +15662 15.31459 -0.000001055 -0.000000413 +15663 15.31555 -0.000001054 -0.000000413 +15664 15.31652 -0.000001054 -0.000000413 +15665 15.31748 -0.000001053 -0.000000413 +15666 15.31845 -0.000001053 -0.000000412 +15667 15.31941 -0.000001053 -0.000000412 +15668 15.32038 -0.000001052 -0.000000412 +15669 15.32134 -0.000001052 -0.000000412 +15670 15.32231 -0.000001051 -0.000000412 +15671 15.32327 -0.000001051 -0.000000412 +15672 15.32424 -0.000001051 -0.000000411 +15673 15.32520 -0.000001050 -0.000000411 +15674 15.32617 -0.000001050 -0.000000411 +15675 15.32713 -0.000001049 -0.000000411 +15676 15.32810 -0.000001049 -0.000000411 +15677 15.32906 -0.000001049 -0.000000410 +15678 15.33003 -0.000001048 -0.000000410 +15679 15.33099 -0.000001048 -0.000000410 +15680 15.33196 -0.000001047 -0.000000410 +15681 15.33292 -0.000001047 -0.000000410 +15682 15.33389 -0.000001047 -0.000000410 +15683 15.33485 -0.000001046 -0.000000409 +15684 15.33582 -0.000001046 -0.000000409 +15685 15.33678 -0.000001045 -0.000000409 +15686 15.33775 -0.000001045 -0.000000409 +15687 15.33871 -0.000001045 -0.000000409 +15688 15.33968 -0.000001044 -0.000000408 +15689 15.34064 -0.000001044 -0.000000408 +15690 15.34161 -0.000001043 -0.000000408 +15691 15.34257 -0.000001043 -0.000000408 +15692 15.34354 -0.000001043 -0.000000408 +15693 15.34450 -0.000001042 -0.000000408 +15694 15.34547 -0.000001042 -0.000000407 +15695 15.34643 -0.000001041 -0.000000407 +15696 15.34740 -0.000001041 -0.000000407 +15697 15.34836 -0.000001041 -0.000000407 +15698 15.34933 -0.000001040 -0.000000407 +15699 15.35029 -0.000001040 -0.000000406 +15700 15.35126 -0.000001039 -0.000000406 +15701 15.35222 -0.000001039 -0.000000406 +15702 15.35319 -0.000001039 -0.000000406 +15703 15.35415 -0.000001038 -0.000000406 +15704 15.35512 -0.000001038 -0.000000406 +15705 15.35608 -0.000001038 -0.000000405 +15706 15.35705 -0.000001037 -0.000000405 +15707 15.35801 -0.000001037 -0.000000405 +15708 15.35898 -0.000001036 -0.000000405 +15709 15.35994 -0.000001036 -0.000000405 +15710 15.36091 -0.000001036 -0.000000404 +15711 15.36187 -0.000001035 -0.000000404 +15712 15.36284 -0.000001035 -0.000000404 +15713 15.36380 -0.000001034 -0.000000404 +15714 15.36477 -0.000001034 -0.000000404 +15715 15.36573 -0.000001034 -0.000000404 +15716 15.36670 -0.000001033 -0.000000403 +15717 15.36766 -0.000001033 -0.000000403 +15718 15.36863 -0.000001032 -0.000000403 +15719 15.36959 -0.000001032 -0.000000403 +15720 15.37056 -0.000001032 -0.000000403 +15721 15.37152 -0.000001031 -0.000000403 +15722 15.37249 -0.000001031 -0.000000402 +15723 15.37345 -0.000001031 -0.000000402 +15724 15.37442 -0.000001030 -0.000000402 +15725 15.37538 -0.000001030 -0.000000402 +15726 15.37635 -0.000001029 -0.000000402 +15727 15.37731 -0.000001029 -0.000000401 +15728 15.37828 -0.000001029 -0.000000401 +15729 15.37924 -0.000001028 -0.000000401 +15730 15.38021 -0.000001028 -0.000000401 +15731 15.38117 -0.000001027 -0.000000401 +15732 15.38214 -0.000001027 -0.000000401 +15733 15.38310 -0.000001027 -0.000000400 +15734 15.38407 -0.000001026 -0.000000400 +15735 15.38503 -0.000001026 -0.000000400 +15736 15.38600 -0.000001025 -0.000000400 +15737 15.38696 -0.000001025 -0.000000400 +15738 15.38793 -0.000001025 -0.000000400 +15739 15.38889 -0.000001024 -0.000000399 +15740 15.38986 -0.000001024 -0.000000399 +15741 15.39082 -0.000001024 -0.000000399 +15742 15.39179 -0.000001023 -0.000000399 +15743 15.39275 -0.000001023 -0.000000399 +15744 15.39372 -0.000001022 -0.000000399 +15745 15.39468 -0.000001022 -0.000000398 +15746 15.39565 -0.000001022 -0.000000398 +15747 15.39661 -0.000001021 -0.000000398 +15748 15.39758 -0.000001021 -0.000000398 +15749 15.39854 -0.000001020 -0.000000398 +15750 15.39951 -0.000001020 -0.000000397 +15751 15.40048 -0.000001020 -0.000000397 +15752 15.40144 -0.000001019 -0.000000397 +15753 15.40241 -0.000001019 -0.000000397 +15754 15.40337 -0.000001019 -0.000000397 +15755 15.40434 -0.000001018 -0.000000397 +15756 15.40530 -0.000001018 -0.000000396 +15757 15.40627 -0.000001017 -0.000000396 +15758 15.40723 -0.000001017 -0.000000396 +15759 15.40820 -0.000001017 -0.000000396 +15760 15.40916 -0.000001016 -0.000000396 +15761 15.41013 -0.000001016 -0.000000396 +15762 15.41109 -0.000001016 -0.000000395 +15763 15.41206 -0.000001015 -0.000000395 +15764 15.41302 -0.000001015 -0.000000395 +15765 15.41399 -0.000001014 -0.000000395 +15766 15.41495 -0.000001014 -0.000000395 +15767 15.41592 -0.000001014 -0.000000395 +15768 15.41688 -0.000001013 -0.000000394 +15769 15.41785 -0.000001013 -0.000000394 +15770 15.41881 -0.000001012 -0.000000394 +15771 15.41978 -0.000001012 -0.000000394 +15772 15.42074 -0.000001012 -0.000000394 +15773 15.42171 -0.000001011 -0.000000393 +15774 15.42267 -0.000001011 -0.000000393 +15775 15.42364 -0.000001011 -0.000000393 +15776 15.42460 -0.000001010 -0.000000393 +15777 15.42557 -0.000001010 -0.000000393 +15778 15.42653 -0.000001009 -0.000000393 +15779 15.42750 -0.000001009 -0.000000392 +15780 15.42846 -0.000001009 -0.000000392 +15781 15.42943 -0.000001008 -0.000000392 +15782 15.43039 -0.000001008 -0.000000392 +15783 15.43136 -0.000001008 -0.000000392 +15784 15.43232 -0.000001007 -0.000000392 +15785 15.43329 -0.000001007 -0.000000391 +15786 15.43425 -0.000001006 -0.000000391 +15787 15.43522 -0.000001006 -0.000000391 +15788 15.43618 -0.000001006 -0.000000391 +15789 15.43715 -0.000001005 -0.000000391 +15790 15.43811 -0.000001005 -0.000000391 +15791 15.43908 -0.000001005 -0.000000390 +15792 15.44004 -0.000001004 -0.000000390 +15793 15.44101 -0.000001004 -0.000000390 +15794 15.44197 -0.000001003 -0.000000390 +15795 15.44294 -0.000001003 -0.000000390 +15796 15.44390 -0.000001003 -0.000000390 +15797 15.44487 -0.000001002 -0.000000389 +15798 15.44583 -0.000001002 -0.000000389 +15799 15.44680 -0.000001002 -0.000000389 +15800 15.44776 -0.000001001 -0.000000389 +15801 15.44873 -0.000001001 -0.000000389 +15802 15.44969 -0.000001000 -0.000000389 +15803 15.45066 -0.000001000 -0.000000388 +15804 15.45162 -0.000001000 -0.000000388 +15805 15.45259 -0.000000999 -0.000000388 +15806 15.45355 -0.000000999 -0.000000388 +15807 15.45452 -0.000000999 -0.000000388 +15808 15.45548 -0.000000998 -0.000000387 +15809 15.45645 -0.000000998 -0.000000387 +15810 15.45741 -0.000000997 -0.000000387 +15811 15.45838 -0.000000997 -0.000000387 +15812 15.45934 -0.000000997 -0.000000387 +15813 15.46031 -0.000000996 -0.000000387 +15814 15.46127 -0.000000996 -0.000000386 +15815 15.46224 -0.000000996 -0.000000386 +15816 15.46320 -0.000000995 -0.000000386 +15817 15.46417 -0.000000995 -0.000000386 +15818 15.46513 -0.000000994 -0.000000386 +15819 15.46610 -0.000000994 -0.000000386 +15820 15.46706 -0.000000994 -0.000000385 +15821 15.46803 -0.000000993 -0.000000385 +15822 15.46899 -0.000000993 -0.000000385 +15823 15.46996 -0.000000993 -0.000000385 +15824 15.47092 -0.000000992 -0.000000385 +15825 15.47189 -0.000000992 -0.000000385 +15826 15.47285 -0.000000991 -0.000000384 +15827 15.47382 -0.000000991 -0.000000384 +15828 15.47478 -0.000000991 -0.000000384 +15829 15.47575 -0.000000990 -0.000000384 +15830 15.47671 -0.000000990 -0.000000384 +15831 15.47768 -0.000000990 -0.000000384 +15832 15.47864 -0.000000989 -0.000000383 +15833 15.47961 -0.000000989 -0.000000383 +15834 15.48057 -0.000000988 -0.000000383 +15835 15.48154 -0.000000988 -0.000000383 +15836 15.48250 -0.000000988 -0.000000383 +15837 15.48347 -0.000000987 -0.000000383 +15838 15.48443 -0.000000987 -0.000000382 +15839 15.48540 -0.000000987 -0.000000382 +15840 15.48636 -0.000000986 -0.000000382 +15841 15.48733 -0.000000986 -0.000000382 +15842 15.48829 -0.000000986 -0.000000382 +15843 15.48926 -0.000000985 -0.000000382 +15844 15.49022 -0.000000985 -0.000000381 +15845 15.49119 -0.000000984 -0.000000381 +15846 15.49215 -0.000000984 -0.000000381 +15847 15.49312 -0.000000984 -0.000000381 +15848 15.49408 -0.000000983 -0.000000381 +15849 15.49505 -0.000000983 -0.000000381 +15850 15.49601 -0.000000983 -0.000000380 +15851 15.49698 -0.000000982 -0.000000380 +15852 15.49794 -0.000000982 -0.000000380 +15853 15.49891 -0.000000981 -0.000000380 +15854 15.49987 -0.000000981 -0.000000380 +15855 15.50084 -0.000000981 -0.000000380 +15856 15.50181 -0.000000980 -0.000000379 +15857 15.50277 -0.000000980 -0.000000379 +15858 15.50374 -0.000000980 -0.000000379 +15859 15.50470 -0.000000979 -0.000000379 +15860 15.50567 -0.000000979 -0.000000379 +15861 15.50663 -0.000000979 -0.000000379 +15862 15.50760 -0.000000978 -0.000000378 +15863 15.50856 -0.000000978 -0.000000378 +15864 15.50953 -0.000000977 -0.000000378 +15865 15.51049 -0.000000977 -0.000000378 +15866 15.51146 -0.000000977 -0.000000378 +15867 15.51242 -0.000000976 -0.000000378 +15868 15.51339 -0.000000976 -0.000000377 +15869 15.51435 -0.000000976 -0.000000377 +15870 15.51532 -0.000000975 -0.000000377 +15871 15.51628 -0.000000975 -0.000000377 +15872 15.51725 -0.000000975 -0.000000377 +15873 15.51821 -0.000000974 -0.000000377 +15874 15.51918 -0.000000974 -0.000000376 +15875 15.52014 -0.000000973 -0.000000376 +15876 15.52111 -0.000000973 -0.000000376 +15877 15.52207 -0.000000973 -0.000000376 +15878 15.52304 -0.000000972 -0.000000376 +15879 15.52400 -0.000000972 -0.000000376 +15880 15.52497 -0.000000972 -0.000000376 +15881 15.52593 -0.000000971 -0.000000375 +15882 15.52690 -0.000000971 -0.000000375 +15883 15.52786 -0.000000971 -0.000000375 +15884 15.52883 -0.000000970 -0.000000375 +15885 15.52979 -0.000000970 -0.000000375 +15886 15.53076 -0.000000969 -0.000000375 +15887 15.53172 -0.000000969 -0.000000374 +15888 15.53269 -0.000000969 -0.000000374 +15889 15.53365 -0.000000968 -0.000000374 +15890 15.53462 -0.000000968 -0.000000374 +15891 15.53558 -0.000000968 -0.000000374 +15892 15.53655 -0.000000967 -0.000000374 +15893 15.53751 -0.000000967 -0.000000373 +15894 15.53848 -0.000000967 -0.000000373 +15895 15.53944 -0.000000966 -0.000000373 +15896 15.54041 -0.000000966 -0.000000373 +15897 15.54137 -0.000000965 -0.000000373 +15898 15.54234 -0.000000965 -0.000000373 +15899 15.54330 -0.000000965 -0.000000372 +15900 15.54427 -0.000000964 -0.000000372 +15901 15.54523 -0.000000964 -0.000000372 +15902 15.54620 -0.000000964 -0.000000372 +15903 15.54716 -0.000000963 -0.000000372 +15904 15.54813 -0.000000963 -0.000000372 +15905 15.54909 -0.000000963 -0.000000371 +15906 15.55006 -0.000000962 -0.000000371 +15907 15.55102 -0.000000962 -0.000000371 +15908 15.55199 -0.000000962 -0.000000371 +15909 15.55295 -0.000000961 -0.000000371 +15910 15.55392 -0.000000961 -0.000000371 +15911 15.55488 -0.000000960 -0.000000370 +15912 15.55585 -0.000000960 -0.000000370 +15913 15.55681 -0.000000960 -0.000000370 +15914 15.55778 -0.000000959 -0.000000370 +15915 15.55874 -0.000000959 -0.000000370 +15916 15.55971 -0.000000959 -0.000000370 +15917 15.56067 -0.000000958 -0.000000370 +15918 15.56164 -0.000000958 -0.000000369 +15919 15.56260 -0.000000958 -0.000000369 +15920 15.56357 -0.000000957 -0.000000369 +15921 15.56453 -0.000000957 -0.000000369 +15922 15.56550 -0.000000957 -0.000000369 +15923 15.56646 -0.000000956 -0.000000369 +15924 15.56743 -0.000000956 -0.000000368 +15925 15.56839 -0.000000955 -0.000000368 +15926 15.56936 -0.000000955 -0.000000368 +15927 15.57032 -0.000000955 -0.000000368 +15928 15.57129 -0.000000954 -0.000000368 +15929 15.57225 -0.000000954 -0.000000368 +15930 15.57322 -0.000000954 -0.000000367 +15931 15.57418 -0.000000953 -0.000000367 +15932 15.57515 -0.000000953 -0.000000367 +15933 15.57611 -0.000000953 -0.000000367 +15934 15.57708 -0.000000952 -0.000000367 +15935 15.57804 -0.000000952 -0.000000367 +15936 15.57901 -0.000000952 -0.000000366 +15937 15.57997 -0.000000951 -0.000000366 +15938 15.58094 -0.000000951 -0.000000366 +15939 15.58190 -0.000000951 -0.000000366 +15940 15.58287 -0.000000950 -0.000000366 +15941 15.58383 -0.000000950 -0.000000366 +15942 15.58480 -0.000000949 -0.000000366 +15943 15.58576 -0.000000949 -0.000000365 +15944 15.58673 -0.000000949 -0.000000365 +15945 15.58769 -0.000000948 -0.000000365 +15946 15.58866 -0.000000948 -0.000000365 +15947 15.58962 -0.000000948 -0.000000365 +15948 15.59059 -0.000000947 -0.000000365 +15949 15.59155 -0.000000947 -0.000000364 +15950 15.59252 -0.000000947 -0.000000364 +15951 15.59348 -0.000000946 -0.000000364 +15952 15.59445 -0.000000946 -0.000000364 +15953 15.59541 -0.000000946 -0.000000364 +15954 15.59638 -0.000000945 -0.000000364 +15955 15.59734 -0.000000945 -0.000000363 +15956 15.59831 -0.000000945 -0.000000363 +15957 15.59927 -0.000000944 -0.000000363 +15958 15.60024 -0.000000944 -0.000000363 +15959 15.60121 -0.000000943 -0.000000363 +15960 15.60217 -0.000000943 -0.000000363 +15961 15.60314 -0.000000943 -0.000000363 +15962 15.60410 -0.000000942 -0.000000362 +15963 15.60507 -0.000000942 -0.000000362 +15964 15.60603 -0.000000942 -0.000000362 +15965 15.60700 -0.000000941 -0.000000362 +15966 15.60796 -0.000000941 -0.000000362 +15967 15.60893 -0.000000941 -0.000000362 +15968 15.60989 -0.000000940 -0.000000361 +15969 15.61086 -0.000000940 -0.000000361 +15970 15.61182 -0.000000940 -0.000000361 +15971 15.61279 -0.000000939 -0.000000361 +15972 15.61375 -0.000000939 -0.000000361 +15973 15.61472 -0.000000939 -0.000000361 +15974 15.61568 -0.000000938 -0.000000361 +15975 15.61665 -0.000000938 -0.000000360 +15976 15.61761 -0.000000938 -0.000000360 +15977 15.61858 -0.000000937 -0.000000360 +15978 15.61954 -0.000000937 -0.000000360 +15979 15.62051 -0.000000937 -0.000000360 +15980 15.62147 -0.000000936 -0.000000360 +15981 15.62244 -0.000000936 -0.000000359 +15982 15.62340 -0.000000935 -0.000000359 +15983 15.62437 -0.000000935 -0.000000359 +15984 15.62533 -0.000000935 -0.000000359 +15985 15.62630 -0.000000934 -0.000000359 +15986 15.62726 -0.000000934 -0.000000359 +15987 15.62823 -0.000000934 -0.000000358 +15988 15.62919 -0.000000933 -0.000000358 +15989 15.63016 -0.000000933 -0.000000358 +15990 15.63112 -0.000000933 -0.000000358 +15991 15.63209 -0.000000932 -0.000000358 +15992 15.63305 -0.000000932 -0.000000358 +15993 15.63402 -0.000000932 -0.000000358 +15994 15.63498 -0.000000931 -0.000000357 +15995 15.63595 -0.000000931 -0.000000357 +15996 15.63691 -0.000000931 -0.000000357 +15997 15.63788 -0.000000930 -0.000000357 +15998 15.63884 -0.000000930 -0.000000357 +15999 15.63981 -0.000000930 -0.000000357 +16000 15.64077 -0.000000929 -0.000000356 +16001 15.64174 -0.000000929 -0.000000356 +16002 15.64270 -0.000000929 -0.000000356 +16003 15.64367 -0.000000928 -0.000000356 +16004 15.64463 -0.000000928 -0.000000356 +16005 15.64560 -0.000000928 -0.000000356 +16006 15.64656 -0.000000927 -0.000000356 +16007 15.64753 -0.000000927 -0.000000355 +16008 15.64849 -0.000000927 -0.000000355 +16009 15.64946 -0.000000926 -0.000000355 +16010 15.65042 -0.000000926 -0.000000355 +16011 15.65139 -0.000000925 -0.000000355 +16012 15.65235 -0.000000925 -0.000000355 +16013 15.65332 -0.000000925 -0.000000354 +16014 15.65428 -0.000000924 -0.000000354 +16015 15.65525 -0.000000924 -0.000000354 +16016 15.65621 -0.000000924 -0.000000354 +16017 15.65718 -0.000000923 -0.000000354 +16018 15.65814 -0.000000923 -0.000000354 +16019 15.65911 -0.000000923 -0.000000354 +16020 15.66007 -0.000000922 -0.000000353 +16021 15.66104 -0.000000922 -0.000000353 +16022 15.66200 -0.000000922 -0.000000353 +16023 15.66297 -0.000000921 -0.000000353 +16024 15.66393 -0.000000921 -0.000000353 +16025 15.66490 -0.000000921 -0.000000353 +16026 15.66586 -0.000000920 -0.000000352 +16027 15.66683 -0.000000920 -0.000000352 +16028 15.66779 -0.000000920 -0.000000352 +16029 15.66876 -0.000000919 -0.000000352 +16030 15.66972 -0.000000919 -0.000000352 +16031 15.67069 -0.000000919 -0.000000352 +16032 15.67165 -0.000000918 -0.000000352 +16033 15.67262 -0.000000918 -0.000000351 +16034 15.67358 -0.000000918 -0.000000351 +16035 15.67455 -0.000000917 -0.000000351 +16036 15.67551 -0.000000917 -0.000000351 +16037 15.67648 -0.000000917 -0.000000351 +16038 15.67744 -0.000000916 -0.000000351 +16039 15.67841 -0.000000916 -0.000000351 +16040 15.67937 -0.000000916 -0.000000350 +16041 15.68034 -0.000000915 -0.000000350 +16042 15.68130 -0.000000915 -0.000000350 +16043 15.68227 -0.000000915 -0.000000350 +16044 15.68323 -0.000000914 -0.000000350 +16045 15.68420 -0.000000914 -0.000000350 +16046 15.68516 -0.000000914 -0.000000349 +16047 15.68613 -0.000000913 -0.000000349 +16048 15.68709 -0.000000913 -0.000000349 +16049 15.68806 -0.000000913 -0.000000349 +16050 15.68902 -0.000000912 -0.000000349 +16051 15.68999 -0.000000912 -0.000000349 +16052 15.69095 -0.000000912 -0.000000349 +16053 15.69192 -0.000000911 -0.000000348 +16054 15.69288 -0.000000911 -0.000000348 +16055 15.69385 -0.000000911 -0.000000348 +16056 15.69481 -0.000000910 -0.000000348 +16057 15.69578 -0.000000910 -0.000000348 +16058 15.69674 -0.000000910 -0.000000348 +16059 15.69771 -0.000000909 -0.000000348 +16060 15.69867 -0.000000909 -0.000000347 +16061 15.69964 -0.000000909 -0.000000347 +16062 15.70061 -0.000000908 -0.000000347 +16063 15.70157 -0.000000908 -0.000000347 +16064 15.70254 -0.000000908 -0.000000347 +16065 15.70350 -0.000000907 -0.000000347 +16066 15.70447 -0.000000907 -0.000000346 +16067 15.70543 -0.000000907 -0.000000346 +16068 15.70640 -0.000000906 -0.000000346 +16069 15.70736 -0.000000906 -0.000000346 +16070 15.70833 -0.000000906 -0.000000346 +16071 15.70929 -0.000000905 -0.000000346 +16072 15.71026 -0.000000905 -0.000000346 +16073 15.71122 -0.000000905 -0.000000345 +16074 15.71219 -0.000000904 -0.000000345 +16075 15.71315 -0.000000904 -0.000000345 +16076 15.71412 -0.000000904 -0.000000345 +16077 15.71508 -0.000000903 -0.000000345 +16078 15.71605 -0.000000903 -0.000000345 +16079 15.71701 -0.000000903 -0.000000345 +16080 15.71798 -0.000000902 -0.000000344 +16081 15.71894 -0.000000902 -0.000000344 +16082 15.71991 -0.000000902 -0.000000344 +16083 15.72087 -0.000000901 -0.000000344 +16084 15.72184 -0.000000901 -0.000000344 +16085 15.72280 -0.000000901 -0.000000344 +16086 15.72377 -0.000000900 -0.000000344 +16087 15.72473 -0.000000900 -0.000000343 +16088 15.72570 -0.000000900 -0.000000343 +16089 15.72666 -0.000000899 -0.000000343 +16090 15.72763 -0.000000899 -0.000000343 +16091 15.72859 -0.000000899 -0.000000343 +16092 15.72956 -0.000000898 -0.000000343 +16093 15.73052 -0.000000898 -0.000000342 +16094 15.73149 -0.000000898 -0.000000342 +16095 15.73245 -0.000000897 -0.000000342 +16096 15.73342 -0.000000897 -0.000000342 +16097 15.73438 -0.000000897 -0.000000342 +16098 15.73535 -0.000000896 -0.000000342 +16099 15.73631 -0.000000896 -0.000000342 +16100 15.73728 -0.000000896 -0.000000341 +16101 15.73824 -0.000000895 -0.000000341 +16102 15.73921 -0.000000895 -0.000000341 +16103 15.74017 -0.000000895 -0.000000341 +16104 15.74114 -0.000000894 -0.000000341 +16105 15.74210 -0.000000894 -0.000000341 +16106 15.74307 -0.000000894 -0.000000341 +16107 15.74403 -0.000000893 -0.000000340 +16108 15.74500 -0.000000893 -0.000000340 +16109 15.74596 -0.000000893 -0.000000340 +16110 15.74693 -0.000000892 -0.000000340 +16111 15.74789 -0.000000892 -0.000000340 +16112 15.74886 -0.000000892 -0.000000340 +16113 15.74982 -0.000000891 -0.000000340 +16114 15.75079 -0.000000891 -0.000000339 +16115 15.75175 -0.000000891 -0.000000339 +16116 15.75272 -0.000000890 -0.000000339 +16117 15.75368 -0.000000890 -0.000000339 +16118 15.75465 -0.000000890 -0.000000339 +16119 15.75561 -0.000000889 -0.000000339 +16120 15.75658 -0.000000889 -0.000000339 +16121 15.75754 -0.000000889 -0.000000338 +16122 15.75851 -0.000000888 -0.000000338 +16123 15.75947 -0.000000888 -0.000000338 +16124 15.76044 -0.000000888 -0.000000338 +16125 15.76140 -0.000000887 -0.000000338 +16126 15.76237 -0.000000887 -0.000000338 +16127 15.76333 -0.000000887 -0.000000338 +16128 15.76430 -0.000000886 -0.000000337 +16129 15.76526 -0.000000886 -0.000000337 +16130 15.76623 -0.000000886 -0.000000337 +16131 15.76719 -0.000000885 -0.000000337 +16132 15.76816 -0.000000885 -0.000000337 +16133 15.76912 -0.000000885 -0.000000337 +16134 15.77009 -0.000000884 -0.000000337 +16135 15.77105 -0.000000884 -0.000000336 +16136 15.77202 -0.000000884 -0.000000336 +16137 15.77298 -0.000000883 -0.000000336 +16138 15.77395 -0.000000883 -0.000000336 +16139 15.77491 -0.000000883 -0.000000336 +16140 15.77588 -0.000000883 -0.000000336 +16141 15.77684 -0.000000882 -0.000000336 +16142 15.77781 -0.000000882 -0.000000335 +16143 15.77877 -0.000000882 -0.000000335 +16144 15.77974 -0.000000881 -0.000000335 +16145 15.78070 -0.000000881 -0.000000335 +16146 15.78167 -0.000000881 -0.000000335 +16147 15.78263 -0.000000880 -0.000000335 +16148 15.78360 -0.000000880 -0.000000334 +16149 15.78456 -0.000000880 -0.000000334 +16150 15.78553 -0.000000879 -0.000000334 +16151 15.78649 -0.000000879 -0.000000334 +16152 15.78746 -0.000000879 -0.000000334 +16153 15.78842 -0.000000878 -0.000000334 +16154 15.78939 -0.000000878 -0.000000334 +16155 15.79035 -0.000000878 -0.000000333 +16156 15.79132 -0.000000877 -0.000000333 +16157 15.79228 -0.000000877 -0.000000333 +16158 15.79325 -0.000000877 -0.000000333 +16159 15.79421 -0.000000876 -0.000000333 +16160 15.79518 -0.000000876 -0.000000333 +16161 15.79614 -0.000000876 -0.000000333 +16162 15.79711 -0.000000875 -0.000000332 +16163 15.79807 -0.000000875 -0.000000332 +16164 15.79904 -0.000000875 -0.000000332 +16165 15.80001 -0.000000874 -0.000000332 +16166 15.80097 -0.000000874 -0.000000332 +16167 15.80194 -0.000000874 -0.000000332 +16168 15.80290 -0.000000873 -0.000000332 +16169 15.80387 -0.000000873 -0.000000332 +16170 15.80483 -0.000000873 -0.000000331 +16171 15.80580 -0.000000873 -0.000000331 +16172 15.80676 -0.000000872 -0.000000331 +16173 15.80773 -0.000000872 -0.000000331 +16174 15.80869 -0.000000872 -0.000000331 +16175 15.80966 -0.000000871 -0.000000331 +16176 15.81062 -0.000000871 -0.000000331 +16177 15.81159 -0.000000871 -0.000000330 +16178 15.81255 -0.000000870 -0.000000330 +16179 15.81352 -0.000000870 -0.000000330 +16180 15.81448 -0.000000870 -0.000000330 +16181 15.81545 -0.000000869 -0.000000330 +16182 15.81641 -0.000000869 -0.000000330 +16183 15.81738 -0.000000869 -0.000000330 +16184 15.81834 -0.000000868 -0.000000329 +16185 15.81931 -0.000000868 -0.000000329 +16186 15.82027 -0.000000868 -0.000000329 +16187 15.82124 -0.000000867 -0.000000329 +16188 15.82220 -0.000000867 -0.000000329 +16189 15.82317 -0.000000867 -0.000000329 +16190 15.82413 -0.000000866 -0.000000329 +16191 15.82510 -0.000000866 -0.000000328 +16192 15.82606 -0.000000866 -0.000000328 +16193 15.82703 -0.000000866 -0.000000328 +16194 15.82799 -0.000000865 -0.000000328 +16195 15.82896 -0.000000865 -0.000000328 +16196 15.82992 -0.000000865 -0.000000328 +16197 15.83089 -0.000000864 -0.000000328 +16198 15.83185 -0.000000864 -0.000000327 +16199 15.83282 -0.000000864 -0.000000327 +16200 15.83378 -0.000000863 -0.000000327 +16201 15.83475 -0.000000863 -0.000000327 +16202 15.83571 -0.000000863 -0.000000327 +16203 15.83668 -0.000000862 -0.000000327 +16204 15.83764 -0.000000862 -0.000000327 +16205 15.83861 -0.000000862 -0.000000326 +16206 15.83957 -0.000000861 -0.000000326 +16207 15.84054 -0.000000861 -0.000000326 +16208 15.84150 -0.000000861 -0.000000326 +16209 15.84247 -0.000000860 -0.000000326 +16210 15.84343 -0.000000860 -0.000000326 +16211 15.84440 -0.000000860 -0.000000326 +16212 15.84536 -0.000000860 -0.000000325 +16213 15.84633 -0.000000859 -0.000000325 +16214 15.84729 -0.000000859 -0.000000325 +16215 15.84826 -0.000000859 -0.000000325 +16216 15.84922 -0.000000858 -0.000000325 +16217 15.85019 -0.000000858 -0.000000325 +16218 15.85115 -0.000000858 -0.000000325 +16219 15.85212 -0.000000857 -0.000000325 +16220 15.85308 -0.000000857 -0.000000324 +16221 15.85405 -0.000000857 -0.000000324 +16222 15.85501 -0.000000856 -0.000000324 +16223 15.85598 -0.000000856 -0.000000324 +16224 15.85694 -0.000000856 -0.000000324 +16225 15.85791 -0.000000855 -0.000000324 +16226 15.85887 -0.000000855 -0.000000324 +16227 15.85984 -0.000000855 -0.000000323 +16228 15.86080 -0.000000855 -0.000000323 +16229 15.86177 -0.000000854 -0.000000323 +16230 15.86273 -0.000000854 -0.000000323 +16231 15.86370 -0.000000854 -0.000000323 +16232 15.86466 -0.000000853 -0.000000323 +16233 15.86563 -0.000000853 -0.000000323 +16234 15.86659 -0.000000853 -0.000000322 +16235 15.86756 -0.000000852 -0.000000322 +16236 15.86852 -0.000000852 -0.000000322 +16237 15.86949 -0.000000852 -0.000000322 +16238 15.87045 -0.000000851 -0.000000322 +16239 15.87142 -0.000000851 -0.000000322 +16240 15.87238 -0.000000851 -0.000000322 +16241 15.87335 -0.000000850 -0.000000321 +16242 15.87431 -0.000000850 -0.000000321 +16243 15.87528 -0.000000850 -0.000000321 +16244 15.87624 -0.000000850 -0.000000321 +16245 15.87721 -0.000000849 -0.000000321 +16246 15.87817 -0.000000849 -0.000000321 +16247 15.87914 -0.000000849 -0.000000321 +16248 15.88010 -0.000000848 -0.000000321 +16249 15.88107 -0.000000848 -0.000000320 +16250 15.88203 -0.000000848 -0.000000320 +16251 15.88300 -0.000000847 -0.000000320 +16252 15.88396 -0.000000847 -0.000000320 +16253 15.88493 -0.000000847 -0.000000320 +16254 15.88589 -0.000000846 -0.000000320 +16255 15.88686 -0.000000846 -0.000000320 +16256 15.88782 -0.000000846 -0.000000319 +16257 15.88879 -0.000000846 -0.000000319 +16258 15.88975 -0.000000845 -0.000000319 +16259 15.89072 -0.000000845 -0.000000319 +16260 15.89168 -0.000000845 -0.000000319 +16261 15.89265 -0.000000844 -0.000000319 +16262 15.89361 -0.000000844 -0.000000319 +16263 15.89458 -0.000000844 -0.000000318 +16264 15.89554 -0.000000843 -0.000000318 +16265 15.89651 -0.000000843 -0.000000318 +16266 15.89747 -0.000000843 -0.000000318 +16267 15.89844 -0.000000842 -0.000000318 +16268 15.89940 -0.000000842 -0.000000318 +16269 15.90037 -0.000000842 -0.000000318 +16270 15.90134 -0.000000842 -0.000000318 +16271 15.90230 -0.000000841 -0.000000317 +16272 15.90327 -0.000000841 -0.000000317 +16273 15.90423 -0.000000841 -0.000000317 +16274 15.90520 -0.000000840 -0.000000317 +16275 15.90616 -0.000000840 -0.000000317 +16276 15.90713 -0.000000840 -0.000000317 +16277 15.90809 -0.000000839 -0.000000317 +16278 15.90906 -0.000000839 -0.000000316 +16279 15.91002 -0.000000839 -0.000000316 +16280 15.91099 -0.000000838 -0.000000316 +16281 15.91195 -0.000000838 -0.000000316 +16282 15.91292 -0.000000838 -0.000000316 +16283 15.91388 -0.000000838 -0.000000316 +16284 15.91485 -0.000000837 -0.000000316 +16285 15.91581 -0.000000837 -0.000000316 +16286 15.91678 -0.000000837 -0.000000315 +16287 15.91774 -0.000000836 -0.000000315 +16288 15.91871 -0.000000836 -0.000000315 +16289 15.91967 -0.000000836 -0.000000315 +16290 15.92064 -0.000000835 -0.000000315 +16291 15.92160 -0.000000835 -0.000000315 +16292 15.92257 -0.000000835 -0.000000315 +16293 15.92353 -0.000000835 -0.000000314 +16294 15.92450 -0.000000834 -0.000000314 +16295 15.92546 -0.000000834 -0.000000314 +16296 15.92643 -0.000000834 -0.000000314 +16297 15.92739 -0.000000833 -0.000000314 +16298 15.92836 -0.000000833 -0.000000314 +16299 15.92932 -0.000000833 -0.000000314 +16300 15.93029 -0.000000832 -0.000000314 +16301 15.93125 -0.000000832 -0.000000313 +16302 15.93222 -0.000000832 -0.000000313 +16303 15.93318 -0.000000832 -0.000000313 +16304 15.93415 -0.000000831 -0.000000313 +16305 15.93511 -0.000000831 -0.000000313 +16306 15.93608 -0.000000831 -0.000000313 +16307 15.93704 -0.000000830 -0.000000313 +16308 15.93801 -0.000000830 -0.000000312 +16309 15.93897 -0.000000830 -0.000000312 +16310 15.93994 -0.000000829 -0.000000312 +16311 15.94090 -0.000000829 -0.000000312 +16312 15.94187 -0.000000829 -0.000000312 +16313 15.94283 -0.000000828 -0.000000312 +16314 15.94380 -0.000000828 -0.000000312 +16315 15.94476 -0.000000828 -0.000000312 +16316 15.94573 -0.000000828 -0.000000311 +16317 15.94669 -0.000000827 -0.000000311 +16318 15.94766 -0.000000827 -0.000000311 +16319 15.94862 -0.000000827 -0.000000311 +16320 15.94959 -0.000000826 -0.000000311 +16321 15.95055 -0.000000826 -0.000000311 +16322 15.95152 -0.000000826 -0.000000311 +16323 15.95248 -0.000000825 -0.000000310 +16324 15.95345 -0.000000825 -0.000000310 +16325 15.95441 -0.000000825 -0.000000310 +16326 15.95538 -0.000000825 -0.000000310 +16327 15.95634 -0.000000824 -0.000000310 +16328 15.95731 -0.000000824 -0.000000310 +16329 15.95827 -0.000000824 -0.000000310 +16330 15.95924 -0.000000823 -0.000000310 +16331 15.96020 -0.000000823 -0.000000309 +16332 15.96117 -0.000000823 -0.000000309 +16333 15.96213 -0.000000823 -0.000000309 +16334 15.96310 -0.000000822 -0.000000309 +16335 15.96406 -0.000000822 -0.000000309 +16336 15.96503 -0.000000822 -0.000000309 +16337 15.96599 -0.000000821 -0.000000309 +16338 15.96696 -0.000000821 -0.000000309 +16339 15.96792 -0.000000821 -0.000000308 +16340 15.96889 -0.000000820 -0.000000308 +16341 15.96985 -0.000000820 -0.000000308 +16342 15.97082 -0.000000820 -0.000000308 +16343 15.97178 -0.000000820 -0.000000308 +16344 15.97275 -0.000000819 -0.000000308 +16345 15.97371 -0.000000819 -0.000000308 +16346 15.97468 -0.000000819 -0.000000307 +16347 15.97564 -0.000000818 -0.000000307 +16348 15.97661 -0.000000818 -0.000000307 +16349 15.97757 -0.000000818 -0.000000307 +16350 15.97854 -0.000000817 -0.000000307 +16351 15.97950 -0.000000817 -0.000000307 +16352 15.98047 -0.000000817 -0.000000307 +16353 15.98143 -0.000000817 -0.000000307 +16354 15.98240 -0.000000816 -0.000000306 +16355 15.98336 -0.000000816 -0.000000306 +16356 15.98433 -0.000000816 -0.000000306 +16357 15.98529 -0.000000815 -0.000000306 +16358 15.98626 -0.000000815 -0.000000306 +16359 15.98722 -0.000000815 -0.000000306 +16360 15.98819 -0.000000814 -0.000000306 +16361 15.98915 -0.000000814 -0.000000306 +16362 15.99012 -0.000000814 -0.000000305 +16363 15.99108 -0.000000814 -0.000000305 +16364 15.99205 -0.000000813 -0.000000305 +16365 15.99301 -0.000000813 -0.000000305 +16366 15.99398 -0.000000813 -0.000000305 +16367 15.99494 -0.000000812 -0.000000305 +16368 15.99591 -0.000000812 -0.000000305 +16369 15.99687 -0.000000812 -0.000000305 +16370 15.99784 -0.000000812 -0.000000304 +16371 15.99880 -0.000000811 -0.000000304 +16372 15.99977 -0.000000811 -0.000000304 +16373 16.00074 -0.000000811 -0.000000304 +16374 16.00170 -0.000000810 -0.000000304 +16375 16.00267 -0.000000810 -0.000000304 +16376 16.00363 -0.000000810 -0.000000304 +16377 16.00460 -0.000000809 -0.000000303 +16378 16.00556 -0.000000809 -0.000000303 +16379 16.00653 -0.000000809 -0.000000303 +16380 16.00749 -0.000000809 -0.000000303 +16381 16.00846 -0.000000808 -0.000000303 +16382 16.00942 -0.000000808 -0.000000303 +16383 16.01039 -0.000000808 -0.000000303 +16384 16.01135 -0.000000807 -0.000000303 +16385 16.01232 -0.000000807 -0.000000302 +16386 16.01328 -0.000000807 -0.000000302 +16387 16.01425 -0.000000807 -0.000000302 +16388 16.01521 -0.000000806 -0.000000302 +16389 16.01618 -0.000000806 -0.000000302 +16390 16.01714 -0.000000806 -0.000000302 +16391 16.01811 -0.000000805 -0.000000302 +16392 16.01907 -0.000000805 -0.000000302 +16393 16.02004 -0.000000805 -0.000000301 +16394 16.02100 -0.000000805 -0.000000301 +16395 16.02197 -0.000000804 -0.000000301 +16396 16.02293 -0.000000804 -0.000000301 +16397 16.02390 -0.000000804 -0.000000301 +16398 16.02486 -0.000000803 -0.000000301 +16399 16.02583 -0.000000803 -0.000000301 +16400 16.02679 -0.000000803 -0.000000301 +16401 16.02776 -0.000000803 -0.000000300 +16402 16.02872 -0.000000802 -0.000000300 +16403 16.02969 -0.000000802 -0.000000300 +16404 16.03065 -0.000000802 -0.000000300 +16405 16.03162 -0.000000801 -0.000000300 +16406 16.03258 -0.000000801 -0.000000300 +16407 16.03355 -0.000000801 -0.000000300 +16408 16.03451 -0.000000800 -0.000000300 +16409 16.03548 -0.000000800 -0.000000299 +16410 16.03644 -0.000000800 -0.000000299 +16411 16.03741 -0.000000800 -0.000000299 +16412 16.03837 -0.000000799 -0.000000299 +16413 16.03934 -0.000000799 -0.000000299 +16414 16.04030 -0.000000799 -0.000000299 +16415 16.04127 -0.000000798 -0.000000299 +16416 16.04223 -0.000000798 -0.000000299 +16417 16.04320 -0.000000798 -0.000000298 +16418 16.04416 -0.000000798 -0.000000298 +16419 16.04513 -0.000000797 -0.000000298 +16420 16.04609 -0.000000797 -0.000000298 +16421 16.04706 -0.000000797 -0.000000298 +16422 16.04802 -0.000000796 -0.000000298 +16423 16.04899 -0.000000796 -0.000000298 +16424 16.04995 -0.000000796 -0.000000298 +16425 16.05092 -0.000000796 -0.000000297 +16426 16.05188 -0.000000795 -0.000000297 +16427 16.05285 -0.000000795 -0.000000297 +16428 16.05381 -0.000000795 -0.000000297 +16429 16.05478 -0.000000794 -0.000000297 +16430 16.05574 -0.000000794 -0.000000297 +16431 16.05671 -0.000000794 -0.000000297 +16432 16.05767 -0.000000794 -0.000000297 +16433 16.05864 -0.000000793 -0.000000296 +16434 16.05960 -0.000000793 -0.000000296 +16435 16.06057 -0.000000793 -0.000000296 +16436 16.06153 -0.000000792 -0.000000296 +16437 16.06250 -0.000000792 -0.000000296 +16438 16.06346 -0.000000792 -0.000000296 +16439 16.06443 -0.000000792 -0.000000296 +16440 16.06539 -0.000000791 -0.000000296 +16441 16.06636 -0.000000791 -0.000000295 +16442 16.06732 -0.000000791 -0.000000295 +16443 16.06829 -0.000000790 -0.000000295 +16444 16.06925 -0.000000790 -0.000000295 +16445 16.07022 -0.000000790 -0.000000295 +16446 16.07118 -0.000000790 -0.000000295 +16447 16.07215 -0.000000789 -0.000000295 +16448 16.07311 -0.000000789 -0.000000295 +16449 16.07408 -0.000000789 -0.000000294 +16450 16.07504 -0.000000788 -0.000000294 +16451 16.07601 -0.000000788 -0.000000294 +16452 16.07697 -0.000000788 -0.000000294 +16453 16.07794 -0.000000788 -0.000000294 +16454 16.07890 -0.000000787 -0.000000294 +16455 16.07987 -0.000000787 -0.000000294 +16456 16.08083 -0.000000787 -0.000000294 +16457 16.08180 -0.000000786 -0.000000293 +16458 16.08276 -0.000000786 -0.000000293 +16459 16.08373 -0.000000786 -0.000000293 +16460 16.08469 -0.000000786 -0.000000293 +16461 16.08566 -0.000000785 -0.000000293 +16462 16.08662 -0.000000785 -0.000000293 +16463 16.08759 -0.000000785 -0.000000293 +16464 16.08855 -0.000000784 -0.000000293 +16465 16.08952 -0.000000784 -0.000000292 +16466 16.09048 -0.000000784 -0.000000292 +16467 16.09145 -0.000000784 -0.000000292 +16468 16.09241 -0.000000783 -0.000000292 +16469 16.09338 -0.000000783 -0.000000292 +16470 16.09434 -0.000000783 -0.000000292 +16471 16.09531 -0.000000783 -0.000000292 +16472 16.09627 -0.000000782 -0.000000292 +16473 16.09724 -0.000000782 -0.000000291 +16474 16.09820 -0.000000782 -0.000000291 +16475 16.09917 -0.000000781 -0.000000291 +16476 16.10014 -0.000000781 -0.000000291 +16477 16.10110 -0.000000781 -0.000000291 +16478 16.10207 -0.000000781 -0.000000291 +16479 16.10303 -0.000000780 -0.000000291 +16480 16.10400 -0.000000780 -0.000000291 +16481 16.10496 -0.000000780 -0.000000290 +16482 16.10593 -0.000000779 -0.000000290 +16483 16.10689 -0.000000779 -0.000000290 +16484 16.10786 -0.000000779 -0.000000290 +16485 16.10882 -0.000000779 -0.000000290 +16486 16.10979 -0.000000778 -0.000000290 +16487 16.11075 -0.000000778 -0.000000290 +16488 16.11172 -0.000000778 -0.000000290 +16489 16.11268 -0.000000777 -0.000000290 +16490 16.11365 -0.000000777 -0.000000289 +16491 16.11461 -0.000000777 -0.000000289 +16492 16.11558 -0.000000777 -0.000000289 +16493 16.11654 -0.000000776 -0.000000289 +16494 16.11751 -0.000000776 -0.000000289 +16495 16.11847 -0.000000776 -0.000000289 +16496 16.11944 -0.000000776 -0.000000289 +16497 16.12040 -0.000000775 -0.000000289 +16498 16.12137 -0.000000775 -0.000000288 +16499 16.12233 -0.000000775 -0.000000288 +16500 16.12330 -0.000000774 -0.000000288 +16501 16.12426 -0.000000774 -0.000000288 +16502 16.12523 -0.000000774 -0.000000288 +16503 16.12619 -0.000000774 -0.000000288 +16504 16.12716 -0.000000773 -0.000000288 +16505 16.12812 -0.000000773 -0.000000288 +16506 16.12909 -0.000000773 -0.000000287 +16507 16.13005 -0.000000772 -0.000000287 +16508 16.13102 -0.000000772 -0.000000287 +16509 16.13198 -0.000000772 -0.000000287 +16510 16.13295 -0.000000772 -0.000000287 +16511 16.13391 -0.000000771 -0.000000287 +16512 16.13488 -0.000000771 -0.000000287 +16513 16.13584 -0.000000771 -0.000000287 +16514 16.13681 -0.000000771 -0.000000286 +16515 16.13777 -0.000000770 -0.000000286 +16516 16.13874 -0.000000770 -0.000000286 +16517 16.13970 -0.000000770 -0.000000286 +16518 16.14067 -0.000000769 -0.000000286 +16519 16.14163 -0.000000769 -0.000000286 +16520 16.14260 -0.000000769 -0.000000286 +16521 16.14356 -0.000000769 -0.000000286 +16522 16.14453 -0.000000768 -0.000000286 +16523 16.14549 -0.000000768 -0.000000285 +16524 16.14646 -0.000000768 -0.000000285 +16525 16.14742 -0.000000767 -0.000000285 +16526 16.14839 -0.000000767 -0.000000285 +16527 16.14935 -0.000000767 -0.000000285 +16528 16.15032 -0.000000767 -0.000000285 +16529 16.15128 -0.000000766 -0.000000285 +16530 16.15225 -0.000000766 -0.000000285 +16531 16.15321 -0.000000766 -0.000000284 +16532 16.15418 -0.000000766 -0.000000284 +16533 16.15514 -0.000000765 -0.000000284 +16534 16.15611 -0.000000765 -0.000000284 +16535 16.15707 -0.000000765 -0.000000284 +16536 16.15804 -0.000000764 -0.000000284 +16537 16.15900 -0.000000764 -0.000000284 +16538 16.15997 -0.000000764 -0.000000284 +16539 16.16093 -0.000000764 -0.000000284 +16540 16.16190 -0.000000763 -0.000000283 +16541 16.16286 -0.000000763 -0.000000283 +16542 16.16383 -0.000000763 -0.000000283 +16543 16.16479 -0.000000763 -0.000000283 +16544 16.16576 -0.000000762 -0.000000283 +16545 16.16672 -0.000000762 -0.000000283 +16546 16.16769 -0.000000762 -0.000000283 +16547 16.16865 -0.000000761 -0.000000283 +16548 16.16962 -0.000000761 -0.000000282 +16549 16.17058 -0.000000761 -0.000000282 +16550 16.17155 -0.000000761 -0.000000282 +16551 16.17251 -0.000000760 -0.000000282 +16552 16.17348 -0.000000760 -0.000000282 +16553 16.17444 -0.000000760 -0.000000282 +16554 16.17541 -0.000000760 -0.000000282 +16555 16.17637 -0.000000759 -0.000000282 +16556 16.17734 -0.000000759 -0.000000282 +16557 16.17830 -0.000000759 -0.000000281 +16558 16.17927 -0.000000758 -0.000000281 +16559 16.18023 -0.000000758 -0.000000281 +16560 16.18120 -0.000000758 -0.000000281 +16561 16.18216 -0.000000758 -0.000000281 +16562 16.18313 -0.000000757 -0.000000281 +16563 16.18409 -0.000000757 -0.000000281 +16564 16.18506 -0.000000757 -0.000000281 +16565 16.18602 -0.000000757 -0.000000280 +16566 16.18699 -0.000000756 -0.000000280 +16567 16.18795 -0.000000756 -0.000000280 +16568 16.18892 -0.000000756 -0.000000280 +16569 16.18988 -0.000000755 -0.000000280 +16570 16.19085 -0.000000755 -0.000000280 +16571 16.19181 -0.000000755 -0.000000280 +16572 16.19278 -0.000000755 -0.000000280 +16573 16.19374 -0.000000754 -0.000000280 +16574 16.19471 -0.000000754 -0.000000279 +16575 16.19567 -0.000000754 -0.000000279 +16576 16.19664 -0.000000754 -0.000000279 +16577 16.19760 -0.000000753 -0.000000279 +16578 16.19857 -0.000000753 -0.000000279 +16579 16.19953 -0.000000753 -0.000000279 +16580 16.20050 -0.000000753 -0.000000279 +16581 16.20147 -0.000000752 -0.000000279 +16582 16.20243 -0.000000752 -0.000000278 +16583 16.20340 -0.000000752 -0.000000278 +16584 16.20436 -0.000000751 -0.000000278 +16585 16.20533 -0.000000751 -0.000000278 +16586 16.20629 -0.000000751 -0.000000278 +16587 16.20726 -0.000000751 -0.000000278 +16588 16.20822 -0.000000750 -0.000000278 +16589 16.20919 -0.000000750 -0.000000278 +16590 16.21015 -0.000000750 -0.000000278 +16591 16.21112 -0.000000750 -0.000000277 +16592 16.21208 -0.000000749 -0.000000277 +16593 16.21305 -0.000000749 -0.000000277 +16594 16.21401 -0.000000749 -0.000000277 +16595 16.21498 -0.000000748 -0.000000277 +16596 16.21594 -0.000000748 -0.000000277 +16597 16.21691 -0.000000748 -0.000000277 +16598 16.21787 -0.000000748 -0.000000277 +16599 16.21884 -0.000000747 -0.000000277 +16600 16.21980 -0.000000747 -0.000000276 +16601 16.22077 -0.000000747 -0.000000276 +16602 16.22173 -0.000000747 -0.000000276 +16603 16.22270 -0.000000746 -0.000000276 +16604 16.22366 -0.000000746 -0.000000276 +16605 16.22463 -0.000000746 -0.000000276 +16606 16.22559 -0.000000746 -0.000000276 +16607 16.22656 -0.000000745 -0.000000276 +16608 16.22752 -0.000000745 -0.000000275 +16609 16.22849 -0.000000745 -0.000000275 +16610 16.22945 -0.000000744 -0.000000275 +16611 16.23042 -0.000000744 -0.000000275 +16612 16.23138 -0.000000744 -0.000000275 +16613 16.23235 -0.000000744 -0.000000275 +16614 16.23331 -0.000000743 -0.000000275 +16615 16.23428 -0.000000743 -0.000000275 +16616 16.23524 -0.000000743 -0.000000275 +16617 16.23621 -0.000000743 -0.000000274 +16618 16.23717 -0.000000742 -0.000000274 +16619 16.23814 -0.000000742 -0.000000274 +16620 16.23910 -0.000000742 -0.000000274 +16621 16.24007 -0.000000742 -0.000000274 +16622 16.24103 -0.000000741 -0.000000274 +16623 16.24200 -0.000000741 -0.000000274 +16624 16.24296 -0.000000741 -0.000000274 +16625 16.24393 -0.000000741 -0.000000274 +16626 16.24489 -0.000000740 -0.000000273 +16627 16.24586 -0.000000740 -0.000000273 +16628 16.24682 -0.000000740 -0.000000273 +16629 16.24779 -0.000000739 -0.000000273 +16630 16.24875 -0.000000739 -0.000000273 +16631 16.24972 -0.000000739 -0.000000273 +16632 16.25068 -0.000000739 -0.000000273 +16633 16.25165 -0.000000738 -0.000000273 +16634 16.25261 -0.000000738 -0.000000273 +16635 16.25358 -0.000000738 -0.000000272 +16636 16.25454 -0.000000738 -0.000000272 +16637 16.25551 -0.000000737 -0.000000272 +16638 16.25647 -0.000000737 -0.000000272 +16639 16.25744 -0.000000737 -0.000000272 +16640 16.25840 -0.000000737 -0.000000272 +16641 16.25937 -0.000000736 -0.000000272 +16642 16.26033 -0.000000736 -0.000000272 +16643 16.26130 -0.000000736 -0.000000271 +16644 16.26226 -0.000000736 -0.000000271 +16645 16.26323 -0.000000735 -0.000000271 +16646 16.26419 -0.000000735 -0.000000271 +16647 16.26516 -0.000000735 -0.000000271 +16648 16.26612 -0.000000734 -0.000000271 +16649 16.26709 -0.000000734 -0.000000271 +16650 16.26805 -0.000000734 -0.000000271 +16651 16.26902 -0.000000734 -0.000000271 +16652 16.26998 -0.000000733 -0.000000270 +16653 16.27095 -0.000000733 -0.000000270 +16654 16.27191 -0.000000733 -0.000000270 +16655 16.27288 -0.000000733 -0.000000270 +16656 16.27384 -0.000000732 -0.000000270 +16657 16.27481 -0.000000732 -0.000000270 +16658 16.27577 -0.000000732 -0.000000270 +16659 16.27674 -0.000000732 -0.000000270 +16660 16.27770 -0.000000731 -0.000000270 +16661 16.27867 -0.000000731 -0.000000269 +16662 16.27963 -0.000000731 -0.000000269 +16663 16.28060 -0.000000731 -0.000000269 +16664 16.28156 -0.000000730 -0.000000269 +16665 16.28253 -0.000000730 -0.000000269 +16666 16.28349 -0.000000730 -0.000000269 +16667 16.28446 -0.000000730 -0.000000269 +16668 16.28542 -0.000000729 -0.000000269 +16669 16.28639 -0.000000729 -0.000000269 +16670 16.28735 -0.000000729 -0.000000268 +16671 16.28832 -0.000000728 -0.000000268 +16672 16.28928 -0.000000728 -0.000000268 +16673 16.29025 -0.000000728 -0.000000268 +16674 16.29121 -0.000000728 -0.000000268 +16675 16.29218 -0.000000727 -0.000000268 +16676 16.29314 -0.000000727 -0.000000268 +16677 16.29411 -0.000000727 -0.000000268 +16678 16.29507 -0.000000727 -0.000000268 +16679 16.29604 -0.000000726 -0.000000267 +16680 16.29700 -0.000000726 -0.000000267 +16681 16.29797 -0.000000726 -0.000000267 +16682 16.29893 -0.000000726 -0.000000267 +16683 16.29990 -0.000000725 -0.000000267 +16684 16.30087 -0.000000725 -0.000000267 +16685 16.30183 -0.000000725 -0.000000267 +16686 16.30280 -0.000000725 -0.000000267 +16687 16.30376 -0.000000724 -0.000000267 +16688 16.30473 -0.000000724 -0.000000266 +16689 16.30569 -0.000000724 -0.000000266 +16690 16.30666 -0.000000724 -0.000000266 +16691 16.30762 -0.000000723 -0.000000266 +16692 16.30859 -0.000000723 -0.000000266 +16693 16.30955 -0.000000723 -0.000000266 +16694 16.31052 -0.000000723 -0.000000266 +16695 16.31148 -0.000000722 -0.000000266 +16696 16.31245 -0.000000722 -0.000000266 +16697 16.31341 -0.000000722 -0.000000265 +16698 16.31438 -0.000000722 -0.000000265 +16699 16.31534 -0.000000721 -0.000000265 +16700 16.31631 -0.000000721 -0.000000265 +16701 16.31727 -0.000000721 -0.000000265 +16702 16.31824 -0.000000721 -0.000000265 +16703 16.31920 -0.000000720 -0.000000265 +16704 16.32017 -0.000000720 -0.000000265 +16705 16.32113 -0.000000720 -0.000000265 +16706 16.32210 -0.000000719 -0.000000264 +16707 16.32306 -0.000000719 -0.000000264 +16708 16.32403 -0.000000719 -0.000000264 +16709 16.32499 -0.000000719 -0.000000264 +16710 16.32596 -0.000000718 -0.000000264 +16711 16.32692 -0.000000718 -0.000000264 +16712 16.32789 -0.000000718 -0.000000264 +16713 16.32885 -0.000000718 -0.000000264 +16714 16.32982 -0.000000717 -0.000000264 +16715 16.33078 -0.000000717 -0.000000264 +16716 16.33175 -0.000000717 -0.000000263 +16717 16.33271 -0.000000717 -0.000000263 +16718 16.33368 -0.000000716 -0.000000263 +16719 16.33464 -0.000000716 -0.000000263 +16720 16.33561 -0.000000716 -0.000000263 +16721 16.33657 -0.000000716 -0.000000263 +16722 16.33754 -0.000000715 -0.000000263 +16723 16.33850 -0.000000715 -0.000000263 +16724 16.33947 -0.000000715 -0.000000263 +16725 16.34043 -0.000000715 -0.000000262 +16726 16.34140 -0.000000714 -0.000000262 +16727 16.34236 -0.000000714 -0.000000262 +16728 16.34333 -0.000000714 -0.000000262 +16729 16.34429 -0.000000714 -0.000000262 +16730 16.34526 -0.000000713 -0.000000262 +16731 16.34622 -0.000000713 -0.000000262 +16732 16.34719 -0.000000713 -0.000000262 +16733 16.34815 -0.000000713 -0.000000262 +16734 16.34912 -0.000000712 -0.000000261 +16735 16.35008 -0.000000712 -0.000000261 +16736 16.35105 -0.000000712 -0.000000261 +16737 16.35201 -0.000000712 -0.000000261 +16738 16.35298 -0.000000711 -0.000000261 +16739 16.35394 -0.000000711 -0.000000261 +16740 16.35491 -0.000000711 -0.000000261 +16741 16.35587 -0.000000711 -0.000000261 +16742 16.35684 -0.000000710 -0.000000261 +16743 16.35780 -0.000000710 -0.000000260 +16744 16.35877 -0.000000710 -0.000000260 +16745 16.35973 -0.000000710 -0.000000260 +16746 16.36070 -0.000000709 -0.000000260 +16747 16.36166 -0.000000709 -0.000000260 +16748 16.36263 -0.000000709 -0.000000260 +16749 16.36359 -0.000000709 -0.000000260 +16750 16.36456 -0.000000708 -0.000000260 +16751 16.36552 -0.000000708 -0.000000260 +16752 16.36649 -0.000000708 -0.000000260 +16753 16.36745 -0.000000708 -0.000000259 +16754 16.36842 -0.000000707 -0.000000259 +16755 16.36938 -0.000000707 -0.000000259 +16756 16.37035 -0.000000707 -0.000000259 +16757 16.37131 -0.000000707 -0.000000259 +16758 16.37228 -0.000000706 -0.000000259 +16759 16.37324 -0.000000706 -0.000000259 +16760 16.37421 -0.000000706 -0.000000259 +16761 16.37517 -0.000000706 -0.000000259 +16762 16.37614 -0.000000705 -0.000000258 +16763 16.37710 -0.000000705 -0.000000258 +16764 16.37807 -0.000000705 -0.000000258 +16765 16.37903 -0.000000705 -0.000000258 +16766 16.38000 -0.000000704 -0.000000258 +16767 16.38096 -0.000000704 -0.000000258 +16768 16.38193 -0.000000704 -0.000000258 +16769 16.38289 -0.000000704 -0.000000258 +16770 16.38386 -0.000000703 -0.000000258 +16771 16.38482 -0.000000703 -0.000000257 +16772 16.38579 -0.000000703 -0.000000257 +16773 16.38675 -0.000000703 -0.000000257 +16774 16.38772 -0.000000702 -0.000000257 +16775 16.38868 -0.000000702 -0.000000257 +16776 16.38965 -0.000000702 -0.000000257 +16777 16.39061 -0.000000702 -0.000000257 +16778 16.39158 -0.000000701 -0.000000257 +16779 16.39254 -0.000000701 -0.000000257 +16780 16.39351 -0.000000701 -0.000000257 +16781 16.39447 -0.000000701 -0.000000256 +16782 16.39544 -0.000000700 -0.000000256 +16783 16.39640 -0.000000700 -0.000000256 +16784 16.39737 -0.000000700 -0.000000256 +16785 16.39833 -0.000000700 -0.000000256 +16786 16.39930 -0.000000699 -0.000000256 +16787 16.40027 -0.000000699 -0.000000256 +16788 16.40123 -0.000000699 -0.000000256 +16789 16.40220 -0.000000699 -0.000000256 +16790 16.40316 -0.000000698 -0.000000255 +16791 16.40413 -0.000000698 -0.000000255 +16792 16.40509 -0.000000698 -0.000000255 +16793 16.40606 -0.000000698 -0.000000255 +16794 16.40702 -0.000000697 -0.000000255 +16795 16.40799 -0.000000697 -0.000000255 +16796 16.40895 -0.000000697 -0.000000255 +16797 16.40992 -0.000000697 -0.000000255 +16798 16.41088 -0.000000696 -0.000000255 +16799 16.41185 -0.000000696 -0.000000255 +16800 16.41281 -0.000000696 -0.000000254 +16801 16.41378 -0.000000696 -0.000000254 +16802 16.41474 -0.000000695 -0.000000254 +16803 16.41571 -0.000000695 -0.000000254 +16804 16.41667 -0.000000695 -0.000000254 +16805 16.41764 -0.000000695 -0.000000254 +16806 16.41860 -0.000000694 -0.000000254 +16807 16.41957 -0.000000694 -0.000000254 +16808 16.42053 -0.000000694 -0.000000254 +16809 16.42150 -0.000000694 -0.000000253 +16810 16.42246 -0.000000694 -0.000000253 +16811 16.42343 -0.000000693 -0.000000253 +16812 16.42439 -0.000000693 -0.000000253 +16813 16.42536 -0.000000693 -0.000000253 +16814 16.42632 -0.000000693 -0.000000253 +16815 16.42729 -0.000000692 -0.000000253 +16816 16.42825 -0.000000692 -0.000000253 +16817 16.42922 -0.000000692 -0.000000253 +16818 16.43018 -0.000000692 -0.000000253 +16819 16.43115 -0.000000691 -0.000000252 +16820 16.43211 -0.000000691 -0.000000252 +16821 16.43308 -0.000000691 -0.000000252 +16822 16.43404 -0.000000691 -0.000000252 +16823 16.43501 -0.000000690 -0.000000252 +16824 16.43597 -0.000000690 -0.000000252 +16825 16.43694 -0.000000690 -0.000000252 +16826 16.43790 -0.000000690 -0.000000252 +16827 16.43887 -0.000000689 -0.000000252 +16828 16.43983 -0.000000689 -0.000000252 +16829 16.44080 -0.000000689 -0.000000251 +16830 16.44176 -0.000000689 -0.000000251 +16831 16.44273 -0.000000688 -0.000000251 +16832 16.44369 -0.000000688 -0.000000251 +16833 16.44466 -0.000000688 -0.000000251 +16834 16.44562 -0.000000688 -0.000000251 +16835 16.44659 -0.000000687 -0.000000251 +16836 16.44755 -0.000000687 -0.000000251 +16837 16.44852 -0.000000687 -0.000000251 +16838 16.44948 -0.000000687 -0.000000250 +16839 16.45045 -0.000000686 -0.000000250 +16840 16.45141 -0.000000686 -0.000000250 +16841 16.45238 -0.000000686 -0.000000250 +16842 16.45334 -0.000000686 -0.000000250 +16843 16.45431 -0.000000685 -0.000000250 +16844 16.45527 -0.000000685 -0.000000250 +16845 16.45624 -0.000000685 -0.000000250 +16846 16.45720 -0.000000685 -0.000000250 +16847 16.45817 -0.000000685 -0.000000250 +16848 16.45913 -0.000000684 -0.000000249 +16849 16.46010 -0.000000684 -0.000000249 +16850 16.46106 -0.000000684 -0.000000249 +16851 16.46203 -0.000000684 -0.000000249 +16852 16.46299 -0.000000683 -0.000000249 +16853 16.46396 -0.000000683 -0.000000249 +16854 16.46492 -0.000000683 -0.000000249 +16855 16.46589 -0.000000683 -0.000000249 +16856 16.46685 -0.000000682 -0.000000249 +16857 16.46782 -0.000000682 -0.000000249 +16858 16.46878 -0.000000682 -0.000000248 +16859 16.46975 -0.000000682 -0.000000248 +16860 16.47071 -0.000000681 -0.000000248 +16861 16.47168 -0.000000681 -0.000000248 +16862 16.47264 -0.000000681 -0.000000248 +16863 16.47361 -0.000000681 -0.000000248 +16864 16.47457 -0.000000680 -0.000000248 +16865 16.47554 -0.000000680 -0.000000248 +16866 16.47650 -0.000000680 -0.000000248 +16867 16.47747 -0.000000680 -0.000000248 +16868 16.47843 -0.000000679 -0.000000247 +16869 16.47940 -0.000000679 -0.000000247 +16870 16.48036 -0.000000679 -0.000000247 +16871 16.48133 -0.000000679 -0.000000247 +16872 16.48229 -0.000000679 -0.000000247 +16873 16.48326 -0.000000678 -0.000000247 +16874 16.48422 -0.000000678 -0.000000247 +16875 16.48519 -0.000000678 -0.000000247 +16876 16.48615 -0.000000678 -0.000000247 +16877 16.48712 -0.000000677 -0.000000247 +16878 16.48808 -0.000000677 -0.000000246 +16879 16.48905 -0.000000677 -0.000000246 +16880 16.49001 -0.000000677 -0.000000246 +16881 16.49098 -0.000000676 -0.000000246 +16882 16.49194 -0.000000676 -0.000000246 +16883 16.49291 -0.000000676 -0.000000246 +16884 16.49387 -0.000000676 -0.000000246 +16885 16.49484 -0.000000675 -0.000000246 +16886 16.49580 -0.000000675 -0.000000246 +16887 16.49677 -0.000000675 -0.000000245 +16888 16.49773 -0.000000675 -0.000000245 +16889 16.49870 -0.000000675 -0.000000245 +16890 16.49966 -0.000000674 -0.000000245 +16891 16.50063 -0.000000674 -0.000000245 +16892 16.50160 -0.000000674 -0.000000245 +16893 16.50256 -0.000000674 -0.000000245 +16894 16.50353 -0.000000673 -0.000000245 +16895 16.50449 -0.000000673 -0.000000245 +16896 16.50546 -0.000000673 -0.000000245 +16897 16.50642 -0.000000673 -0.000000244 +16898 16.50739 -0.000000672 -0.000000244 +16899 16.50835 -0.000000672 -0.000000244 +16900 16.50932 -0.000000672 -0.000000244 +16901 16.51028 -0.000000672 -0.000000244 +16902 16.51125 -0.000000671 -0.000000244 +16903 16.51221 -0.000000671 -0.000000244 +16904 16.51318 -0.000000671 -0.000000244 +16905 16.51414 -0.000000671 -0.000000244 +16906 16.51511 -0.000000670 -0.000000244 +16907 16.51607 -0.000000670 -0.000000243 +16908 16.51704 -0.000000670 -0.000000243 +16909 16.51800 -0.000000670 -0.000000243 +16910 16.51897 -0.000000670 -0.000000243 +16911 16.51993 -0.000000669 -0.000000243 +16912 16.52090 -0.000000669 -0.000000243 +16913 16.52186 -0.000000669 -0.000000243 +16914 16.52283 -0.000000669 -0.000000243 +16915 16.52379 -0.000000668 -0.000000243 +16916 16.52476 -0.000000668 -0.000000243 +16917 16.52572 -0.000000668 -0.000000242 +16918 16.52669 -0.000000668 -0.000000242 +16919 16.52765 -0.000000667 -0.000000242 +16920 16.52862 -0.000000667 -0.000000242 +16921 16.52958 -0.000000667 -0.000000242 +16922 16.53055 -0.000000667 -0.000000242 +16923 16.53151 -0.000000667 -0.000000242 +16924 16.53248 -0.000000666 -0.000000242 +16925 16.53344 -0.000000666 -0.000000242 +16926 16.53441 -0.000000666 -0.000000242 +16927 16.53537 -0.000000666 -0.000000242 +16928 16.53634 -0.000000665 -0.000000241 +16929 16.53730 -0.000000665 -0.000000241 +16930 16.53827 -0.000000665 -0.000000241 +16931 16.53923 -0.000000665 -0.000000241 +16932 16.54020 -0.000000664 -0.000000241 +16933 16.54116 -0.000000664 -0.000000241 +16934 16.54213 -0.000000664 -0.000000241 +16935 16.54309 -0.000000664 -0.000000241 +16936 16.54406 -0.000000663 -0.000000241 +16937 16.54502 -0.000000663 -0.000000241 +16938 16.54599 -0.000000663 -0.000000240 +16939 16.54695 -0.000000663 -0.000000240 +16940 16.54792 -0.000000663 -0.000000240 +16941 16.54888 -0.000000662 -0.000000240 +16942 16.54985 -0.000000662 -0.000000240 +16943 16.55081 -0.000000662 -0.000000240 +16944 16.55178 -0.000000662 -0.000000240 +16945 16.55274 -0.000000661 -0.000000240 +16946 16.55371 -0.000000661 -0.000000240 +16947 16.55467 -0.000000661 -0.000000240 +16948 16.55564 -0.000000661 -0.000000239 +16949 16.55660 -0.000000660 -0.000000239 +16950 16.55757 -0.000000660 -0.000000239 +16951 16.55853 -0.000000660 -0.000000239 +16952 16.55950 -0.000000660 -0.000000239 +16953 16.56046 -0.000000660 -0.000000239 +16954 16.56143 -0.000000659 -0.000000239 +16955 16.56239 -0.000000659 -0.000000239 +16956 16.56336 -0.000000659 -0.000000239 +16957 16.56432 -0.000000659 -0.000000239 +16958 16.56529 -0.000000658 -0.000000238 +16959 16.56625 -0.000000658 -0.000000238 +16960 16.56722 -0.000000658 -0.000000238 +16961 16.56818 -0.000000658 -0.000000238 +16962 16.56915 -0.000000657 -0.000000238 +16963 16.57011 -0.000000657 -0.000000238 +16964 16.57108 -0.000000657 -0.000000238 +16965 16.57204 -0.000000657 -0.000000238 +16966 16.57301 -0.000000657 -0.000000238 +16967 16.57397 -0.000000656 -0.000000238 +16968 16.57494 -0.000000656 -0.000000238 +16969 16.57590 -0.000000656 -0.000000237 +16970 16.57687 -0.000000656 -0.000000237 +16971 16.57783 -0.000000655 -0.000000237 +16972 16.57880 -0.000000655 -0.000000237 +16973 16.57976 -0.000000655 -0.000000237 +16974 16.58073 -0.000000655 -0.000000237 +16975 16.58169 -0.000000654 -0.000000237 +16976 16.58266 -0.000000654 -0.000000237 +16977 16.58362 -0.000000654 -0.000000237 +16978 16.58459 -0.000000654 -0.000000237 +16979 16.58555 -0.000000654 -0.000000236 +16980 16.58652 -0.000000653 -0.000000236 +16981 16.58748 -0.000000653 -0.000000236 +16982 16.58845 -0.000000653 -0.000000236 +16983 16.58941 -0.000000653 -0.000000236 +16984 16.59038 -0.000000652 -0.000000236 +16985 16.59134 -0.000000652 -0.000000236 +16986 16.59231 -0.000000652 -0.000000236 +16987 16.59327 -0.000000652 -0.000000236 +16988 16.59424 -0.000000652 -0.000000236 +16989 16.59520 -0.000000651 -0.000000235 +16990 16.59617 -0.000000651 -0.000000235 +16991 16.59713 -0.000000651 -0.000000235 +16992 16.59810 -0.000000651 -0.000000235 +16993 16.59906 -0.000000650 -0.000000235 +16994 16.60003 -0.000000650 -0.000000235 +16995 16.60100 -0.000000650 -0.000000235 +16996 16.60196 -0.000000650 -0.000000235 +16997 16.60293 -0.000000649 -0.000000235 +16998 16.60389 -0.000000649 -0.000000235 +16999 16.60486 -0.000000649 -0.000000235 +17000 16.60582 -0.000000649 -0.000000234 +17001 16.60679 -0.000000649 -0.000000234 +17002 16.60775 -0.000000648 -0.000000234 +17003 16.60872 -0.000000648 -0.000000234 +17004 16.60968 -0.000000648 -0.000000234 +17005 16.61065 -0.000000648 -0.000000234 +17006 16.61161 -0.000000647 -0.000000234 +17007 16.61258 -0.000000647 -0.000000234 +17008 16.61354 -0.000000647 -0.000000234 +17009 16.61451 -0.000000647 -0.000000234 +17010 16.61547 -0.000000647 -0.000000233 +17011 16.61644 -0.000000646 -0.000000233 +17012 16.61740 -0.000000646 -0.000000233 +17013 16.61837 -0.000000646 -0.000000233 +17014 16.61933 -0.000000646 -0.000000233 +17015 16.62030 -0.000000645 -0.000000233 +17016 16.62126 -0.000000645 -0.000000233 +17017 16.62223 -0.000000645 -0.000000233 +17018 16.62319 -0.000000645 -0.000000233 +17019 16.62416 -0.000000645 -0.000000233 +17020 16.62512 -0.000000644 -0.000000233 +17021 16.62609 -0.000000644 -0.000000232 +17022 16.62705 -0.000000644 -0.000000232 +17023 16.62802 -0.000000644 -0.000000232 +17024 16.62898 -0.000000643 -0.000000232 +17025 16.62995 -0.000000643 -0.000000232 +17026 16.63091 -0.000000643 -0.000000232 +17027 16.63188 -0.000000643 -0.000000232 +17028 16.63284 -0.000000643 -0.000000232 +17029 16.63381 -0.000000642 -0.000000232 +17030 16.63477 -0.000000642 -0.000000232 +17031 16.63574 -0.000000642 -0.000000231 +17032 16.63670 -0.000000642 -0.000000231 +17033 16.63767 -0.000000641 -0.000000231 +17034 16.63863 -0.000000641 -0.000000231 +17035 16.63960 -0.000000641 -0.000000231 +17036 16.64056 -0.000000641 -0.000000231 +17037 16.64153 -0.000000641 -0.000000231 +17038 16.64249 -0.000000640 -0.000000231 +17039 16.64346 -0.000000640 -0.000000231 +17040 16.64442 -0.000000640 -0.000000231 +17041 16.64539 -0.000000640 -0.000000231 +17042 16.64635 -0.000000639 -0.000000230 +17043 16.64732 -0.000000639 -0.000000230 +17044 16.64828 -0.000000639 -0.000000230 +17045 16.64925 -0.000000639 -0.000000230 +17046 16.65021 -0.000000639 -0.000000230 +17047 16.65118 -0.000000638 -0.000000230 +17048 16.65214 -0.000000638 -0.000000230 +17049 16.65311 -0.000000638 -0.000000230 +17050 16.65407 -0.000000638 -0.000000230 +17051 16.65504 -0.000000637 -0.000000230 +17052 16.65600 -0.000000637 -0.000000230 +17053 16.65697 -0.000000637 -0.000000229 +17054 16.65793 -0.000000637 -0.000000229 +17055 16.65890 -0.000000637 -0.000000229 +17056 16.65986 -0.000000636 -0.000000229 +17057 16.66083 -0.000000636 -0.000000229 +17058 16.66179 -0.000000636 -0.000000229 +17059 16.66276 -0.000000636 -0.000000229 +17060 16.66372 -0.000000635 -0.000000229 +17061 16.66469 -0.000000635 -0.000000229 +17062 16.66565 -0.000000635 -0.000000229 +17063 16.66662 -0.000000635 -0.000000229 +17064 16.66758 -0.000000635 -0.000000228 +17065 16.66855 -0.000000634 -0.000000228 +17066 16.66951 -0.000000634 -0.000000228 +17067 16.67048 -0.000000634 -0.000000228 +17068 16.67144 -0.000000634 -0.000000228 +17069 16.67241 -0.000000633 -0.000000228 +17070 16.67337 -0.000000633 -0.000000228 +17071 16.67434 -0.000000633 -0.000000228 +17072 16.67530 -0.000000633 -0.000000228 +17073 16.67627 -0.000000633 -0.000000228 +17074 16.67723 -0.000000632 -0.000000227 +17075 16.67820 -0.000000632 -0.000000227 +17076 16.67916 -0.000000632 -0.000000227 +17077 16.68013 -0.000000632 -0.000000227 +17078 16.68109 -0.000000631 -0.000000227 +17079 16.68206 -0.000000631 -0.000000227 +17080 16.68302 -0.000000631 -0.000000227 +17081 16.68399 -0.000000631 -0.000000227 +17082 16.68495 -0.000000631 -0.000000227 +17083 16.68592 -0.000000630 -0.000000227 +17084 16.68688 -0.000000630 -0.000000227 +17085 16.68785 -0.000000630 -0.000000226 +17086 16.68881 -0.000000630 -0.000000226 +17087 16.68978 -0.000000629 -0.000000226 +17088 16.69074 -0.000000629 -0.000000226 +17089 16.69171 -0.000000629 -0.000000226 +17090 16.69267 -0.000000629 -0.000000226 +17091 16.69364 -0.000000629 -0.000000226 +17092 16.69460 -0.000000628 -0.000000226 +17093 16.69557 -0.000000628 -0.000000226 +17094 16.69653 -0.000000628 -0.000000226 +17095 16.69750 -0.000000628 -0.000000226 +17096 16.69846 -0.000000628 -0.000000225 +17097 16.69943 -0.000000627 -0.000000225 +17098 16.70040 -0.000000627 -0.000000225 +17099 16.70136 -0.000000627 -0.000000225 +17100 16.70233 -0.000000627 -0.000000225 +17101 16.70329 -0.000000626 -0.000000225 +17102 16.70426 -0.000000626 -0.000000225 +17103 16.70522 -0.000000626 -0.000000225 +17104 16.70619 -0.000000626 -0.000000225 +17105 16.70715 -0.000000626 -0.000000225 +17106 16.70812 -0.000000625 -0.000000225 +17107 16.70908 -0.000000625 -0.000000224 +17108 16.71005 -0.000000625 -0.000000224 +17109 16.71101 -0.000000625 -0.000000224 +17110 16.71198 -0.000000624 -0.000000224 +17111 16.71294 -0.000000624 -0.000000224 +17112 16.71391 -0.000000624 -0.000000224 +17113 16.71487 -0.000000624 -0.000000224 +17114 16.71584 -0.000000624 -0.000000224 +17115 16.71680 -0.000000623 -0.000000224 +17116 16.71777 -0.000000623 -0.000000224 +17117 16.71873 -0.000000623 -0.000000224 +17118 16.71970 -0.000000623 -0.000000223 +17119 16.72066 -0.000000623 -0.000000223 +17120 16.72163 -0.000000622 -0.000000223 +17121 16.72259 -0.000000622 -0.000000223 +17122 16.72356 -0.000000622 -0.000000223 +17123 16.72452 -0.000000622 -0.000000223 +17124 16.72549 -0.000000621 -0.000000223 +17125 16.72645 -0.000000621 -0.000000223 +17126 16.72742 -0.000000621 -0.000000223 +17127 16.72838 -0.000000621 -0.000000223 +17128 16.72935 -0.000000621 -0.000000223 +17129 16.73031 -0.000000620 -0.000000222 +17130 16.73128 -0.000000620 -0.000000222 +17131 16.73224 -0.000000620 -0.000000222 +17132 16.73321 -0.000000620 -0.000000222 +17133 16.73417 -0.000000620 -0.000000222 +17134 16.73514 -0.000000619 -0.000000222 +17135 16.73610 -0.000000619 -0.000000222 +17136 16.73707 -0.000000619 -0.000000222 +17137 16.73803 -0.000000619 -0.000000222 +17138 16.73900 -0.000000618 -0.000000222 +17139 16.73996 -0.000000618 -0.000000222 +17140 16.74093 -0.000000618 -0.000000222 +17141 16.74189 -0.000000618 -0.000000221 +17142 16.74286 -0.000000618 -0.000000221 +17143 16.74382 -0.000000617 -0.000000221 +17144 16.74479 -0.000000617 -0.000000221 +17145 16.74575 -0.000000617 -0.000000221 +17146 16.74672 -0.000000617 -0.000000221 +17147 16.74768 -0.000000617 -0.000000221 +17148 16.74865 -0.000000616 -0.000000221 +17149 16.74961 -0.000000616 -0.000000221 +17150 16.75058 -0.000000616 -0.000000221 +17151 16.75154 -0.000000616 -0.000000221 +17152 16.75251 -0.000000615 -0.000000220 +17153 16.75347 -0.000000615 -0.000000220 +17154 16.75444 -0.000000615 -0.000000220 +17155 16.75540 -0.000000615 -0.000000220 +17156 16.75637 -0.000000615 -0.000000220 +17157 16.75733 -0.000000614 -0.000000220 +17158 16.75830 -0.000000614 -0.000000220 +17159 16.75926 -0.000000614 -0.000000220 +17160 16.76023 -0.000000614 -0.000000220 +17161 16.76119 -0.000000614 -0.000000220 +17162 16.76216 -0.000000613 -0.000000220 +17163 16.76312 -0.000000613 -0.000000219 +17164 16.76409 -0.000000613 -0.000000219 +17165 16.76505 -0.000000613 -0.000000219 +17166 16.76602 -0.000000612 -0.000000219 +17167 16.76698 -0.000000612 -0.000000219 +17168 16.76795 -0.000000612 -0.000000219 +17169 16.76891 -0.000000612 -0.000000219 +17170 16.76988 -0.000000612 -0.000000219 +17171 16.77084 -0.000000611 -0.000000219 +17172 16.77181 -0.000000611 -0.000000219 +17173 16.77277 -0.000000611 -0.000000219 +17174 16.77374 -0.000000611 -0.000000218 +17175 16.77470 -0.000000611 -0.000000218 +17176 16.77567 -0.000000610 -0.000000218 +17177 16.77663 -0.000000610 -0.000000218 +17178 16.77760 -0.000000610 -0.000000218 +17179 16.77856 -0.000000610 -0.000000218 +17180 16.77953 -0.000000610 -0.000000218 +17181 16.78049 -0.000000609 -0.000000218 +17182 16.78146 -0.000000609 -0.000000218 +17183 16.78242 -0.000000609 -0.000000218 +17184 16.78339 -0.000000609 -0.000000218 +17185 16.78435 -0.000000608 -0.000000218 +17186 16.78532 -0.000000608 -0.000000217 +17187 16.78628 -0.000000608 -0.000000217 +17188 16.78725 -0.000000608 -0.000000217 +17189 16.78821 -0.000000608 -0.000000217 +17190 16.78918 -0.000000607 -0.000000217 +17191 16.79014 -0.000000607 -0.000000217 +17192 16.79111 -0.000000607 -0.000000217 +17193 16.79207 -0.000000607 -0.000000217 +17194 16.79304 -0.000000607 -0.000000217 +17195 16.79400 -0.000000606 -0.000000217 +17196 16.79497 -0.000000606 -0.000000217 +17197 16.79593 -0.000000606 -0.000000216 +17198 16.79690 -0.000000606 -0.000000216 +17199 16.79786 -0.000000606 -0.000000216 +17200 16.79883 -0.000000605 -0.000000216 +17201 16.79979 -0.000000605 -0.000000216 +17202 16.80076 -0.000000605 -0.000000216 +17203 16.80173 -0.000000605 -0.000000216 +17204 16.80269 -0.000000605 -0.000000216 +17205 16.80366 -0.000000604 -0.000000216 +17206 16.80462 -0.000000604 -0.000000216 +17207 16.80559 -0.000000604 -0.000000216 +17208 16.80655 -0.000000604 -0.000000216 +17209 16.80752 -0.000000603 -0.000000215 +17210 16.80848 -0.000000603 -0.000000215 +17211 16.80945 -0.000000603 -0.000000215 +17212 16.81041 -0.000000603 -0.000000215 +17213 16.81138 -0.000000603 -0.000000215 +17214 16.81234 -0.000000602 -0.000000215 +17215 16.81331 -0.000000602 -0.000000215 +17216 16.81427 -0.000000602 -0.000000215 +17217 16.81524 -0.000000602 -0.000000215 +17218 16.81620 -0.000000602 -0.000000215 +17219 16.81717 -0.000000601 -0.000000215 +17220 16.81813 -0.000000601 -0.000000214 +17221 16.81910 -0.000000601 -0.000000214 +17222 16.82006 -0.000000601 -0.000000214 +17223 16.82103 -0.000000601 -0.000000214 +17224 16.82199 -0.000000600 -0.000000214 +17225 16.82296 -0.000000600 -0.000000214 +17226 16.82392 -0.000000600 -0.000000214 +17227 16.82489 -0.000000600 -0.000000214 +17228 16.82585 -0.000000600 -0.000000214 +17229 16.82682 -0.000000599 -0.000000214 +17230 16.82778 -0.000000599 -0.000000214 +17231 16.82875 -0.000000599 -0.000000214 +17232 16.82971 -0.000000599 -0.000000213 +17233 16.83068 -0.000000599 -0.000000213 +17234 16.83164 -0.000000598 -0.000000213 +17235 16.83261 -0.000000598 -0.000000213 +17236 16.83357 -0.000000598 -0.000000213 +17237 16.83454 -0.000000598 -0.000000213 +17238 16.83550 -0.000000597 -0.000000213 +17239 16.83647 -0.000000597 -0.000000213 +17240 16.83743 -0.000000597 -0.000000213 +17241 16.83840 -0.000000597 -0.000000213 +17242 16.83936 -0.000000597 -0.000000213 +17243 16.84033 -0.000000596 -0.000000213 +17244 16.84129 -0.000000596 -0.000000212 +17245 16.84226 -0.000000596 -0.000000212 +17246 16.84322 -0.000000596 -0.000000212 +17247 16.84419 -0.000000596 -0.000000212 +17248 16.84515 -0.000000595 -0.000000212 +17249 16.84612 -0.000000595 -0.000000212 +17250 16.84708 -0.000000595 -0.000000212 +17251 16.84805 -0.000000595 -0.000000212 +17252 16.84901 -0.000000595 -0.000000212 +17253 16.84998 -0.000000594 -0.000000212 +17254 16.85094 -0.000000594 -0.000000212 +17255 16.85191 -0.000000594 -0.000000211 +17256 16.85287 -0.000000594 -0.000000211 +17257 16.85384 -0.000000594 -0.000000211 +17258 16.85480 -0.000000593 -0.000000211 +17259 16.85577 -0.000000593 -0.000000211 +17260 16.85673 -0.000000593 -0.000000211 +17261 16.85770 -0.000000593 -0.000000211 +17262 16.85866 -0.000000593 -0.000000211 +17263 16.85963 -0.000000592 -0.000000211 +17264 16.86059 -0.000000592 -0.000000211 +17265 16.86156 -0.000000592 -0.000000211 +17266 16.86252 -0.000000592 -0.000000211 +17267 16.86349 -0.000000592 -0.000000210 +17268 16.86445 -0.000000591 -0.000000210 +17269 16.86542 -0.000000591 -0.000000210 +17270 16.86638 -0.000000591 -0.000000210 +17271 16.86735 -0.000000591 -0.000000210 +17272 16.86831 -0.000000591 -0.000000210 +17273 16.86928 -0.000000590 -0.000000210 +17274 16.87024 -0.000000590 -0.000000210 +17275 16.87121 -0.000000590 -0.000000210 +17276 16.87217 -0.000000590 -0.000000210 +17277 16.87314 -0.000000590 -0.000000210 +17278 16.87410 -0.000000589 -0.000000210 +17279 16.87507 -0.000000589 -0.000000209 +17280 16.87603 -0.000000589 -0.000000209 +17281 16.87700 -0.000000589 -0.000000209 +17282 16.87796 -0.000000589 -0.000000209 +17283 16.87893 -0.000000588 -0.000000209 +17284 16.87989 -0.000000588 -0.000000209 +17285 16.88086 -0.000000588 -0.000000209 +17286 16.88182 -0.000000588 -0.000000209 +17287 16.88279 -0.000000588 -0.000000209 +17288 16.88375 -0.000000587 -0.000000209 +17289 16.88472 -0.000000587 -0.000000209 +17290 16.88568 -0.000000587 -0.000000209 +17291 16.88665 -0.000000587 -0.000000208 +17292 16.88761 -0.000000587 -0.000000208 +17293 16.88858 -0.000000586 -0.000000208 +17294 16.88954 -0.000000586 -0.000000208 +17295 16.89051 -0.000000586 -0.000000208 +17296 16.89147 -0.000000586 -0.000000208 +17297 16.89244 -0.000000586 -0.000000208 +17298 16.89340 -0.000000585 -0.000000208 +17299 16.89437 -0.000000585 -0.000000208 +17300 16.89533 -0.000000585 -0.000000208 +17301 16.89630 -0.000000585 -0.000000208 +17302 16.89726 -0.000000584 -0.000000208 +17303 16.89823 -0.000000584 -0.000000207 +17304 16.89919 -0.000000584 -0.000000207 +17305 16.90016 -0.000000584 -0.000000207 +17306 16.90113 -0.000000584 -0.000000207 +17307 16.90209 -0.000000583 -0.000000207 +17308 16.90306 -0.000000583 -0.000000207 +17309 16.90402 -0.000000583 -0.000000207 +17310 16.90499 -0.000000583 -0.000000207 +17311 16.90595 -0.000000583 -0.000000207 +17312 16.90692 -0.000000583 -0.000000207 +17313 16.90788 -0.000000582 -0.000000207 +17314 16.90885 -0.000000582 -0.000000207 +17315 16.90981 -0.000000582 -0.000000206 +17316 16.91078 -0.000000582 -0.000000206 +17317 16.91174 -0.000000582 -0.000000206 +17318 16.91271 -0.000000581 -0.000000206 +17319 16.91367 -0.000000581 -0.000000206 +17320 16.91464 -0.000000581 -0.000000206 +17321 16.91560 -0.000000581 -0.000000206 +17322 16.91657 -0.000000581 -0.000000206 +17323 16.91753 -0.000000580 -0.000000206 +17324 16.91850 -0.000000580 -0.000000206 +17325 16.91946 -0.000000580 -0.000000206 +17326 16.92043 -0.000000580 -0.000000206 +17327 16.92139 -0.000000580 -0.000000205 +17328 16.92236 -0.000000579 -0.000000205 +17329 16.92332 -0.000000579 -0.000000205 +17330 16.92429 -0.000000579 -0.000000205 +17331 16.92525 -0.000000579 -0.000000205 +17332 16.92622 -0.000000579 -0.000000205 +17333 16.92718 -0.000000578 -0.000000205 +17334 16.92815 -0.000000578 -0.000000205 +17335 16.92911 -0.000000578 -0.000000205 +17336 16.93008 -0.000000578 -0.000000205 +17337 16.93104 -0.000000578 -0.000000205 +17338 16.93201 -0.000000577 -0.000000205 +17339 16.93297 -0.000000577 -0.000000205 +17340 16.93394 -0.000000577 -0.000000204 +17341 16.93490 -0.000000577 -0.000000204 +17342 16.93587 -0.000000577 -0.000000204 +17343 16.93683 -0.000000576 -0.000000204 +17344 16.93780 -0.000000576 -0.000000204 +17345 16.93876 -0.000000576 -0.000000204 +17346 16.93973 -0.000000576 -0.000000204 +17347 16.94069 -0.000000576 -0.000000204 +17348 16.94166 -0.000000575 -0.000000204 +17349 16.94262 -0.000000575 -0.000000204 +17350 16.94359 -0.000000575 -0.000000204 +17351 16.94455 -0.000000575 -0.000000204 +17352 16.94552 -0.000000575 -0.000000203 +17353 16.94648 -0.000000574 -0.000000203 +17354 16.94745 -0.000000574 -0.000000203 +17355 16.94841 -0.000000574 -0.000000203 +17356 16.94938 -0.000000574 -0.000000203 +17357 16.95034 -0.000000574 -0.000000203 +17358 16.95131 -0.000000573 -0.000000203 +17359 16.95227 -0.000000573 -0.000000203 +17360 16.95324 -0.000000573 -0.000000203 +17361 16.95420 -0.000000573 -0.000000203 +17362 16.95517 -0.000000573 -0.000000203 +17363 16.95613 -0.000000572 -0.000000203 +17364 16.95710 -0.000000572 -0.000000202 +17365 16.95806 -0.000000572 -0.000000202 +17366 16.95903 -0.000000572 -0.000000202 +17367 16.95999 -0.000000572 -0.000000202 +17368 16.96096 -0.000000571 -0.000000202 +17369 16.96192 -0.000000571 -0.000000202 +17370 16.96289 -0.000000571 -0.000000202 +17371 16.96385 -0.000000571 -0.000000202 +17372 16.96482 -0.000000571 -0.000000202 +17373 16.96578 -0.000000570 -0.000000202 +17374 16.96675 -0.000000570 -0.000000202 +17375 16.96771 -0.000000570 -0.000000202 +17376 16.96868 -0.000000570 -0.000000202 +17377 16.96964 -0.000000570 -0.000000201 +17378 16.97061 -0.000000570 -0.000000201 +17379 16.97157 -0.000000569 -0.000000201 +17380 16.97254 -0.000000569 -0.000000201 +17381 16.97350 -0.000000569 -0.000000201 +17382 16.97447 -0.000000569 -0.000000201 +17383 16.97543 -0.000000569 -0.000000201 +17384 16.97640 -0.000000568 -0.000000201 +17385 16.97736 -0.000000568 -0.000000201 +17386 16.97833 -0.000000568 -0.000000201 +17387 16.97929 -0.000000568 -0.000000201 +17388 16.98026 -0.000000568 -0.000000201 +17389 16.98122 -0.000000567 -0.000000200 +17390 16.98219 -0.000000567 -0.000000200 +17391 16.98315 -0.000000567 -0.000000200 +17392 16.98412 -0.000000567 -0.000000200 +17393 16.98508 -0.000000567 -0.000000200 +17394 16.98605 -0.000000566 -0.000000200 +17395 16.98701 -0.000000566 -0.000000200 +17396 16.98798 -0.000000566 -0.000000200 +17397 16.98894 -0.000000566 -0.000000200 +17398 16.98991 -0.000000566 -0.000000200 +17399 16.99087 -0.000000565 -0.000000200 +17400 16.99184 -0.000000565 -0.000000200 +17401 16.99280 -0.000000565 -0.000000200 +17402 16.99377 -0.000000565 -0.000000199 +17403 16.99473 -0.000000565 -0.000000199 +17404 16.99570 -0.000000564 -0.000000199 +17405 16.99666 -0.000000564 -0.000000199 +17406 16.99763 -0.000000564 -0.000000199 +17407 16.99859 -0.000000564 -0.000000199 +17408 16.99956 -0.000000564 -0.000000199 +17409 17.00053 -0.000000564 -0.000000199 +17410 17.00149 -0.000000563 -0.000000199 +17411 17.00246 -0.000000563 -0.000000199 +17412 17.00342 -0.000000563 -0.000000199 +17413 17.00439 -0.000000563 -0.000000199 +17414 17.00535 -0.000000563 -0.000000198 +17415 17.00632 -0.000000562 -0.000000198 +17416 17.00728 -0.000000562 -0.000000198 +17417 17.00825 -0.000000562 -0.000000198 +17418 17.00921 -0.000000562 -0.000000198 +17419 17.01018 -0.000000562 -0.000000198 +17420 17.01114 -0.000000561 -0.000000198 +17421 17.01211 -0.000000561 -0.000000198 +17422 17.01307 -0.000000561 -0.000000198 +17423 17.01404 -0.000000561 -0.000000198 +17424 17.01500 -0.000000561 -0.000000198 +17425 17.01597 -0.000000560 -0.000000198 +17426 17.01693 -0.000000560 -0.000000198 +17427 17.01790 -0.000000560 -0.000000197 +17428 17.01886 -0.000000560 -0.000000197 +17429 17.01983 -0.000000560 -0.000000197 +17430 17.02079 -0.000000560 -0.000000197 +17431 17.02176 -0.000000559 -0.000000197 +17432 17.02272 -0.000000559 -0.000000197 +17433 17.02369 -0.000000559 -0.000000197 +17434 17.02465 -0.000000559 -0.000000197 +17435 17.02562 -0.000000559 -0.000000197 +17436 17.02658 -0.000000558 -0.000000197 +17437 17.02755 -0.000000558 -0.000000197 +17438 17.02851 -0.000000558 -0.000000197 +17439 17.02948 -0.000000558 -0.000000197 +17440 17.03044 -0.000000558 -0.000000196 +17441 17.03141 -0.000000557 -0.000000196 +17442 17.03237 -0.000000557 -0.000000196 +17443 17.03334 -0.000000557 -0.000000196 +17444 17.03430 -0.000000557 -0.000000196 +17445 17.03527 -0.000000557 -0.000000196 +17446 17.03623 -0.000000556 -0.000000196 +17447 17.03720 -0.000000556 -0.000000196 +17448 17.03816 -0.000000556 -0.000000196 +17449 17.03913 -0.000000556 -0.000000196 +17450 17.04009 -0.000000556 -0.000000196 +17451 17.04106 -0.000000556 -0.000000196 +17452 17.04202 -0.000000555 -0.000000196 +17453 17.04299 -0.000000555 -0.000000195 +17454 17.04395 -0.000000555 -0.000000195 +17455 17.04492 -0.000000555 -0.000000195 +17456 17.04588 -0.000000555 -0.000000195 +17457 17.04685 -0.000000554 -0.000000195 +17458 17.04781 -0.000000554 -0.000000195 +17459 17.04878 -0.000000554 -0.000000195 +17460 17.04974 -0.000000554 -0.000000195 +17461 17.05071 -0.000000554 -0.000000195 +17462 17.05167 -0.000000553 -0.000000195 +17463 17.05264 -0.000000553 -0.000000195 +17464 17.05360 -0.000000553 -0.000000195 +17465 17.05457 -0.000000553 -0.000000195 +17466 17.05553 -0.000000553 -0.000000194 +17467 17.05650 -0.000000553 -0.000000194 +17468 17.05746 -0.000000552 -0.000000194 +17469 17.05843 -0.000000552 -0.000000194 +17470 17.05939 -0.000000552 -0.000000194 +17471 17.06036 -0.000000552 -0.000000194 +17472 17.06132 -0.000000552 -0.000000194 +17473 17.06229 -0.000000551 -0.000000194 +17474 17.06325 -0.000000551 -0.000000194 +17475 17.06422 -0.000000551 -0.000000194 +17476 17.06518 -0.000000551 -0.000000194 +17477 17.06615 -0.000000551 -0.000000194 +17478 17.06711 -0.000000550 -0.000000194 +17479 17.06808 -0.000000550 -0.000000193 +17480 17.06904 -0.000000550 -0.000000193 +17481 17.07001 -0.000000550 -0.000000193 +17482 17.07097 -0.000000550 -0.000000193 +17483 17.07194 -0.000000550 -0.000000193 +17484 17.07290 -0.000000549 -0.000000193 +17485 17.07387 -0.000000549 -0.000000193 +17486 17.07483 -0.000000549 -0.000000193 +17487 17.07580 -0.000000549 -0.000000193 +17488 17.07676 -0.000000549 -0.000000193 +17489 17.07773 -0.000000548 -0.000000193 +17490 17.07869 -0.000000548 -0.000000193 +17491 17.07966 -0.000000548 -0.000000193 +17492 17.08062 -0.000000548 -0.000000192 +17493 17.08159 -0.000000548 -0.000000192 +17494 17.08255 -0.000000547 -0.000000192 +17495 17.08352 -0.000000547 -0.000000192 +17496 17.08448 -0.000000547 -0.000000192 +17497 17.08545 -0.000000547 -0.000000192 +17498 17.08641 -0.000000547 -0.000000192 +17499 17.08738 -0.000000547 -0.000000192 +17500 17.08834 -0.000000546 -0.000000192 +17501 17.08931 -0.000000546 -0.000000192 +17502 17.09027 -0.000000546 -0.000000192 +17503 17.09124 -0.000000546 -0.000000192 +17504 17.09220 -0.000000546 -0.000000192 +17505 17.09317 -0.000000545 -0.000000191 +17506 17.09413 -0.000000545 -0.000000191 +17507 17.09510 -0.000000545 -0.000000191 +17508 17.09606 -0.000000545 -0.000000191 +17509 17.09703 -0.000000545 -0.000000191 +17510 17.09799 -0.000000545 -0.000000191 +17511 17.09896 -0.000000544 -0.000000191 +17512 17.09992 -0.000000544 -0.000000191 +17513 17.10089 -0.000000544 -0.000000191 +17514 17.10186 -0.000000544 -0.000000191 +17515 17.10282 -0.000000544 -0.000000191 +17516 17.10379 -0.000000543 -0.000000191 +17517 17.10475 -0.000000543 -0.000000191 +17518 17.10572 -0.000000543 -0.000000190 +17519 17.10668 -0.000000543 -0.000000190 +17520 17.10765 -0.000000543 -0.000000190 +17521 17.10861 -0.000000542 -0.000000190 +17522 17.10958 -0.000000542 -0.000000190 +17523 17.11054 -0.000000542 -0.000000190 +17524 17.11151 -0.000000542 -0.000000190 +17525 17.11247 -0.000000542 -0.000000190 +17526 17.11344 -0.000000542 -0.000000190 +17527 17.11440 -0.000000541 -0.000000190 +17528 17.11537 -0.000000541 -0.000000190 +17529 17.11633 -0.000000541 -0.000000190 +17530 17.11730 -0.000000541 -0.000000190 +17531 17.11826 -0.000000541 -0.000000190 +17532 17.11923 -0.000000540 -0.000000189 +17533 17.12019 -0.000000540 -0.000000189 +17534 17.12116 -0.000000540 -0.000000189 +17535 17.12212 -0.000000540 -0.000000189 +17536 17.12309 -0.000000540 -0.000000189 +17537 17.12405 -0.000000540 -0.000000189 +17538 17.12502 -0.000000539 -0.000000189 +17539 17.12598 -0.000000539 -0.000000189 +17540 17.12695 -0.000000539 -0.000000189 +17541 17.12791 -0.000000539 -0.000000189 +17542 17.12888 -0.000000539 -0.000000189 +17543 17.12984 -0.000000538 -0.000000189 +17544 17.13081 -0.000000538 -0.000000189 +17545 17.13177 -0.000000538 -0.000000188 +17546 17.13274 -0.000000538 -0.000000188 +17547 17.13370 -0.000000538 -0.000000188 +17548 17.13467 -0.000000538 -0.000000188 +17549 17.13563 -0.000000537 -0.000000188 +17550 17.13660 -0.000000537 -0.000000188 +17551 17.13756 -0.000000537 -0.000000188 +17552 17.13853 -0.000000537 -0.000000188 +17553 17.13949 -0.000000537 -0.000000188 +17554 17.14046 -0.000000536 -0.000000188 +17555 17.14142 -0.000000536 -0.000000188 +17556 17.14239 -0.000000536 -0.000000188 +17557 17.14335 -0.000000536 -0.000000188 +17558 17.14432 -0.000000536 -0.000000187 +17559 17.14528 -0.000000536 -0.000000187 +17560 17.14625 -0.000000535 -0.000000187 +17561 17.14721 -0.000000535 -0.000000187 +17562 17.14818 -0.000000535 -0.000000187 +17563 17.14914 -0.000000535 -0.000000187 +17564 17.15011 -0.000000535 -0.000000187 +17565 17.15107 -0.000000534 -0.000000187 +17566 17.15204 -0.000000534 -0.000000187 +17567 17.15300 -0.000000534 -0.000000187 +17568 17.15397 -0.000000534 -0.000000187 +17569 17.15493 -0.000000534 -0.000000187 +17570 17.15590 -0.000000534 -0.000000187 +17571 17.15686 -0.000000533 -0.000000187 +17572 17.15783 -0.000000533 -0.000000186 +17573 17.15879 -0.000000533 -0.000000186 +17574 17.15976 -0.000000533 -0.000000186 +17575 17.16072 -0.000000533 -0.000000186 +17576 17.16169 -0.000000533 -0.000000186 +17577 17.16265 -0.000000532 -0.000000186 +17578 17.16362 -0.000000532 -0.000000186 +17579 17.16458 -0.000000532 -0.000000186 +17580 17.16555 -0.000000532 -0.000000186 +17581 17.16651 -0.000000532 -0.000000186 +17582 17.16748 -0.000000531 -0.000000186 +17583 17.16844 -0.000000531 -0.000000186 +17584 17.16941 -0.000000531 -0.000000186 +17585 17.17037 -0.000000531 -0.000000186 +17586 17.17134 -0.000000531 -0.000000185 +17587 17.17230 -0.000000531 -0.000000185 +17588 17.17327 -0.000000530 -0.000000185 +17589 17.17423 -0.000000530 -0.000000185 +17590 17.17520 -0.000000530 -0.000000185 +17591 17.17616 -0.000000530 -0.000000185 +17592 17.17713 -0.000000530 -0.000000185 +17593 17.17809 -0.000000529 -0.000000185 +17594 17.17906 -0.000000529 -0.000000185 +17595 17.18002 -0.000000529 -0.000000185 +17596 17.18099 -0.000000529 -0.000000185 +17597 17.18195 -0.000000529 -0.000000185 +17598 17.18292 -0.000000529 -0.000000185 +17599 17.18388 -0.000000528 -0.000000184 +17600 17.18485 -0.000000528 -0.000000184 +17601 17.18581 -0.000000528 -0.000000184 +17602 17.18678 -0.000000528 -0.000000184 +17603 17.18774 -0.000000528 -0.000000184 +17604 17.18871 -0.000000528 -0.000000184 +17605 17.18967 -0.000000527 -0.000000184 +17606 17.19064 -0.000000527 -0.000000184 +17607 17.19160 -0.000000527 -0.000000184 +17608 17.19257 -0.000000527 -0.000000184 +17609 17.19353 -0.000000527 -0.000000184 +17610 17.19450 -0.000000526 -0.000000184 +17611 17.19546 -0.000000526 -0.000000184 +17612 17.19643 -0.000000526 -0.000000184 +17613 17.19739 -0.000000526 -0.000000183 +17614 17.19836 -0.000000526 -0.000000183 +17615 17.19932 -0.000000526 -0.000000183 +17616 17.20029 -0.000000525 -0.000000183 +17617 17.20126 -0.000000525 -0.000000183 +17618 17.20222 -0.000000525 -0.000000183 +17619 17.20319 -0.000000525 -0.000000183 +17620 17.20415 -0.000000525 -0.000000183 +17621 17.20512 -0.000000524 -0.000000183 +17622 17.20608 -0.000000524 -0.000000183 +17623 17.20705 -0.000000524 -0.000000183 +17624 17.20801 -0.000000524 -0.000000183 +17625 17.20898 -0.000000524 -0.000000183 +17626 17.20994 -0.000000524 -0.000000183 +17627 17.21091 -0.000000523 -0.000000182 +17628 17.21187 -0.000000523 -0.000000182 +17629 17.21284 -0.000000523 -0.000000182 +17630 17.21380 -0.000000523 -0.000000182 +17631 17.21477 -0.000000523 -0.000000182 +17632 17.21573 -0.000000523 -0.000000182 +17633 17.21670 -0.000000522 -0.000000182 +17634 17.21766 -0.000000522 -0.000000182 +17635 17.21863 -0.000000522 -0.000000182 +17636 17.21959 -0.000000522 -0.000000182 +17637 17.22056 -0.000000522 -0.000000182 +17638 17.22152 -0.000000521 -0.000000182 +17639 17.22249 -0.000000521 -0.000000182 +17640 17.22345 -0.000000521 -0.000000182 +17641 17.22442 -0.000000521 -0.000000181 +17642 17.22538 -0.000000521 -0.000000181 +17643 17.22635 -0.000000521 -0.000000181 +17644 17.22731 -0.000000520 -0.000000181 +17645 17.22828 -0.000000520 -0.000000181 +17646 17.22924 -0.000000520 -0.000000181 +17647 17.23021 -0.000000520 -0.000000181 +17648 17.23117 -0.000000520 -0.000000181 +17649 17.23214 -0.000000520 -0.000000181 +17650 17.23310 -0.000000519 -0.000000181 +17651 17.23407 -0.000000519 -0.000000181 +17652 17.23503 -0.000000519 -0.000000181 +17653 17.23600 -0.000000519 -0.000000181 +17654 17.23696 -0.000000519 -0.000000181 +17655 17.23793 -0.000000519 -0.000000180 +17656 17.23889 -0.000000518 -0.000000180 +17657 17.23986 -0.000000518 -0.000000180 +17658 17.24082 -0.000000518 -0.000000180 +17659 17.24179 -0.000000518 -0.000000180 +17660 17.24275 -0.000000518 -0.000000180 +17661 17.24372 -0.000000517 -0.000000180 +17662 17.24468 -0.000000517 -0.000000180 +17663 17.24565 -0.000000517 -0.000000180 +17664 17.24661 -0.000000517 -0.000000180 +17665 17.24758 -0.000000517 -0.000000180 +17666 17.24854 -0.000000517 -0.000000180 +17667 17.24951 -0.000000516 -0.000000180 +17668 17.25047 -0.000000516 -0.000000180 +17669 17.25144 -0.000000516 -0.000000179 +17670 17.25240 -0.000000516 -0.000000179 +17671 17.25337 -0.000000516 -0.000000179 +17672 17.25433 -0.000000516 -0.000000179 +17673 17.25530 -0.000000515 -0.000000179 +17674 17.25626 -0.000000515 -0.000000179 +17675 17.25723 -0.000000515 -0.000000179 +17676 17.25819 -0.000000515 -0.000000179 +17677 17.25916 -0.000000515 -0.000000179 +17678 17.26012 -0.000000515 -0.000000179 +17679 17.26109 -0.000000514 -0.000000179 +17680 17.26205 -0.000000514 -0.000000179 +17681 17.26302 -0.000000514 -0.000000179 +17682 17.26398 -0.000000514 -0.000000179 +17683 17.26495 -0.000000514 -0.000000179 +17684 17.26591 -0.000000514 -0.000000178 +17685 17.26688 -0.000000513 -0.000000178 +17686 17.26784 -0.000000513 -0.000000178 +17687 17.26881 -0.000000513 -0.000000178 +17688 17.26977 -0.000000513 -0.000000178 +17689 17.27074 -0.000000513 -0.000000178 +17690 17.27170 -0.000000512 -0.000000178 +17691 17.27267 -0.000000512 -0.000000178 +17692 17.27363 -0.000000512 -0.000000178 +17693 17.27460 -0.000000512 -0.000000178 +17694 17.27556 -0.000000512 -0.000000178 +17695 17.27653 -0.000000512 -0.000000178 +17696 17.27749 -0.000000511 -0.000000178 +17697 17.27846 -0.000000511 -0.000000178 +17698 17.27942 -0.000000511 -0.000000177 +17699 17.28039 -0.000000511 -0.000000177 +17700 17.28135 -0.000000511 -0.000000177 +17701 17.28232 -0.000000511 -0.000000177 +17702 17.28328 -0.000000510 -0.000000177 +17703 17.28425 -0.000000510 -0.000000177 +17704 17.28521 -0.000000510 -0.000000177 +17705 17.28618 -0.000000510 -0.000000177 +17706 17.28714 -0.000000510 -0.000000177 +17707 17.28811 -0.000000510 -0.000000177 +17708 17.28907 -0.000000509 -0.000000177 +17709 17.29004 -0.000000509 -0.000000177 +17710 17.29100 -0.000000509 -0.000000177 +17711 17.29197 -0.000000509 -0.000000177 +17712 17.29293 -0.000000509 -0.000000177 +17713 17.29390 -0.000000509 -0.000000176 +17714 17.29486 -0.000000508 -0.000000176 +17715 17.29583 -0.000000508 -0.000000176 +17716 17.29679 -0.000000508 -0.000000176 +17717 17.29776 -0.000000508 -0.000000176 +17718 17.29872 -0.000000508 -0.000000176 +17719 17.29969 -0.000000508 -0.000000176 +17720 17.30066 -0.000000507 -0.000000176 +17721 17.30162 -0.000000507 -0.000000176 +17722 17.30259 -0.000000507 -0.000000176 +17723 17.30355 -0.000000507 -0.000000176 +17724 17.30452 -0.000000507 -0.000000176 +17725 17.30548 -0.000000507 -0.000000176 +17726 17.30645 -0.000000506 -0.000000176 +17727 17.30741 -0.000000506 -0.000000175 +17728 17.30838 -0.000000506 -0.000000175 +17729 17.30934 -0.000000506 -0.000000175 +17730 17.31031 -0.000000506 -0.000000175 +17731 17.31127 -0.000000505 -0.000000175 +17732 17.31224 -0.000000505 -0.000000175 +17733 17.31320 -0.000000505 -0.000000175 +17734 17.31417 -0.000000505 -0.000000175 +17735 17.31513 -0.000000505 -0.000000175 +17736 17.31610 -0.000000505 -0.000000175 +17737 17.31706 -0.000000504 -0.000000175 +17738 17.31803 -0.000000504 -0.000000175 +17739 17.31899 -0.000000504 -0.000000175 +17740 17.31996 -0.000000504 -0.000000175 +17741 17.32092 -0.000000504 -0.000000175 +17742 17.32189 -0.000000504 -0.000000174 +17743 17.32285 -0.000000503 -0.000000174 +17744 17.32382 -0.000000503 -0.000000174 +17745 17.32478 -0.000000503 -0.000000174 +17746 17.32575 -0.000000503 -0.000000174 +17747 17.32671 -0.000000503 -0.000000174 +17748 17.32768 -0.000000503 -0.000000174 +17749 17.32864 -0.000000502 -0.000000174 +17750 17.32961 -0.000000502 -0.000000174 +17751 17.33057 -0.000000502 -0.000000174 +17752 17.33154 -0.000000502 -0.000000174 +17753 17.33250 -0.000000502 -0.000000174 +17754 17.33347 -0.000000502 -0.000000174 +17755 17.33443 -0.000000501 -0.000000174 +17756 17.33540 -0.000000501 -0.000000173 +17757 17.33636 -0.000000501 -0.000000173 +17758 17.33733 -0.000000501 -0.000000173 +17759 17.33829 -0.000000501 -0.000000173 +17760 17.33926 -0.000000501 -0.000000173 +17761 17.34022 -0.000000500 -0.000000173 +17762 17.34119 -0.000000500 -0.000000173 +17763 17.34215 -0.000000500 -0.000000173 +17764 17.34312 -0.000000500 -0.000000173 +17765 17.34408 -0.000000500 -0.000000173 +17766 17.34505 -0.000000500 -0.000000173 +17767 17.34601 -0.000000499 -0.000000173 +17768 17.34698 -0.000000499 -0.000000173 +17769 17.34794 -0.000000499 -0.000000173 +17770 17.34891 -0.000000499 -0.000000173 +17771 17.34987 -0.000000499 -0.000000172 +17772 17.35084 -0.000000499 -0.000000172 +17773 17.35180 -0.000000498 -0.000000172 +17774 17.35277 -0.000000498 -0.000000172 +17775 17.35373 -0.000000498 -0.000000172 +17776 17.35470 -0.000000498 -0.000000172 +17777 17.35566 -0.000000498 -0.000000172 +17778 17.35663 -0.000000498 -0.000000172 +17779 17.35759 -0.000000497 -0.000000172 +17780 17.35856 -0.000000497 -0.000000172 +17781 17.35952 -0.000000497 -0.000000172 +17782 17.36049 -0.000000497 -0.000000172 +17783 17.36145 -0.000000497 -0.000000172 +17784 17.36242 -0.000000497 -0.000000172 +17785 17.36338 -0.000000496 -0.000000172 +17786 17.36435 -0.000000496 -0.000000171 +17787 17.36531 -0.000000496 -0.000000171 +17788 17.36628 -0.000000496 -0.000000171 +17789 17.36724 -0.000000496 -0.000000171 +17790 17.36821 -0.000000496 -0.000000171 +17791 17.36917 -0.000000495 -0.000000171 +17792 17.37014 -0.000000495 -0.000000171 +17793 17.37110 -0.000000495 -0.000000171 +17794 17.37207 -0.000000495 -0.000000171 +17795 17.37303 -0.000000495 -0.000000171 +17796 17.37400 -0.000000495 -0.000000171 +17797 17.37496 -0.000000494 -0.000000171 +17798 17.37593 -0.000000494 -0.000000171 +17799 17.37689 -0.000000494 -0.000000171 +17800 17.37786 -0.000000494 -0.000000171 +17801 17.37882 -0.000000494 -0.000000170 +17802 17.37979 -0.000000494 -0.000000170 +17803 17.38075 -0.000000493 -0.000000170 +17804 17.38172 -0.000000493 -0.000000170 +17805 17.38268 -0.000000493 -0.000000170 +17806 17.38365 -0.000000493 -0.000000170 +17807 17.38461 -0.000000493 -0.000000170 +17808 17.38558 -0.000000493 -0.000000170 +17809 17.38654 -0.000000492 -0.000000170 +17810 17.38751 -0.000000492 -0.000000170 +17811 17.38847 -0.000000492 -0.000000170 +17812 17.38944 -0.000000492 -0.000000170 +17813 17.39040 -0.000000492 -0.000000170 +17814 17.39137 -0.000000492 -0.000000170 +17815 17.39233 -0.000000492 -0.000000170 +17816 17.39330 -0.000000491 -0.000000169 +17817 17.39426 -0.000000491 -0.000000169 +17818 17.39523 -0.000000491 -0.000000169 +17819 17.39619 -0.000000491 -0.000000169 +17820 17.39716 -0.000000491 -0.000000169 +17821 17.39812 -0.000000491 -0.000000169 +17822 17.39909 -0.000000490 -0.000000169 +17823 17.40006 -0.000000490 -0.000000169 +17824 17.40102 -0.000000490 -0.000000169 +17825 17.40199 -0.000000490 -0.000000169 +17826 17.40295 -0.000000490 -0.000000169 +17827 17.40392 -0.000000490 -0.000000169 +17828 17.40488 -0.000000489 -0.000000169 +17829 17.40585 -0.000000489 -0.000000169 +17830 17.40681 -0.000000489 -0.000000169 +17831 17.40778 -0.000000489 -0.000000169 +17832 17.40874 -0.000000489 -0.000000168 +17833 17.40971 -0.000000489 -0.000000168 +17834 17.41067 -0.000000488 -0.000000168 +17835 17.41164 -0.000000488 -0.000000168 +17836 17.41260 -0.000000488 -0.000000168 +17837 17.41357 -0.000000488 -0.000000168 +17838 17.41453 -0.000000488 -0.000000168 +17839 17.41550 -0.000000488 -0.000000168 +17840 17.41646 -0.000000487 -0.000000168 +17841 17.41743 -0.000000487 -0.000000168 +17842 17.41839 -0.000000487 -0.000000168 +17843 17.41936 -0.000000487 -0.000000168 +17844 17.42032 -0.000000487 -0.000000168 +17845 17.42129 -0.000000487 -0.000000168 +17846 17.42225 -0.000000486 -0.000000168 +17847 17.42322 -0.000000486 -0.000000167 +17848 17.42418 -0.000000486 -0.000000167 +17849 17.42515 -0.000000486 -0.000000167 +17850 17.42611 -0.000000486 -0.000000167 +17851 17.42708 -0.000000486 -0.000000167 +17852 17.42804 -0.000000486 -0.000000167 +17853 17.42901 -0.000000485 -0.000000167 +17854 17.42997 -0.000000485 -0.000000167 +17855 17.43094 -0.000000485 -0.000000167 +17856 17.43190 -0.000000485 -0.000000167 +17857 17.43287 -0.000000485 -0.000000167 +17858 17.43383 -0.000000485 -0.000000167 +17859 17.43480 -0.000000484 -0.000000167 +17860 17.43576 -0.000000484 -0.000000167 +17861 17.43673 -0.000000484 -0.000000167 +17862 17.43769 -0.000000484 -0.000000166 +17863 17.43866 -0.000000484 -0.000000166 +17864 17.43962 -0.000000484 -0.000000166 +17865 17.44059 -0.000000483 -0.000000166 +17866 17.44155 -0.000000483 -0.000000166 +17867 17.44252 -0.000000483 -0.000000166 +17868 17.44348 -0.000000483 -0.000000166 +17869 17.44445 -0.000000483 -0.000000166 +17870 17.44541 -0.000000483 -0.000000166 +17871 17.44638 -0.000000482 -0.000000166 +17872 17.44734 -0.000000482 -0.000000166 +17873 17.44831 -0.000000482 -0.000000166 +17874 17.44927 -0.000000482 -0.000000166 +17875 17.45024 -0.000000482 -0.000000166 +17876 17.45120 -0.000000482 -0.000000166 +17877 17.45217 -0.000000481 -0.000000166 +17878 17.45313 -0.000000481 -0.000000165 +17879 17.45410 -0.000000481 -0.000000165 +17880 17.45506 -0.000000481 -0.000000165 +17881 17.45603 -0.000000481 -0.000000165 +17882 17.45699 -0.000000481 -0.000000165 +17883 17.45796 -0.000000481 -0.000000165 +17884 17.45892 -0.000000480 -0.000000165 +17885 17.45989 -0.000000480 -0.000000165 +17886 17.46085 -0.000000480 -0.000000165 +17887 17.46182 -0.000000480 -0.000000165 +17888 17.46278 -0.000000480 -0.000000165 +17889 17.46375 -0.000000480 -0.000000165 +17890 17.46471 -0.000000479 -0.000000165 +17891 17.46568 -0.000000479 -0.000000165 +17892 17.46664 -0.000000479 -0.000000165 +17893 17.46761 -0.000000479 -0.000000165 +17894 17.46857 -0.000000479 -0.000000164 +17895 17.46954 -0.000000479 -0.000000164 +17896 17.47050 -0.000000478 -0.000000164 +17897 17.47147 -0.000000478 -0.000000164 +17898 17.47243 -0.000000478 -0.000000164 +17899 17.47340 -0.000000478 -0.000000164 +17900 17.47436 -0.000000478 -0.000000164 +17901 17.47533 -0.000000478 -0.000000164 +17902 17.47629 -0.000000478 -0.000000164 +17903 17.47726 -0.000000477 -0.000000164 +17904 17.47822 -0.000000477 -0.000000164 +17905 17.47919 -0.000000477 -0.000000164 +17906 17.48015 -0.000000477 -0.000000164 +17907 17.48112 -0.000000477 -0.000000164 +17908 17.48208 -0.000000477 -0.000000164 +17909 17.48305 -0.000000476 -0.000000163 +17910 17.48401 -0.000000476 -0.000000163 +17911 17.48498 -0.000000476 -0.000000163 +17912 17.48594 -0.000000476 -0.000000163 +17913 17.48691 -0.000000476 -0.000000163 +17914 17.48787 -0.000000476 -0.000000163 +17915 17.48884 -0.000000475 -0.000000163 +17916 17.48980 -0.000000475 -0.000000163 +17917 17.49077 -0.000000475 -0.000000163 +17918 17.49173 -0.000000475 -0.000000163 +17919 17.49270 -0.000000475 -0.000000163 +17920 17.49366 -0.000000475 -0.000000163 +17921 17.49463 -0.000000475 -0.000000163 +17922 17.49559 -0.000000474 -0.000000163 +17923 17.49656 -0.000000474 -0.000000163 +17924 17.49752 -0.000000474 -0.000000163 +17925 17.49849 -0.000000474 -0.000000162 +17926 17.49945 -0.000000474 -0.000000162 +17927 17.50042 -0.000000474 -0.000000162 +17928 17.50139 -0.000000473 -0.000000162 +17929 17.50235 -0.000000473 -0.000000162 +17930 17.50332 -0.000000473 -0.000000162 +17931 17.50428 -0.000000473 -0.000000162 +17932 17.50525 -0.000000473 -0.000000162 +17933 17.50621 -0.000000473 -0.000000162 +17934 17.50718 -0.000000472 -0.000000162 +17935 17.50814 -0.000000472 -0.000000162 +17936 17.50911 -0.000000472 -0.000000162 +17937 17.51007 -0.000000472 -0.000000162 +17938 17.51104 -0.000000472 -0.000000162 +17939 17.51200 -0.000000472 -0.000000162 +17940 17.51297 -0.000000472 -0.000000162 +17941 17.51393 -0.000000471 -0.000000161 +17942 17.51490 -0.000000471 -0.000000161 +17943 17.51586 -0.000000471 -0.000000161 +17944 17.51683 -0.000000471 -0.000000161 +17945 17.51779 -0.000000471 -0.000000161 +17946 17.51876 -0.000000471 -0.000000161 +17947 17.51972 -0.000000470 -0.000000161 +17948 17.52069 -0.000000470 -0.000000161 +17949 17.52165 -0.000000470 -0.000000161 +17950 17.52262 -0.000000470 -0.000000161 +17951 17.52358 -0.000000470 -0.000000161 +17952 17.52455 -0.000000470 -0.000000161 +17953 17.52551 -0.000000470 -0.000000161 +17954 17.52648 -0.000000469 -0.000000161 +17955 17.52744 -0.000000469 -0.000000161 +17956 17.52841 -0.000000469 -0.000000161 +17957 17.52937 -0.000000469 -0.000000160 +17958 17.53034 -0.000000469 -0.000000160 +17959 17.53130 -0.000000469 -0.000000160 +17960 17.53227 -0.000000468 -0.000000160 +17961 17.53323 -0.000000468 -0.000000160 +17962 17.53420 -0.000000468 -0.000000160 +17963 17.53516 -0.000000468 -0.000000160 +17964 17.53613 -0.000000468 -0.000000160 +17965 17.53709 -0.000000468 -0.000000160 +17966 17.53806 -0.000000468 -0.000000160 +17967 17.53902 -0.000000467 -0.000000160 +17968 17.53999 -0.000000467 -0.000000160 +17969 17.54095 -0.000000467 -0.000000160 +17970 17.54192 -0.000000467 -0.000000160 +17971 17.54288 -0.000000467 -0.000000160 +17972 17.54385 -0.000000467 -0.000000160 +17973 17.54481 -0.000000466 -0.000000160 +17974 17.54578 -0.000000466 -0.000000159 +17975 17.54674 -0.000000466 -0.000000159 +17976 17.54771 -0.000000466 -0.000000159 +17977 17.54867 -0.000000466 -0.000000159 +17978 17.54964 -0.000000466 -0.000000159 +17979 17.55060 -0.000000466 -0.000000159 +17980 17.55157 -0.000000465 -0.000000159 +17981 17.55253 -0.000000465 -0.000000159 +17982 17.55350 -0.000000465 -0.000000159 +17983 17.55446 -0.000000465 -0.000000159 +17984 17.55543 -0.000000465 -0.000000159 +17985 17.55639 -0.000000465 -0.000000159 +17986 17.55736 -0.000000464 -0.000000159 +17987 17.55832 -0.000000464 -0.000000159 +17988 17.55929 -0.000000464 -0.000000159 +17989 17.56025 -0.000000464 -0.000000159 +17990 17.56122 -0.000000464 -0.000000158 +17991 17.56218 -0.000000464 -0.000000158 +17992 17.56315 -0.000000464 -0.000000158 +17993 17.56411 -0.000000463 -0.000000158 +17994 17.56508 -0.000000463 -0.000000158 +17995 17.56604 -0.000000463 -0.000000158 +17996 17.56701 -0.000000463 -0.000000158 +17997 17.56797 -0.000000463 -0.000000158 +17998 17.56894 -0.000000463 -0.000000158 +17999 17.56990 -0.000000462 -0.000000158 +18000 17.57087 -0.000000462 -0.000000158 +18001 17.57183 -0.000000462 -0.000000158 +18002 17.57280 -0.000000462 -0.000000158 +18003 17.57376 -0.000000462 -0.000000158 +18004 17.57473 -0.000000462 -0.000000158 +18005 17.57569 -0.000000462 -0.000000158 +18006 17.57666 -0.000000461 -0.000000157 +18007 17.57762 -0.000000461 -0.000000157 +18008 17.57859 -0.000000461 -0.000000157 +18009 17.57955 -0.000000461 -0.000000157 +18010 17.58052 -0.000000461 -0.000000157 +18011 17.58148 -0.000000461 -0.000000157 +18012 17.58245 -0.000000460 -0.000000157 +18013 17.58341 -0.000000460 -0.000000157 +18014 17.58438 -0.000000460 -0.000000157 +18015 17.58534 -0.000000460 -0.000000157 +18016 17.58631 -0.000000460 -0.000000157 +18017 17.58727 -0.000000460 -0.000000157 +18018 17.58824 -0.000000460 -0.000000157 +18019 17.58920 -0.000000459 -0.000000157 +18020 17.59017 -0.000000459 -0.000000157 +18021 17.59113 -0.000000459 -0.000000157 +18022 17.59210 -0.000000459 -0.000000157 +18023 17.59306 -0.000000459 -0.000000156 +18024 17.59403 -0.000000459 -0.000000156 +18025 17.59499 -0.000000459 -0.000000156 +18026 17.59596 -0.000000458 -0.000000156 +18027 17.59692 -0.000000458 -0.000000156 +18028 17.59789 -0.000000458 -0.000000156 +18029 17.59885 -0.000000458 -0.000000156 +18030 17.59982 -0.000000458 -0.000000156 +18031 17.60079 -0.000000458 -0.000000156 +18032 17.60175 -0.000000457 -0.000000156 +18033 17.60272 -0.000000457 -0.000000156 +18034 17.60368 -0.000000457 -0.000000156 +18035 17.60465 -0.000000457 -0.000000156 +18036 17.60561 -0.000000457 -0.000000156 +18037 17.60658 -0.000000457 -0.000000156 +18038 17.60754 -0.000000457 -0.000000156 +18039 17.60851 -0.000000456 -0.000000156 +18040 17.60947 -0.000000456 -0.000000155 +18041 17.61044 -0.000000456 -0.000000155 +18042 17.61140 -0.000000456 -0.000000155 +18043 17.61237 -0.000000456 -0.000000155 +18044 17.61333 -0.000000456 -0.000000155 +18045 17.61430 -0.000000456 -0.000000155 +18046 17.61526 -0.000000455 -0.000000155 +18047 17.61623 -0.000000455 -0.000000155 +18048 17.61719 -0.000000455 -0.000000155 +18049 17.61816 -0.000000455 -0.000000155 +18050 17.61912 -0.000000455 -0.000000155 +18051 17.62009 -0.000000455 -0.000000155 +18052 17.62105 -0.000000454 -0.000000155 +18053 17.62202 -0.000000454 -0.000000155 +18054 17.62298 -0.000000454 -0.000000155 +18055 17.62395 -0.000000454 -0.000000155 +18056 17.62491 -0.000000454 -0.000000155 +18057 17.62588 -0.000000454 -0.000000154 +18058 17.62684 -0.000000454 -0.000000154 +18059 17.62781 -0.000000453 -0.000000154 +18060 17.62877 -0.000000453 -0.000000154 +18061 17.62974 -0.000000453 -0.000000154 +18062 17.63070 -0.000000453 -0.000000154 +18063 17.63167 -0.000000453 -0.000000154 +18064 17.63263 -0.000000453 -0.000000154 +18065 17.63360 -0.000000453 -0.000000154 +18066 17.63456 -0.000000452 -0.000000154 +18067 17.63553 -0.000000452 -0.000000154 +18068 17.63649 -0.000000452 -0.000000154 +18069 17.63746 -0.000000452 -0.000000154 +18070 17.63842 -0.000000452 -0.000000154 +18071 17.63939 -0.000000452 -0.000000154 +18072 17.64035 -0.000000451 -0.000000154 +18073 17.64132 -0.000000451 -0.000000154 +18074 17.64228 -0.000000451 -0.000000153 +18075 17.64325 -0.000000451 -0.000000153 +18076 17.64421 -0.000000451 -0.000000153 +18077 17.64518 -0.000000451 -0.000000153 +18078 17.64614 -0.000000451 -0.000000153 +18079 17.64711 -0.000000450 -0.000000153 +18080 17.64807 -0.000000450 -0.000000153 +18081 17.64904 -0.000000450 -0.000000153 +18082 17.65000 -0.000000450 -0.000000153 +18083 17.65097 -0.000000450 -0.000000153 +18084 17.65193 -0.000000450 -0.000000153 +18085 17.65290 -0.000000450 -0.000000153 +18086 17.65386 -0.000000449 -0.000000153 +18087 17.65483 -0.000000449 -0.000000153 +18088 17.65579 -0.000000449 -0.000000153 +18089 17.65676 -0.000000449 -0.000000153 +18090 17.65772 -0.000000449 -0.000000153 +18091 17.65869 -0.000000449 -0.000000152 +18092 17.65965 -0.000000449 -0.000000152 +18093 17.66062 -0.000000448 -0.000000152 +18094 17.66158 -0.000000448 -0.000000152 +18095 17.66255 -0.000000448 -0.000000152 +18096 17.66351 -0.000000448 -0.000000152 +18097 17.66448 -0.000000448 -0.000000152 +18098 17.66544 -0.000000448 -0.000000152 +18099 17.66641 -0.000000447 -0.000000152 +18100 17.66737 -0.000000447 -0.000000152 +18101 17.66834 -0.000000447 -0.000000152 +18102 17.66930 -0.000000447 -0.000000152 +18103 17.67027 -0.000000447 -0.000000152 +18104 17.67123 -0.000000447 -0.000000152 +18105 17.67220 -0.000000447 -0.000000152 +18106 17.67316 -0.000000446 -0.000000152 +18107 17.67413 -0.000000446 -0.000000152 +18108 17.67509 -0.000000446 -0.000000151 +18109 17.67606 -0.000000446 -0.000000151 +18110 17.67702 -0.000000446 -0.000000151 +18111 17.67799 -0.000000446 -0.000000151 +18112 17.67895 -0.000000446 -0.000000151 +18113 17.67992 -0.000000445 -0.000000151 +18114 17.68088 -0.000000445 -0.000000151 +18115 17.68185 -0.000000445 -0.000000151 +18116 17.68281 -0.000000445 -0.000000151 +18117 17.68378 -0.000000445 -0.000000151 +18118 17.68474 -0.000000445 -0.000000151 +18119 17.68571 -0.000000445 -0.000000151 +18120 17.68667 -0.000000444 -0.000000151 +18121 17.68764 -0.000000444 -0.000000151 +18122 17.68860 -0.000000444 -0.000000151 +18123 17.68957 -0.000000444 -0.000000151 +18124 17.69053 -0.000000444 -0.000000151 +18125 17.69150 -0.000000444 -0.000000150 +18126 17.69246 -0.000000444 -0.000000150 +18127 17.69343 -0.000000443 -0.000000150 +18128 17.69439 -0.000000443 -0.000000150 +18129 17.69536 -0.000000443 -0.000000150 +18130 17.69632 -0.000000443 -0.000000150 +18131 17.69729 -0.000000443 -0.000000150 +18132 17.69825 -0.000000443 -0.000000150 +18133 17.69922 -0.000000443 -0.000000150 +18134 17.70019 -0.000000442 -0.000000150 +18135 17.70115 -0.000000442 -0.000000150 +18136 17.70212 -0.000000442 -0.000000150 +18137 17.70308 -0.000000442 -0.000000150 +18138 17.70405 -0.000000442 -0.000000150 +18139 17.70501 -0.000000442 -0.000000150 +18140 17.70598 -0.000000442 -0.000000150 +18141 17.70694 -0.000000441 -0.000000150 +18142 17.70791 -0.000000441 -0.000000150 +18143 17.70887 -0.000000441 -0.000000149 +18144 17.70984 -0.000000441 -0.000000149 +18145 17.71080 -0.000000441 -0.000000149 +18146 17.71177 -0.000000441 -0.000000149 +18147 17.71273 -0.000000441 -0.000000149 +18148 17.71370 -0.000000440 -0.000000149 +18149 17.71466 -0.000000440 -0.000000149 +18150 17.71563 -0.000000440 -0.000000149 +18151 17.71659 -0.000000440 -0.000000149 +18152 17.71756 -0.000000440 -0.000000149 +18153 17.71852 -0.000000440 -0.000000149 +18154 17.71949 -0.000000440 -0.000000149 +18155 17.72045 -0.000000439 -0.000000149 +18156 17.72142 -0.000000439 -0.000000149 +18157 17.72238 -0.000000439 -0.000000149 +18158 17.72335 -0.000000439 -0.000000149 +18159 17.72431 -0.000000439 -0.000000149 +18160 17.72528 -0.000000439 -0.000000148 +18161 17.72624 -0.000000439 -0.000000148 +18162 17.72721 -0.000000438 -0.000000148 +18163 17.72817 -0.000000438 -0.000000148 +18164 17.72914 -0.000000438 -0.000000148 +18165 17.73010 -0.000000438 -0.000000148 +18166 17.73107 -0.000000438 -0.000000148 +18167 17.73203 -0.000000438 -0.000000148 +18168 17.73300 -0.000000438 -0.000000148 +18169 17.73396 -0.000000437 -0.000000148 +18170 17.73493 -0.000000437 -0.000000148 +18171 17.73589 -0.000000437 -0.000000148 +18172 17.73686 -0.000000437 -0.000000148 +18173 17.73782 -0.000000437 -0.000000148 +18174 17.73879 -0.000000437 -0.000000148 +18175 17.73975 -0.000000437 -0.000000148 +18176 17.74072 -0.000000436 -0.000000148 +18177 17.74168 -0.000000436 -0.000000148 +18178 17.74265 -0.000000436 -0.000000147 +18179 17.74361 -0.000000436 -0.000000147 +18180 17.74458 -0.000000436 -0.000000147 +18181 17.74554 -0.000000436 -0.000000147 +18182 17.74651 -0.000000436 -0.000000147 +18183 17.74747 -0.000000435 -0.000000147 +18184 17.74844 -0.000000435 -0.000000147 +18185 17.74940 -0.000000435 -0.000000147 +18186 17.75037 -0.000000435 -0.000000147 +18187 17.75133 -0.000000435 -0.000000147 +18188 17.75230 -0.000000435 -0.000000147 +18189 17.75326 -0.000000435 -0.000000147 +18190 17.75423 -0.000000434 -0.000000147 +18191 17.75519 -0.000000434 -0.000000147 +18192 17.75616 -0.000000434 -0.000000147 +18193 17.75712 -0.000000434 -0.000000147 +18194 17.75809 -0.000000434 -0.000000147 +18195 17.75905 -0.000000434 -0.000000147 +18196 17.76002 -0.000000434 -0.000000146 +18197 17.76098 -0.000000433 -0.000000146 +18198 17.76195 -0.000000433 -0.000000146 +18199 17.76291 -0.000000433 -0.000000146 +18200 17.76388 -0.000000433 -0.000000146 +18201 17.76484 -0.000000433 -0.000000146 +18202 17.76581 -0.000000433 -0.000000146 +18203 17.76677 -0.000000433 -0.000000146 +18204 17.76774 -0.000000432 -0.000000146 +18205 17.76870 -0.000000432 -0.000000146 +18206 17.76967 -0.000000432 -0.000000146 +18207 17.77063 -0.000000432 -0.000000146 +18208 17.77160 -0.000000432 -0.000000146 +18209 17.77256 -0.000000432 -0.000000146 +18210 17.77353 -0.000000432 -0.000000146 +18211 17.77449 -0.000000431 -0.000000146 +18212 17.77546 -0.000000431 -0.000000146 +18213 17.77642 -0.000000431 -0.000000146 +18214 17.77739 -0.000000431 -0.000000145 +18215 17.77835 -0.000000431 -0.000000145 +18216 17.77932 -0.000000431 -0.000000145 +18217 17.78028 -0.000000431 -0.000000145 +18218 17.78125 -0.000000430 -0.000000145 +18219 17.78221 -0.000000430 -0.000000145 +18220 17.78318 -0.000000430 -0.000000145 +18221 17.78414 -0.000000430 -0.000000145 +18222 17.78511 -0.000000430 -0.000000145 +18223 17.78607 -0.000000430 -0.000000145 +18224 17.78704 -0.000000430 -0.000000145 +18225 17.78800 -0.000000429 -0.000000145 +18226 17.78897 -0.000000429 -0.000000145 +18227 17.78993 -0.000000429 -0.000000145 +18228 17.79090 -0.000000429 -0.000000145 +18229 17.79186 -0.000000429 -0.000000145 +18230 17.79283 -0.000000429 -0.000000145 +18231 17.79379 -0.000000429 -0.000000145 +18232 17.79476 -0.000000428 -0.000000144 +18233 17.79572 -0.000000428 -0.000000144 +18234 17.79669 -0.000000428 -0.000000144 +18235 17.79765 -0.000000428 -0.000000144 +18236 17.79862 -0.000000428 -0.000000144 +18237 17.79958 -0.000000428 -0.000000144 +18238 17.80055 -0.000000428 -0.000000144 +18239 17.80152 -0.000000428 -0.000000144 +18240 17.80248 -0.000000427 -0.000000144 +18241 17.80345 -0.000000427 -0.000000144 +18242 17.80441 -0.000000427 -0.000000144 +18243 17.80538 -0.000000427 -0.000000144 +18244 17.80634 -0.000000427 -0.000000144 +18245 17.80731 -0.000000427 -0.000000144 +18246 17.80827 -0.000000427 -0.000000144 +18247 17.80924 -0.000000426 -0.000000144 +18248 17.81020 -0.000000426 -0.000000144 +18249 17.81117 -0.000000426 -0.000000144 +18250 17.81213 -0.000000426 -0.000000143 +18251 17.81310 -0.000000426 -0.000000143 +18252 17.81406 -0.000000426 -0.000000143 +18253 17.81503 -0.000000426 -0.000000143 +18254 17.81599 -0.000000425 -0.000000143 +18255 17.81696 -0.000000425 -0.000000143 +18256 17.81792 -0.000000425 -0.000000143 +18257 17.81889 -0.000000425 -0.000000143 +18258 17.81985 -0.000000425 -0.000000143 +18259 17.82082 -0.000000425 -0.000000143 +18260 17.82178 -0.000000425 -0.000000143 +18261 17.82275 -0.000000424 -0.000000143 +18262 17.82371 -0.000000424 -0.000000143 +18263 17.82468 -0.000000424 -0.000000143 +18264 17.82564 -0.000000424 -0.000000143 +18265 17.82661 -0.000000424 -0.000000143 +18266 17.82757 -0.000000424 -0.000000143 +18267 17.82854 -0.000000424 -0.000000143 +18268 17.82950 -0.000000423 -0.000000143 +18269 17.83047 -0.000000423 -0.000000142 +18270 17.83143 -0.000000423 -0.000000142 +18271 17.83240 -0.000000423 -0.000000142 +18272 17.83336 -0.000000423 -0.000000142 +18273 17.83433 -0.000000423 -0.000000142 +18274 17.83529 -0.000000423 -0.000000142 +18275 17.83626 -0.000000423 -0.000000142 +18276 17.83722 -0.000000422 -0.000000142 +18277 17.83819 -0.000000422 -0.000000142 +18278 17.83915 -0.000000422 -0.000000142 +18279 17.84012 -0.000000422 -0.000000142 +18280 17.84108 -0.000000422 -0.000000142 +18281 17.84205 -0.000000422 -0.000000142 +18282 17.84301 -0.000000422 -0.000000142 +18283 17.84398 -0.000000421 -0.000000142 +18284 17.84494 -0.000000421 -0.000000142 +18285 17.84591 -0.000000421 -0.000000142 +18286 17.84687 -0.000000421 -0.000000142 +18287 17.84784 -0.000000421 -0.000000141 +18288 17.84880 -0.000000421 -0.000000141 +18289 17.84977 -0.000000421 -0.000000141 +18290 17.85073 -0.000000420 -0.000000141 +18291 17.85170 -0.000000420 -0.000000141 +18292 17.85266 -0.000000420 -0.000000141 +18293 17.85363 -0.000000420 -0.000000141 +18294 17.85459 -0.000000420 -0.000000141 +18295 17.85556 -0.000000420 -0.000000141 +18296 17.85652 -0.000000420 -0.000000141 +18297 17.85749 -0.000000420 -0.000000141 +18298 17.85845 -0.000000419 -0.000000141 +18299 17.85942 -0.000000419 -0.000000141 +18300 17.86038 -0.000000419 -0.000000141 +18301 17.86135 -0.000000419 -0.000000141 +18302 17.86231 -0.000000419 -0.000000141 +18303 17.86328 -0.000000419 -0.000000141 +18304 17.86424 -0.000000419 -0.000000141 +18305 17.86521 -0.000000418 -0.000000141 +18306 17.86617 -0.000000418 -0.000000140 +18307 17.86714 -0.000000418 -0.000000140 +18308 17.86810 -0.000000418 -0.000000140 +18309 17.86907 -0.000000418 -0.000000140 +18310 17.87003 -0.000000418 -0.000000140 +18311 17.87100 -0.000000418 -0.000000140 +18312 17.87196 -0.000000417 -0.000000140 +18313 17.87293 -0.000000417 -0.000000140 +18314 17.87389 -0.000000417 -0.000000140 +18315 17.87486 -0.000000417 -0.000000140 +18316 17.87582 -0.000000417 -0.000000140 +18317 17.87679 -0.000000417 -0.000000140 +18318 17.87775 -0.000000417 -0.000000140 +18319 17.87872 -0.000000417 -0.000000140 +18320 17.87968 -0.000000416 -0.000000140 +18321 17.88065 -0.000000416 -0.000000140 +18322 17.88161 -0.000000416 -0.000000140 +18323 17.88258 -0.000000416 -0.000000140 +18324 17.88354 -0.000000416 -0.000000140 +18325 17.88451 -0.000000416 -0.000000139 +18326 17.88547 -0.000000416 -0.000000139 +18327 17.88644 -0.000000415 -0.000000139 +18328 17.88740 -0.000000415 -0.000000139 +18329 17.88837 -0.000000415 -0.000000139 +18330 17.88933 -0.000000415 -0.000000139 +18331 17.89030 -0.000000415 -0.000000139 +18332 17.89126 -0.000000415 -0.000000139 +18333 17.89223 -0.000000415 -0.000000139 +18334 17.89319 -0.000000415 -0.000000139 +18335 17.89416 -0.000000414 -0.000000139 +18336 17.89512 -0.000000414 -0.000000139 +18337 17.89609 -0.000000414 -0.000000139 +18338 17.89705 -0.000000414 -0.000000139 +18339 17.89802 -0.000000414 -0.000000139 +18340 17.89898 -0.000000414 -0.000000139 +18341 17.89995 -0.000000414 -0.000000139 +18342 17.90092 -0.000000413 -0.000000139 +18343 17.90188 -0.000000413 -0.000000139 +18344 17.90285 -0.000000413 -0.000000138 +18345 17.90381 -0.000000413 -0.000000138 +18346 17.90478 -0.000000413 -0.000000138 +18347 17.90574 -0.000000413 -0.000000138 +18348 17.90671 -0.000000413 -0.000000138 +18349 17.90767 -0.000000413 -0.000000138 +18350 17.90864 -0.000000412 -0.000000138 +18351 17.90960 -0.000000412 -0.000000138 +18352 17.91057 -0.000000412 -0.000000138 +18353 17.91153 -0.000000412 -0.000000138 +18354 17.91250 -0.000000412 -0.000000138 +18355 17.91346 -0.000000412 -0.000000138 +18356 17.91443 -0.000000412 -0.000000138 +18357 17.91539 -0.000000411 -0.000000138 +18358 17.91636 -0.000000411 -0.000000138 +18359 17.91732 -0.000000411 -0.000000138 +18360 17.91829 -0.000000411 -0.000000138 +18361 17.91925 -0.000000411 -0.000000138 +18362 17.92022 -0.000000411 -0.000000138 +18363 17.92118 -0.000000411 -0.000000137 +18364 17.92215 -0.000000411 -0.000000137 +18365 17.92311 -0.000000410 -0.000000137 +18366 17.92408 -0.000000410 -0.000000137 +18367 17.92504 -0.000000410 -0.000000137 +18368 17.92601 -0.000000410 -0.000000137 +18369 17.92697 -0.000000410 -0.000000137 +18370 17.92794 -0.000000410 -0.000000137 +18371 17.92890 -0.000000410 -0.000000137 +18372 17.92987 -0.000000409 -0.000000137 +18373 17.93083 -0.000000409 -0.000000137 +18374 17.93180 -0.000000409 -0.000000137 +18375 17.93276 -0.000000409 -0.000000137 +18376 17.93373 -0.000000409 -0.000000137 +18377 17.93469 -0.000000409 -0.000000137 +18378 17.93566 -0.000000409 -0.000000137 +18379 17.93662 -0.000000409 -0.000000137 +18380 17.93759 -0.000000408 -0.000000137 +18381 17.93855 -0.000000408 -0.000000137 +18382 17.93952 -0.000000408 -0.000000137 +18383 17.94048 -0.000000408 -0.000000136 +18384 17.94145 -0.000000408 -0.000000136 +18385 17.94241 -0.000000408 -0.000000136 +18386 17.94338 -0.000000408 -0.000000136 +18387 17.94434 -0.000000407 -0.000000136 +18388 17.94531 -0.000000407 -0.000000136 +18389 17.94627 -0.000000407 -0.000000136 +18390 17.94724 -0.000000407 -0.000000136 +18391 17.94820 -0.000000407 -0.000000136 +18392 17.94917 -0.000000407 -0.000000136 +18393 17.95013 -0.000000407 -0.000000136 +18394 17.95110 -0.000000407 -0.000000136 +18395 17.95206 -0.000000406 -0.000000136 +18396 17.95303 -0.000000406 -0.000000136 +18397 17.95399 -0.000000406 -0.000000136 +18398 17.95496 -0.000000406 -0.000000136 +18399 17.95592 -0.000000406 -0.000000136 +18400 17.95689 -0.000000406 -0.000000136 +18401 17.95785 -0.000000406 -0.000000136 +18402 17.95882 -0.000000406 -0.000000135 +18403 17.95978 -0.000000405 -0.000000135 +18404 17.96075 -0.000000405 -0.000000135 +18405 17.96171 -0.000000405 -0.000000135 +18406 17.96268 -0.000000405 -0.000000135 +18407 17.96364 -0.000000405 -0.000000135 +18408 17.96461 -0.000000405 -0.000000135 +18409 17.96557 -0.000000405 -0.000000135 +18410 17.96654 -0.000000404 -0.000000135 +18411 17.96750 -0.000000404 -0.000000135 +18412 17.96847 -0.000000404 -0.000000135 +18413 17.96943 -0.000000404 -0.000000135 +18414 17.97040 -0.000000404 -0.000000135 +18415 17.97136 -0.000000404 -0.000000135 +18416 17.97233 -0.000000404 -0.000000135 +18417 17.97329 -0.000000404 -0.000000135 +18418 17.97426 -0.000000403 -0.000000135 +18419 17.97522 -0.000000403 -0.000000135 +18420 17.97619 -0.000000403 -0.000000135 +18421 17.97715 -0.000000403 -0.000000135 +18422 17.97812 -0.000000403 -0.000000134 +18423 17.97908 -0.000000403 -0.000000134 +18424 17.98005 -0.000000403 -0.000000134 +18425 17.98101 -0.000000403 -0.000000134 +18426 17.98198 -0.000000402 -0.000000134 +18427 17.98294 -0.000000402 -0.000000134 +18428 17.98391 -0.000000402 -0.000000134 +18429 17.98487 -0.000000402 -0.000000134 +18430 17.98584 -0.000000402 -0.000000134 +18431 17.98680 -0.000000402 -0.000000134 +18432 17.98777 -0.000000402 -0.000000134 +18433 17.98873 -0.000000401 -0.000000134 +18434 17.98970 -0.000000401 -0.000000134 +18435 17.99066 -0.000000401 -0.000000134 +18436 17.99163 -0.000000401 -0.000000134 +18437 17.99259 -0.000000401 -0.000000134 +18438 17.99356 -0.000000401 -0.000000134 +18439 17.99452 -0.000000401 -0.000000134 +18440 17.99549 -0.000000401 -0.000000134 +18441 17.99645 -0.000000400 -0.000000134 +18442 17.99742 -0.000000400 -0.000000133 +18443 17.99838 -0.000000400 -0.000000133 +18444 17.99935 -0.000000400 -0.000000133 +18445 18.00032 -0.000000400 -0.000000133 +18446 18.00128 -0.000000400 -0.000000133 +18447 18.00225 -0.000000400 -0.000000133 +18448 18.00321 -0.000000400 -0.000000133 +18449 18.00418 -0.000000399 -0.000000133 +18450 18.00514 -0.000000399 -0.000000133 +18451 18.00611 -0.000000399 -0.000000133 +18452 18.00707 -0.000000399 -0.000000133 +18453 18.00804 -0.000000399 -0.000000133 +18454 18.00900 -0.000000399 -0.000000133 +18455 18.00997 -0.000000399 -0.000000133 +18456 18.01093 -0.000000399 -0.000000133 +18457 18.01190 -0.000000398 -0.000000133 +18458 18.01286 -0.000000398 -0.000000133 +18459 18.01383 -0.000000398 -0.000000133 +18460 18.01479 -0.000000398 -0.000000133 +18461 18.01576 -0.000000398 -0.000000133 +18462 18.01672 -0.000000398 -0.000000132 +18463 18.01769 -0.000000398 -0.000000132 +18464 18.01865 -0.000000398 -0.000000132 +18465 18.01962 -0.000000397 -0.000000132 +18466 18.02058 -0.000000397 -0.000000132 +18467 18.02155 -0.000000397 -0.000000132 +18468 18.02251 -0.000000397 -0.000000132 +18469 18.02348 -0.000000397 -0.000000132 +18470 18.02444 -0.000000397 -0.000000132 +18471 18.02541 -0.000000397 -0.000000132 +18472 18.02637 -0.000000396 -0.000000132 +18473 18.02734 -0.000000396 -0.000000132 +18474 18.02830 -0.000000396 -0.000000132 +18475 18.02927 -0.000000396 -0.000000132 +18476 18.03023 -0.000000396 -0.000000132 +18477 18.03120 -0.000000396 -0.000000132 +18478 18.03216 -0.000000396 -0.000000132 +18479 18.03313 -0.000000396 -0.000000132 +18480 18.03409 -0.000000395 -0.000000132 +18481 18.03506 -0.000000395 -0.000000132 +18482 18.03602 -0.000000395 -0.000000131 +18483 18.03699 -0.000000395 -0.000000131 +18484 18.03795 -0.000000395 -0.000000131 +18485 18.03892 -0.000000395 -0.000000131 +18486 18.03988 -0.000000395 -0.000000131 +18487 18.04085 -0.000000395 -0.000000131 +18488 18.04181 -0.000000394 -0.000000131 +18489 18.04278 -0.000000394 -0.000000131 +18490 18.04374 -0.000000394 -0.000000131 +18491 18.04471 -0.000000394 -0.000000131 +18492 18.04567 -0.000000394 -0.000000131 +18493 18.04664 -0.000000394 -0.000000131 +18494 18.04760 -0.000000394 -0.000000131 +18495 18.04857 -0.000000394 -0.000000131 +18496 18.04953 -0.000000393 -0.000000131 +18497 18.05050 -0.000000393 -0.000000131 +18498 18.05146 -0.000000393 -0.000000131 +18499 18.05243 -0.000000393 -0.000000131 +18500 18.05339 -0.000000393 -0.000000131 +18501 18.05436 -0.000000393 -0.000000131 +18502 18.05532 -0.000000393 -0.000000130 +18503 18.05629 -0.000000393 -0.000000130 +18504 18.05725 -0.000000392 -0.000000130 +18505 18.05822 -0.000000392 -0.000000130 +18506 18.05918 -0.000000392 -0.000000130 +18507 18.06015 -0.000000392 -0.000000130 +18508 18.06111 -0.000000392 -0.000000130 +18509 18.06208 -0.000000392 -0.000000130 +18510 18.06304 -0.000000392 -0.000000130 +18511 18.06401 -0.000000392 -0.000000130 +18512 18.06497 -0.000000391 -0.000000130 +18513 18.06594 -0.000000391 -0.000000130 +18514 18.06690 -0.000000391 -0.000000130 +18515 18.06787 -0.000000391 -0.000000130 +18516 18.06883 -0.000000391 -0.000000130 +18517 18.06980 -0.000000391 -0.000000130 +18518 18.07076 -0.000000391 -0.000000130 +18519 18.07173 -0.000000391 -0.000000130 +18520 18.07269 -0.000000390 -0.000000130 +18521 18.07366 -0.000000390 -0.000000130 +18522 18.07462 -0.000000390 -0.000000130 +18523 18.07559 -0.000000390 -0.000000129 +18524 18.07655 -0.000000390 -0.000000129 +18525 18.07752 -0.000000390 -0.000000129 +18526 18.07848 -0.000000390 -0.000000129 +18527 18.07945 -0.000000390 -0.000000129 +18528 18.08041 -0.000000389 -0.000000129 +18529 18.08138 -0.000000389 -0.000000129 +18530 18.08234 -0.000000389 -0.000000129 +18531 18.08331 -0.000000389 -0.000000129 +18532 18.08427 -0.000000389 -0.000000129 +18533 18.08524 -0.000000389 -0.000000129 +18534 18.08620 -0.000000389 -0.000000129 +18535 18.08717 -0.000000389 -0.000000129 +18536 18.08813 -0.000000388 -0.000000129 +18537 18.08910 -0.000000388 -0.000000129 +18538 18.09006 -0.000000388 -0.000000129 +18539 18.09103 -0.000000388 -0.000000129 +18540 18.09199 -0.000000388 -0.000000129 +18541 18.09296 -0.000000388 -0.000000129 +18542 18.09392 -0.000000388 -0.000000129 +18543 18.09489 -0.000000388 -0.000000129 +18544 18.09585 -0.000000387 -0.000000128 +18545 18.09682 -0.000000387 -0.000000128 +18546 18.09778 -0.000000387 -0.000000128 +18547 18.09875 -0.000000387 -0.000000128 +18548 18.09971 -0.000000387 -0.000000128 +18549 18.10068 -0.000000387 -0.000000128 +18550 18.10165 -0.000000387 -0.000000128 +18551 18.10261 -0.000000387 -0.000000128 +18552 18.10358 -0.000000386 -0.000000128 +18553 18.10454 -0.000000386 -0.000000128 +18554 18.10551 -0.000000386 -0.000000128 +18555 18.10647 -0.000000386 -0.000000128 +18556 18.10744 -0.000000386 -0.000000128 +18557 18.10840 -0.000000386 -0.000000128 +18558 18.10937 -0.000000386 -0.000000128 +18559 18.11033 -0.000000386 -0.000000128 +18560 18.11130 -0.000000385 -0.000000128 +18561 18.11226 -0.000000385 -0.000000128 +18562 18.11323 -0.000000385 -0.000000128 +18563 18.11419 -0.000000385 -0.000000128 +18564 18.11516 -0.000000385 -0.000000128 +18565 18.11612 -0.000000385 -0.000000127 +18566 18.11709 -0.000000385 -0.000000127 +18567 18.11805 -0.000000385 -0.000000127 +18568 18.11902 -0.000000384 -0.000000127 +18569 18.11998 -0.000000384 -0.000000127 +18570 18.12095 -0.000000384 -0.000000127 +18571 18.12191 -0.000000384 -0.000000127 +18572 18.12288 -0.000000384 -0.000000127 +18573 18.12384 -0.000000384 -0.000000127 +18574 18.12481 -0.000000384 -0.000000127 +18575 18.12577 -0.000000384 -0.000000127 +18576 18.12674 -0.000000383 -0.000000127 +18577 18.12770 -0.000000383 -0.000000127 +18578 18.12867 -0.000000383 -0.000000127 +18579 18.12963 -0.000000383 -0.000000127 +18580 18.13060 -0.000000383 -0.000000127 +18581 18.13156 -0.000000383 -0.000000127 +18582 18.13253 -0.000000383 -0.000000127 +18583 18.13349 -0.000000383 -0.000000127 +18584 18.13446 -0.000000383 -0.000000127 +18585 18.13542 -0.000000382 -0.000000127 +18586 18.13639 -0.000000382 -0.000000126 +18587 18.13735 -0.000000382 -0.000000126 +18588 18.13832 -0.000000382 -0.000000126 +18589 18.13928 -0.000000382 -0.000000126 +18590 18.14025 -0.000000382 -0.000000126 +18591 18.14121 -0.000000382 -0.000000126 +18592 18.14218 -0.000000382 -0.000000126 +18593 18.14314 -0.000000381 -0.000000126 +18594 18.14411 -0.000000381 -0.000000126 +18595 18.14507 -0.000000381 -0.000000126 +18596 18.14604 -0.000000381 -0.000000126 +18597 18.14700 -0.000000381 -0.000000126 +18598 18.14797 -0.000000381 -0.000000126 +18599 18.14893 -0.000000381 -0.000000126 +18600 18.14990 -0.000000381 -0.000000126 +18601 18.15086 -0.000000380 -0.000000126 +18602 18.15183 -0.000000380 -0.000000126 +18603 18.15279 -0.000000380 -0.000000126 +18604 18.15376 -0.000000380 -0.000000126 +18605 18.15472 -0.000000380 -0.000000126 +18606 18.15569 -0.000000380 -0.000000126 +18607 18.15665 -0.000000380 -0.000000125 +18608 18.15762 -0.000000380 -0.000000125 +18609 18.15858 -0.000000379 -0.000000125 +18610 18.15955 -0.000000379 -0.000000125 +18611 18.16051 -0.000000379 -0.000000125 +18612 18.16148 -0.000000379 -0.000000125 +18613 18.16244 -0.000000379 -0.000000125 +18614 18.16341 -0.000000379 -0.000000125 +18615 18.16437 -0.000000379 -0.000000125 +18616 18.16534 -0.000000379 -0.000000125 +18617 18.16630 -0.000000379 -0.000000125 +18618 18.16727 -0.000000378 -0.000000125 +18619 18.16823 -0.000000378 -0.000000125 +18620 18.16920 -0.000000378 -0.000000125 +18621 18.17016 -0.000000378 -0.000000125 +18622 18.17113 -0.000000378 -0.000000125 +18623 18.17209 -0.000000378 -0.000000125 +18624 18.17306 -0.000000378 -0.000000125 +18625 18.17402 -0.000000378 -0.000000125 +18626 18.17499 -0.000000377 -0.000000125 +18627 18.17595 -0.000000377 -0.000000125 +18628 18.17692 -0.000000377 -0.000000125 +18629 18.17788 -0.000000377 -0.000000124 +18630 18.17885 -0.000000377 -0.000000124 +18631 18.17981 -0.000000377 -0.000000124 +18632 18.18078 -0.000000377 -0.000000124 +18633 18.18174 -0.000000377 -0.000000124 +18634 18.18271 -0.000000376 -0.000000124 +18635 18.18367 -0.000000376 -0.000000124 +18636 18.18464 -0.000000376 -0.000000124 +18637 18.18560 -0.000000376 -0.000000124 +18638 18.18657 -0.000000376 -0.000000124 +18639 18.18753 -0.000000376 -0.000000124 +18640 18.18850 -0.000000376 -0.000000124 +18641 18.18946 -0.000000376 -0.000000124 +18642 18.19043 -0.000000376 -0.000000124 +18643 18.19139 -0.000000375 -0.000000124 +18644 18.19236 -0.000000375 -0.000000124 +18645 18.19332 -0.000000375 -0.000000124 +18646 18.19429 -0.000000375 -0.000000124 +18647 18.19525 -0.000000375 -0.000000124 +18648 18.19622 -0.000000375 -0.000000124 +18649 18.19718 -0.000000375 -0.000000124 +18650 18.19815 -0.000000375 -0.000000123 +18651 18.19911 -0.000000374 -0.000000123 +18652 18.20008 -0.000000374 -0.000000123 +18653 18.20105 -0.000000374 -0.000000123 +18654 18.20201 -0.000000374 -0.000000123 +18655 18.20298 -0.000000374 -0.000000123 +18656 18.20394 -0.000000374 -0.000000123 +18657 18.20491 -0.000000374 -0.000000123 +18658 18.20587 -0.000000374 -0.000000123 +18659 18.20684 -0.000000373 -0.000000123 +18660 18.20780 -0.000000373 -0.000000123 +18661 18.20877 -0.000000373 -0.000000123 +18662 18.20973 -0.000000373 -0.000000123 +18663 18.21070 -0.000000373 -0.000000123 +18664 18.21166 -0.000000373 -0.000000123 +18665 18.21263 -0.000000373 -0.000000123 +18666 18.21359 -0.000000373 -0.000000123 +18667 18.21456 -0.000000373 -0.000000123 +18668 18.21552 -0.000000372 -0.000000123 +18669 18.21649 -0.000000372 -0.000000123 +18670 18.21745 -0.000000372 -0.000000123 +18671 18.21842 -0.000000372 -0.000000123 +18672 18.21938 -0.000000372 -0.000000122 +18673 18.22035 -0.000000372 -0.000000122 +18674 18.22131 -0.000000372 -0.000000122 +18675 18.22228 -0.000000372 -0.000000122 +18676 18.22324 -0.000000371 -0.000000122 +18677 18.22421 -0.000000371 -0.000000122 +18678 18.22517 -0.000000371 -0.000000122 +18679 18.22614 -0.000000371 -0.000000122 +18680 18.22710 -0.000000371 -0.000000122 +18681 18.22807 -0.000000371 -0.000000122 +18682 18.22903 -0.000000371 -0.000000122 +18683 18.23000 -0.000000371 -0.000000122 +18684 18.23096 -0.000000371 -0.000000122 +18685 18.23193 -0.000000370 -0.000000122 +18686 18.23289 -0.000000370 -0.000000122 +18687 18.23386 -0.000000370 -0.000000122 +18688 18.23482 -0.000000370 -0.000000122 +18689 18.23579 -0.000000370 -0.000000122 +18690 18.23675 -0.000000370 -0.000000122 +18691 18.23772 -0.000000370 -0.000000122 +18692 18.23868 -0.000000370 -0.000000122 +18693 18.23965 -0.000000369 -0.000000122 +18694 18.24061 -0.000000369 -0.000000121 +18695 18.24158 -0.000000369 -0.000000121 +18696 18.24254 -0.000000369 -0.000000121 +18697 18.24351 -0.000000369 -0.000000121 +18698 18.24447 -0.000000369 -0.000000121 +18699 18.24544 -0.000000369 -0.000000121 +18700 18.24640 -0.000000369 -0.000000121 +18701 18.24737 -0.000000369 -0.000000121 +18702 18.24833 -0.000000368 -0.000000121 +18703 18.24930 -0.000000368 -0.000000121 +18704 18.25026 -0.000000368 -0.000000121 +18705 18.25123 -0.000000368 -0.000000121 +18706 18.25219 -0.000000368 -0.000000121 +18707 18.25316 -0.000000368 -0.000000121 +18708 18.25412 -0.000000368 -0.000000121 +18709 18.25509 -0.000000368 -0.000000121 +18710 18.25605 -0.000000367 -0.000000121 +18711 18.25702 -0.000000367 -0.000000121 +18712 18.25798 -0.000000367 -0.000000121 +18713 18.25895 -0.000000367 -0.000000121 +18714 18.25991 -0.000000367 -0.000000121 +18715 18.26088 -0.000000367 -0.000000121 +18716 18.26184 -0.000000367 -0.000000121 +18717 18.26281 -0.000000367 -0.000000120 +18718 18.26377 -0.000000367 -0.000000120 +18719 18.26474 -0.000000366 -0.000000120 +18720 18.26570 -0.000000366 -0.000000120 +18721 18.26667 -0.000000366 -0.000000120 +18722 18.26763 -0.000000366 -0.000000120 +18723 18.26860 -0.000000366 -0.000000120 +18724 18.26956 -0.000000366 -0.000000120 +18725 18.27053 -0.000000366 -0.000000120 +18726 18.27149 -0.000000366 -0.000000120 +18727 18.27246 -0.000000366 -0.000000120 +18728 18.27342 -0.000000365 -0.000000120 +18729 18.27439 -0.000000365 -0.000000120 +18730 18.27535 -0.000000365 -0.000000120 +18731 18.27632 -0.000000365 -0.000000120 +18732 18.27728 -0.000000365 -0.000000120 +18733 18.27825 -0.000000365 -0.000000120 +18734 18.27921 -0.000000365 -0.000000120 +18735 18.28018 -0.000000365 -0.000000120 +18736 18.28114 -0.000000364 -0.000000120 +18737 18.28211 -0.000000364 -0.000000120 +18738 18.28307 -0.000000364 -0.000000120 +18739 18.28404 -0.000000364 -0.000000119 +18740 18.28500 -0.000000364 -0.000000119 +18741 18.28597 -0.000000364 -0.000000119 +18742 18.28693 -0.000000364 -0.000000119 +18743 18.28790 -0.000000364 -0.000000119 +18744 18.28886 -0.000000364 -0.000000119 +18745 18.28983 -0.000000363 -0.000000119 +18746 18.29079 -0.000000363 -0.000000119 +18747 18.29176 -0.000000363 -0.000000119 +18748 18.29272 -0.000000363 -0.000000119 +18749 18.29369 -0.000000363 -0.000000119 +18750 18.29465 -0.000000363 -0.000000119 +18751 18.29562 -0.000000363 -0.000000119 +18752 18.29658 -0.000000363 -0.000000119 +18753 18.29755 -0.000000363 -0.000000119 +18754 18.29851 -0.000000362 -0.000000119 +18755 18.29948 -0.000000362 -0.000000119 +18756 18.30045 -0.000000362 -0.000000119 +18757 18.30141 -0.000000362 -0.000000119 +18758 18.30238 -0.000000362 -0.000000119 +18759 18.30334 -0.000000362 -0.000000119 +18760 18.30431 -0.000000362 -0.000000119 +18761 18.30527 -0.000000362 -0.000000119 +18762 18.30624 -0.000000361 -0.000000118 +18763 18.30720 -0.000000361 -0.000000118 +18764 18.30817 -0.000000361 -0.000000118 +18765 18.30913 -0.000000361 -0.000000118 +18766 18.31010 -0.000000361 -0.000000118 +18767 18.31106 -0.000000361 -0.000000118 +18768 18.31203 -0.000000361 -0.000000118 +18769 18.31299 -0.000000361 -0.000000118 +18770 18.31396 -0.000000361 -0.000000118 +18771 18.31492 -0.000000360 -0.000000118 +18772 18.31589 -0.000000360 -0.000000118 +18773 18.31685 -0.000000360 -0.000000118 +18774 18.31782 -0.000000360 -0.000000118 +18775 18.31878 -0.000000360 -0.000000118 +18776 18.31975 -0.000000360 -0.000000118 +18777 18.32071 -0.000000360 -0.000000118 +18778 18.32168 -0.000000360 -0.000000118 +18779 18.32264 -0.000000360 -0.000000118 +18780 18.32361 -0.000000359 -0.000000118 +18781 18.32457 -0.000000359 -0.000000118 +18782 18.32554 -0.000000359 -0.000000118 +18783 18.32650 -0.000000359 -0.000000118 +18784 18.32747 -0.000000359 -0.000000118 +18785 18.32843 -0.000000359 -0.000000117 +18786 18.32940 -0.000000359 -0.000000117 +18787 18.33036 -0.000000359 -0.000000117 +18788 18.33133 -0.000000359 -0.000000117 +18789 18.33229 -0.000000358 -0.000000117 +18790 18.33326 -0.000000358 -0.000000117 +18791 18.33422 -0.000000358 -0.000000117 +18792 18.33519 -0.000000358 -0.000000117 +18793 18.33615 -0.000000358 -0.000000117 +18794 18.33712 -0.000000358 -0.000000117 +18795 18.33808 -0.000000358 -0.000000117 +18796 18.33905 -0.000000358 -0.000000117 +18797 18.34001 -0.000000358 -0.000000117 +18798 18.34098 -0.000000357 -0.000000117 +18799 18.34194 -0.000000357 -0.000000117 +18800 18.34291 -0.000000357 -0.000000117 +18801 18.34387 -0.000000357 -0.000000117 +18802 18.34484 -0.000000357 -0.000000117 +18803 18.34580 -0.000000357 -0.000000117 +18804 18.34677 -0.000000357 -0.000000117 +18805 18.34773 -0.000000357 -0.000000117 +18806 18.34870 -0.000000356 -0.000000117 +18807 18.34966 -0.000000356 -0.000000117 +18808 18.35063 -0.000000356 -0.000000116 +18809 18.35159 -0.000000356 -0.000000116 +18810 18.35256 -0.000000356 -0.000000116 +18811 18.35352 -0.000000356 -0.000000116 +18812 18.35449 -0.000000356 -0.000000116 +18813 18.35545 -0.000000356 -0.000000116 +18814 18.35642 -0.000000356 -0.000000116 +18815 18.35738 -0.000000355 -0.000000116 +18816 18.35835 -0.000000355 -0.000000116 +18817 18.35931 -0.000000355 -0.000000116 +18818 18.36028 -0.000000355 -0.000000116 +18819 18.36124 -0.000000355 -0.000000116 +18820 18.36221 -0.000000355 -0.000000116 +18821 18.36317 -0.000000355 -0.000000116 +18822 18.36414 -0.000000355 -0.000000116 +18823 18.36510 -0.000000355 -0.000000116 +18824 18.36607 -0.000000354 -0.000000116 +18825 18.36703 -0.000000354 -0.000000116 +18826 18.36800 -0.000000354 -0.000000116 +18827 18.36896 -0.000000354 -0.000000116 +18828 18.36993 -0.000000354 -0.000000116 +18829 18.37089 -0.000000354 -0.000000116 +18830 18.37186 -0.000000354 -0.000000116 +18831 18.37282 -0.000000354 -0.000000116 +18832 18.37379 -0.000000354 -0.000000115 +18833 18.37475 -0.000000353 -0.000000115 +18834 18.37572 -0.000000353 -0.000000115 +18835 18.37668 -0.000000353 -0.000000115 +18836 18.37765 -0.000000353 -0.000000115 +18837 18.37861 -0.000000353 -0.000000115 +18838 18.37958 -0.000000353 -0.000000115 +18839 18.38054 -0.000000353 -0.000000115 +18840 18.38151 -0.000000353 -0.000000115 +18841 18.38247 -0.000000353 -0.000000115 +18842 18.38344 -0.000000352 -0.000000115 +18843 18.38440 -0.000000352 -0.000000115 +18844 18.38537 -0.000000352 -0.000000115 +18845 18.38633 -0.000000352 -0.000000115 +18846 18.38730 -0.000000352 -0.000000115 +18847 18.38826 -0.000000352 -0.000000115 +18848 18.38923 -0.000000352 -0.000000115 +18849 18.39019 -0.000000352 -0.000000115 +18850 18.39116 -0.000000352 -0.000000115 +18851 18.39212 -0.000000351 -0.000000115 +18852 18.39309 -0.000000351 -0.000000115 +18853 18.39405 -0.000000351 -0.000000115 +18854 18.39502 -0.000000351 -0.000000115 +18855 18.39598 -0.000000351 -0.000000114 +18856 18.39695 -0.000000351 -0.000000114 +18857 18.39791 -0.000000351 -0.000000114 +18858 18.39888 -0.000000351 -0.000000114 +18859 18.39984 -0.000000351 -0.000000114 +18860 18.40081 -0.000000350 -0.000000114 +18861 18.40178 -0.000000350 -0.000000114 +18862 18.40274 -0.000000350 -0.000000114 +18863 18.40371 -0.000000350 -0.000000114 +18864 18.40467 -0.000000350 -0.000000114 +18865 18.40564 -0.000000350 -0.000000114 +18866 18.40660 -0.000000350 -0.000000114 +18867 18.40757 -0.000000350 -0.000000114 +18868 18.40853 -0.000000350 -0.000000114 +18869 18.40950 -0.000000349 -0.000000114 +18870 18.41046 -0.000000349 -0.000000114 +18871 18.41143 -0.000000349 -0.000000114 +18872 18.41239 -0.000000349 -0.000000114 +18873 18.41336 -0.000000349 -0.000000114 +18874 18.41432 -0.000000349 -0.000000114 +18875 18.41529 -0.000000349 -0.000000114 +18876 18.41625 -0.000000349 -0.000000114 +18877 18.41722 -0.000000349 -0.000000114 +18878 18.41818 -0.000000348 -0.000000114 +18879 18.41915 -0.000000348 -0.000000113 +18880 18.42011 -0.000000348 -0.000000113 +18881 18.42108 -0.000000348 -0.000000113 +18882 18.42204 -0.000000348 -0.000000113 +18883 18.42301 -0.000000348 -0.000000113 +18884 18.42397 -0.000000348 -0.000000113 +18885 18.42494 -0.000000348 -0.000000113 +18886 18.42590 -0.000000348 -0.000000113 +18887 18.42687 -0.000000348 -0.000000113 +18888 18.42783 -0.000000347 -0.000000113 +18889 18.42880 -0.000000347 -0.000000113 +18890 18.42976 -0.000000347 -0.000000113 +18891 18.43073 -0.000000347 -0.000000113 +18892 18.43169 -0.000000347 -0.000000113 +18893 18.43266 -0.000000347 -0.000000113 +18894 18.43362 -0.000000347 -0.000000113 +18895 18.43459 -0.000000347 -0.000000113 +18896 18.43555 -0.000000347 -0.000000113 +18897 18.43652 -0.000000346 -0.000000113 +18898 18.43748 -0.000000346 -0.000000113 +18899 18.43845 -0.000000346 -0.000000113 +18900 18.43941 -0.000000346 -0.000000113 +18901 18.44038 -0.000000346 -0.000000113 +18902 18.44134 -0.000000346 -0.000000113 +18903 18.44231 -0.000000346 -0.000000112 +18904 18.44327 -0.000000346 -0.000000112 +18905 18.44424 -0.000000346 -0.000000112 +18906 18.44520 -0.000000345 -0.000000112 +18907 18.44617 -0.000000345 -0.000000112 +18908 18.44713 -0.000000345 -0.000000112 +18909 18.44810 -0.000000345 -0.000000112 +18910 18.44906 -0.000000345 -0.000000112 +18911 18.45003 -0.000000345 -0.000000112 +18912 18.45099 -0.000000345 -0.000000112 +18913 18.45196 -0.000000345 -0.000000112 +18914 18.45292 -0.000000345 -0.000000112 +18915 18.45389 -0.000000344 -0.000000112 +18916 18.45485 -0.000000344 -0.000000112 +18917 18.45582 -0.000000344 -0.000000112 +18918 18.45678 -0.000000344 -0.000000112 +18919 18.45775 -0.000000344 -0.000000112 +18920 18.45871 -0.000000344 -0.000000112 +18921 18.45968 -0.000000344 -0.000000112 +18922 18.46064 -0.000000344 -0.000000112 +18923 18.46161 -0.000000344 -0.000000112 +18924 18.46257 -0.000000344 -0.000000112 +18925 18.46354 -0.000000343 -0.000000112 +18926 18.46450 -0.000000343 -0.000000112 +18927 18.46547 -0.000000343 -0.000000112 +18928 18.46643 -0.000000343 -0.000000111 +18929 18.46740 -0.000000343 -0.000000111 +18930 18.46836 -0.000000343 -0.000000111 +18931 18.46933 -0.000000343 -0.000000111 +18932 18.47029 -0.000000343 -0.000000111 +18933 18.47126 -0.000000343 -0.000000111 +18934 18.47222 -0.000000342 -0.000000111 +18935 18.47319 -0.000000342 -0.000000111 +18936 18.47415 -0.000000342 -0.000000111 +18937 18.47512 -0.000000342 -0.000000111 +18938 18.47608 -0.000000342 -0.000000111 +18939 18.47705 -0.000000342 -0.000000111 +18940 18.47801 -0.000000342 -0.000000111 +18941 18.47898 -0.000000342 -0.000000111 +18942 18.47994 -0.000000342 -0.000000111 +18943 18.48091 -0.000000341 -0.000000111 +18944 18.48187 -0.000000341 -0.000000111 +18945 18.48284 -0.000000341 -0.000000111 +18946 18.48380 -0.000000341 -0.000000111 +18947 18.48477 -0.000000341 -0.000000111 +18948 18.48573 -0.000000341 -0.000000111 +18949 18.48670 -0.000000341 -0.000000111 +18950 18.48766 -0.000000341 -0.000000111 +18951 18.48863 -0.000000341 -0.000000111 +18952 18.48959 -0.000000341 -0.000000110 +18953 18.49056 -0.000000340 -0.000000110 +18954 18.49152 -0.000000340 -0.000000110 +18955 18.49249 -0.000000340 -0.000000110 +18956 18.49345 -0.000000340 -0.000000110 +18957 18.49442 -0.000000340 -0.000000110 +18958 18.49538 -0.000000340 -0.000000110 +18959 18.49635 -0.000000340 -0.000000110 +18960 18.49731 -0.000000340 -0.000000110 +18961 18.49828 -0.000000340 -0.000000110 +18962 18.49924 -0.000000339 -0.000000110 +18963 18.50021 -0.000000339 -0.000000110 +18964 18.50118 -0.000000339 -0.000000110 +18965 18.50214 -0.000000339 -0.000000110 +18966 18.50311 -0.000000339 -0.000000110 +18967 18.50407 -0.000000339 -0.000000110 +18968 18.50504 -0.000000339 -0.000000110 +18969 18.50600 -0.000000339 -0.000000110 +18970 18.50697 -0.000000339 -0.000000110 +18971 18.50793 -0.000000338 -0.000000110 +18972 18.50890 -0.000000338 -0.000000110 +18973 18.50986 -0.000000338 -0.000000110 +18974 18.51083 -0.000000338 -0.000000110 +18975 18.51179 -0.000000338 -0.000000110 +18976 18.51276 -0.000000338 -0.000000110 +18977 18.51372 -0.000000338 -0.000000109 +18978 18.51469 -0.000000338 -0.000000109 +18979 18.51565 -0.000000338 -0.000000109 +18980 18.51662 -0.000000338 -0.000000109 +18981 18.51758 -0.000000337 -0.000000109 +18982 18.51855 -0.000000337 -0.000000109 +18983 18.51951 -0.000000337 -0.000000109 +18984 18.52048 -0.000000337 -0.000000109 +18985 18.52144 -0.000000337 -0.000000109 +18986 18.52241 -0.000000337 -0.000000109 +18987 18.52337 -0.000000337 -0.000000109 +18988 18.52434 -0.000000337 -0.000000109 +18989 18.52530 -0.000000337 -0.000000109 +18990 18.52627 -0.000000336 -0.000000109 +18991 18.52723 -0.000000336 -0.000000109 +18992 18.52820 -0.000000336 -0.000000109 +18993 18.52916 -0.000000336 -0.000000109 +18994 18.53013 -0.000000336 -0.000000109 +18995 18.53109 -0.000000336 -0.000000109 +18996 18.53206 -0.000000336 -0.000000109 +18997 18.53302 -0.000000336 -0.000000109 +18998 18.53399 -0.000000336 -0.000000109 +18999 18.53495 -0.000000336 -0.000000109 +19000 18.53592 -0.000000335 -0.000000109 +19001 18.53688 -0.000000335 -0.000000109 +19002 18.53785 -0.000000335 -0.000000108 +19003 18.53881 -0.000000335 -0.000000108 +19004 18.53978 -0.000000335 -0.000000108 +19005 18.54074 -0.000000335 -0.000000108 +19006 18.54171 -0.000000335 -0.000000108 +19007 18.54267 -0.000000335 -0.000000108 +19008 18.54364 -0.000000335 -0.000000108 +19009 18.54460 -0.000000334 -0.000000108 +19010 18.54557 -0.000000334 -0.000000108 +19011 18.54653 -0.000000334 -0.000000108 +19012 18.54750 -0.000000334 -0.000000108 +19013 18.54846 -0.000000334 -0.000000108 +19014 18.54943 -0.000000334 -0.000000108 +19015 18.55039 -0.000000334 -0.000000108 +19016 18.55136 -0.000000334 -0.000000108 +19017 18.55232 -0.000000334 -0.000000108 +19018 18.55329 -0.000000334 -0.000000108 +19019 18.55425 -0.000000333 -0.000000108 +19020 18.55522 -0.000000333 -0.000000108 +19021 18.55618 -0.000000333 -0.000000108 +19022 18.55715 -0.000000333 -0.000000108 +19023 18.55811 -0.000000333 -0.000000108 +19024 18.55908 -0.000000333 -0.000000108 +19025 18.56004 -0.000000333 -0.000000108 +19026 18.56101 -0.000000333 -0.000000108 +19027 18.56197 -0.000000333 -0.000000108 +19028 18.56294 -0.000000333 -0.000000107 +19029 18.56390 -0.000000332 -0.000000107 +19030 18.56487 -0.000000332 -0.000000107 +19031 18.56583 -0.000000332 -0.000000107 +19032 18.56680 -0.000000332 -0.000000107 +19033 18.56776 -0.000000332 -0.000000107 +19034 18.56873 -0.000000332 -0.000000107 +19035 18.56969 -0.000000332 -0.000000107 +19036 18.57066 -0.000000332 -0.000000107 +19037 18.57162 -0.000000332 -0.000000107 +19038 18.57259 -0.000000331 -0.000000107 +19039 18.57355 -0.000000331 -0.000000107 +19040 18.57452 -0.000000331 -0.000000107 +19041 18.57548 -0.000000331 -0.000000107 +19042 18.57645 -0.000000331 -0.000000107 +19043 18.57741 -0.000000331 -0.000000107 +19044 18.57838 -0.000000331 -0.000000107 +19045 18.57934 -0.000000331 -0.000000107 +19046 18.58031 -0.000000331 -0.000000107 +19047 18.58127 -0.000000331 -0.000000107 +19048 18.58224 -0.000000330 -0.000000107 +19049 18.58320 -0.000000330 -0.000000107 +19050 18.58417 -0.000000330 -0.000000107 +19051 18.58513 -0.000000330 -0.000000107 +19052 18.58610 -0.000000330 -0.000000107 +19053 18.58706 -0.000000330 -0.000000107 +19054 18.58803 -0.000000330 -0.000000106 +19055 18.58899 -0.000000330 -0.000000106 +19056 18.58996 -0.000000330 -0.000000106 +19057 18.59092 -0.000000330 -0.000000106 +19058 18.59189 -0.000000329 -0.000000106 +19059 18.59285 -0.000000329 -0.000000106 +19060 18.59382 -0.000000329 -0.000000106 +19061 18.59478 -0.000000329 -0.000000106 +19062 18.59575 -0.000000329 -0.000000106 +19063 18.59671 -0.000000329 -0.000000106 +19064 18.59768 -0.000000329 -0.000000106 +19065 18.59864 -0.000000329 -0.000000106 +19066 18.59961 -0.000000329 -0.000000106 +19067 18.60058 -0.000000328 -0.000000106 +19068 18.60154 -0.000000328 -0.000000106 +19069 18.60251 -0.000000328 -0.000000106 +19070 18.60347 -0.000000328 -0.000000106 +19071 18.60444 -0.000000328 -0.000000106 +19072 18.60540 -0.000000328 -0.000000106 +19073 18.60637 -0.000000328 -0.000000106 +19074 18.60733 -0.000000328 -0.000000106 +19075 18.60830 -0.000000328 -0.000000106 +19076 18.60926 -0.000000328 -0.000000106 +19077 18.61023 -0.000000327 -0.000000106 +19078 18.61119 -0.000000327 -0.000000106 +19079 18.61216 -0.000000327 -0.000000106 +19080 18.61312 -0.000000327 -0.000000105 +19081 18.61409 -0.000000327 -0.000000105 +19082 18.61505 -0.000000327 -0.000000105 +19083 18.61602 -0.000000327 -0.000000105 +19084 18.61698 -0.000000327 -0.000000105 +19085 18.61795 -0.000000327 -0.000000105 +19086 18.61891 -0.000000327 -0.000000105 +19087 18.61988 -0.000000326 -0.000000105 +19088 18.62084 -0.000000326 -0.000000105 +19089 18.62181 -0.000000326 -0.000000105 +19090 18.62277 -0.000000326 -0.000000105 +19091 18.62374 -0.000000326 -0.000000105 +19092 18.62470 -0.000000326 -0.000000105 +19093 18.62567 -0.000000326 -0.000000105 +19094 18.62663 -0.000000326 -0.000000105 +19095 18.62760 -0.000000326 -0.000000105 +19096 18.62856 -0.000000326 -0.000000105 +19097 18.62953 -0.000000325 -0.000000105 +19098 18.63049 -0.000000325 -0.000000105 +19099 18.63146 -0.000000325 -0.000000105 +19100 18.63242 -0.000000325 -0.000000105 +19101 18.63339 -0.000000325 -0.000000105 +19102 18.63435 -0.000000325 -0.000000105 +19103 18.63532 -0.000000325 -0.000000105 +19104 18.63628 -0.000000325 -0.000000105 +19105 18.63725 -0.000000325 -0.000000105 +19106 18.63821 -0.000000325 -0.000000104 +19107 18.63918 -0.000000324 -0.000000104 +19108 18.64014 -0.000000324 -0.000000104 +19109 18.64111 -0.000000324 -0.000000104 +19110 18.64207 -0.000000324 -0.000000104 +19111 18.64304 -0.000000324 -0.000000104 +19112 18.64400 -0.000000324 -0.000000104 +19113 18.64497 -0.000000324 -0.000000104 +19114 18.64593 -0.000000324 -0.000000104 +19115 18.64690 -0.000000324 -0.000000104 +19116 18.64786 -0.000000324 -0.000000104 +19117 18.64883 -0.000000323 -0.000000104 +19118 18.64979 -0.000000323 -0.000000104 +19119 18.65076 -0.000000323 -0.000000104 +19120 18.65172 -0.000000323 -0.000000104 +19121 18.65269 -0.000000323 -0.000000104 +19122 18.65365 -0.000000323 -0.000000104 +19123 18.65462 -0.000000323 -0.000000104 +19124 18.65558 -0.000000323 -0.000000104 +19125 18.65655 -0.000000323 -0.000000104 +19126 18.65751 -0.000000323 -0.000000104 +19127 18.65848 -0.000000322 -0.000000104 +19128 18.65944 -0.000000322 -0.000000104 +19129 18.66041 -0.000000322 -0.000000104 +19130 18.66137 -0.000000322 -0.000000104 +19131 18.66234 -0.000000322 -0.000000104 +19132 18.66330 -0.000000322 -0.000000103 +19133 18.66427 -0.000000322 -0.000000103 +19134 18.66523 -0.000000322 -0.000000103 +19135 18.66620 -0.000000322 -0.000000103 +19136 18.66716 -0.000000322 -0.000000103 +19137 18.66813 -0.000000321 -0.000000103 +19138 18.66909 -0.000000321 -0.000000103 +19139 18.67006 -0.000000321 -0.000000103 +19140 18.67102 -0.000000321 -0.000000103 +19141 18.67199 -0.000000321 -0.000000103 +19142 18.67295 -0.000000321 -0.000000103 +19143 18.67392 -0.000000321 -0.000000103 +19144 18.67488 -0.000000321 -0.000000103 +19145 18.67585 -0.000000321 -0.000000103 +19146 18.67681 -0.000000321 -0.000000103 +19147 18.67778 -0.000000320 -0.000000103 +19148 18.67874 -0.000000320 -0.000000103 +19149 18.67971 -0.000000320 -0.000000103 +19150 18.68067 -0.000000320 -0.000000103 +19151 18.68164 -0.000000320 -0.000000103 +19152 18.68260 -0.000000320 -0.000000103 +19153 18.68357 -0.000000320 -0.000000103 +19154 18.68453 -0.000000320 -0.000000103 +19155 18.68550 -0.000000320 -0.000000103 +19156 18.68646 -0.000000320 -0.000000103 +19157 18.68743 -0.000000319 -0.000000103 +19158 18.68839 -0.000000319 -0.000000103 +19159 18.68936 -0.000000319 -0.000000102 +19160 18.69032 -0.000000319 -0.000000102 +19161 18.69129 -0.000000319 -0.000000102 +19162 18.69225 -0.000000319 -0.000000102 +19163 18.69322 -0.000000319 -0.000000102 +19164 18.69418 -0.000000319 -0.000000102 +19165 18.69515 -0.000000319 -0.000000102 +19166 18.69611 -0.000000319 -0.000000102 +19167 18.69708 -0.000000318 -0.000000102 +19168 18.69804 -0.000000318 -0.000000102 +19169 18.69901 -0.000000318 -0.000000102 +19170 18.69997 -0.000000318 -0.000000102 +19171 18.70094 -0.000000318 -0.000000102 +19172 18.70191 -0.000000318 -0.000000102 +19173 18.70287 -0.000000318 -0.000000102 +19174 18.70384 -0.000000318 -0.000000102 +19175 18.70480 -0.000000318 -0.000000102 +19176 18.70577 -0.000000318 -0.000000102 +19177 18.70673 -0.000000317 -0.000000102 +19178 18.70770 -0.000000317 -0.000000102 +19179 18.70866 -0.000000317 -0.000000102 +19180 18.70963 -0.000000317 -0.000000102 +19181 18.71059 -0.000000317 -0.000000102 +19182 18.71156 -0.000000317 -0.000000102 +19183 18.71252 -0.000000317 -0.000000102 +19184 18.71349 -0.000000317 -0.000000102 +19185 18.71445 -0.000000317 -0.000000102 +19186 18.71542 -0.000000317 -0.000000101 +19187 18.71638 -0.000000316 -0.000000101 +19188 18.71735 -0.000000316 -0.000000101 +19189 18.71831 -0.000000316 -0.000000101 +19190 18.71928 -0.000000316 -0.000000101 +19191 18.72024 -0.000000316 -0.000000101 +19192 18.72121 -0.000000316 -0.000000101 +19193 18.72217 -0.000000316 -0.000000101 +19194 18.72314 -0.000000316 -0.000000101 +19195 18.72410 -0.000000316 -0.000000101 +19196 18.72507 -0.000000316 -0.000000101 +19197 18.72603 -0.000000316 -0.000000101 +19198 18.72700 -0.000000315 -0.000000101 +19199 18.72796 -0.000000315 -0.000000101 +19200 18.72893 -0.000000315 -0.000000101 +19201 18.72989 -0.000000315 -0.000000101 +19202 18.73086 -0.000000315 -0.000000101 +19203 18.73182 -0.000000315 -0.000000101 +19204 18.73279 -0.000000315 -0.000000101 +19205 18.73375 -0.000000315 -0.000000101 +19206 18.73472 -0.000000315 -0.000000101 +19207 18.73568 -0.000000315 -0.000000101 +19208 18.73665 -0.000000314 -0.000000101 +19209 18.73761 -0.000000314 -0.000000101 +19210 18.73858 -0.000000314 -0.000000101 +19211 18.73954 -0.000000314 -0.000000101 +19212 18.74051 -0.000000314 -0.000000101 +19213 18.74147 -0.000000314 -0.000000101 +19214 18.74244 -0.000000314 -0.000000100 +19215 18.74340 -0.000000314 -0.000000100 +19216 18.74437 -0.000000314 -0.000000100 +19217 18.74533 -0.000000314 -0.000000100 +19218 18.74630 -0.000000313 -0.000000100 +19219 18.74726 -0.000000313 -0.000000100 +19220 18.74823 -0.000000313 -0.000000100 +19221 18.74919 -0.000000313 -0.000000100 +19222 18.75016 -0.000000313 -0.000000100 +19223 18.75112 -0.000000313 -0.000000100 +19224 18.75209 -0.000000313 -0.000000100 +19225 18.75305 -0.000000313 -0.000000100 +19226 18.75402 -0.000000313 -0.000000100 +19227 18.75498 -0.000000313 -0.000000100 +19228 18.75595 -0.000000312 -0.000000100 +19229 18.75691 -0.000000312 -0.000000100 +19230 18.75788 -0.000000312 -0.000000100 +19231 18.75884 -0.000000312 -0.000000100 +19232 18.75981 -0.000000312 -0.000000100 +19233 18.76077 -0.000000312 -0.000000100 +19234 18.76174 -0.000000312 -0.000000100 +19235 18.76270 -0.000000312 -0.000000100 +19236 18.76367 -0.000000312 -0.000000100 +19237 18.76463 -0.000000312 -0.000000100 +19238 18.76560 -0.000000312 -0.000000100 +19239 18.76656 -0.000000311 -0.000000100 +19240 18.76753 -0.000000311 -0.000000100 +19241 18.76849 -0.000000311 -0.000000100 +19242 18.76946 -0.000000311 -0.000000099 +19243 18.77042 -0.000000311 -0.000000099 +19244 18.77139 -0.000000311 -0.000000099 +19245 18.77235 -0.000000311 -0.000000099 +19246 18.77332 -0.000000311 -0.000000099 +19247 18.77428 -0.000000311 -0.000000099 +19248 18.77525 -0.000000311 -0.000000099 +19249 18.77621 -0.000000310 -0.000000099 +19250 18.77718 -0.000000310 -0.000000099 +19251 18.77814 -0.000000310 -0.000000099 +19252 18.77911 -0.000000310 -0.000000099 +19253 18.78007 -0.000000310 -0.000000099 +19254 18.78104 -0.000000310 -0.000000099 +19255 18.78200 -0.000000310 -0.000000099 +19256 18.78297 -0.000000310 -0.000000099 +19257 18.78393 -0.000000310 -0.000000099 +19258 18.78490 -0.000000310 -0.000000099 +19259 18.78586 -0.000000310 -0.000000099 +19260 18.78683 -0.000000309 -0.000000099 +19261 18.78779 -0.000000309 -0.000000099 +19262 18.78876 -0.000000309 -0.000000099 +19263 18.78972 -0.000000309 -0.000000099 +19264 18.79069 -0.000000309 -0.000000099 +19265 18.79165 -0.000000309 -0.000000099 +19266 18.79262 -0.000000309 -0.000000099 +19267 18.79358 -0.000000309 -0.000000099 +19268 18.79455 -0.000000309 -0.000000099 +19269 18.79551 -0.000000309 -0.000000099 +19270 18.79648 -0.000000308 -0.000000098 +19271 18.79744 -0.000000308 -0.000000098 +19272 18.79841 -0.000000308 -0.000000098 +19273 18.79937 -0.000000308 -0.000000098 +19274 18.80034 -0.000000308 -0.000000098 +19275 18.80131 -0.000000308 -0.000000098 +19276 18.80227 -0.000000308 -0.000000098 +19277 18.80324 -0.000000308 -0.000000098 +19278 18.80420 -0.000000308 -0.000000098 +19279 18.80517 -0.000000308 -0.000000098 +19280 18.80613 -0.000000308 -0.000000098 +19281 18.80710 -0.000000307 -0.000000098 +19282 18.80806 -0.000000307 -0.000000098 +19283 18.80903 -0.000000307 -0.000000098 +19284 18.80999 -0.000000307 -0.000000098 +19285 18.81096 -0.000000307 -0.000000098 +19286 18.81192 -0.000000307 -0.000000098 +19287 18.81289 -0.000000307 -0.000000098 +19288 18.81385 -0.000000307 -0.000000098 +19289 18.81482 -0.000000307 -0.000000098 +19290 18.81578 -0.000000307 -0.000000098 +19291 18.81675 -0.000000306 -0.000000098 +19292 18.81771 -0.000000306 -0.000000098 +19293 18.81868 -0.000000306 -0.000000098 +19294 18.81964 -0.000000306 -0.000000098 +19295 18.82061 -0.000000306 -0.000000098 +19296 18.82157 -0.000000306 -0.000000098 +19297 18.82254 -0.000000306 -0.000000098 +19298 18.82350 -0.000000306 -0.000000097 +19299 18.82447 -0.000000306 -0.000000097 +19300 18.82543 -0.000000306 -0.000000097 +19301 18.82640 -0.000000306 -0.000000097 +19302 18.82736 -0.000000305 -0.000000097 +19303 18.82833 -0.000000305 -0.000000097 +19304 18.82929 -0.000000305 -0.000000097 +19305 18.83026 -0.000000305 -0.000000097 +19306 18.83122 -0.000000305 -0.000000097 +19307 18.83219 -0.000000305 -0.000000097 +19308 18.83315 -0.000000305 -0.000000097 +19309 18.83412 -0.000000305 -0.000000097 +19310 18.83508 -0.000000305 -0.000000097 +19311 18.83605 -0.000000305 -0.000000097 +19312 18.83701 -0.000000305 -0.000000097 +19313 18.83798 -0.000000304 -0.000000097 +19314 18.83894 -0.000000304 -0.000000097 +19315 18.83991 -0.000000304 -0.000000097 +19316 18.84087 -0.000000304 -0.000000097 +19317 18.84184 -0.000000304 -0.000000097 +19318 18.84280 -0.000000304 -0.000000097 +19319 18.84377 -0.000000304 -0.000000097 +19320 18.84473 -0.000000304 -0.000000097 +19321 18.84570 -0.000000304 -0.000000097 +19322 18.84666 -0.000000304 -0.000000097 +19323 18.84763 -0.000000303 -0.000000097 +19324 18.84859 -0.000000303 -0.000000097 +19325 18.84956 -0.000000303 -0.000000097 +19326 18.85052 -0.000000303 -0.000000097 +19327 18.85149 -0.000000303 -0.000000096 +19328 18.85245 -0.000000303 -0.000000096 +19329 18.85342 -0.000000303 -0.000000096 +19330 18.85438 -0.000000303 -0.000000096 +19331 18.85535 -0.000000303 -0.000000096 +19332 18.85631 -0.000000303 -0.000000096 +19333 18.85728 -0.000000303 -0.000000096 +19334 18.85824 -0.000000302 -0.000000096 +19335 18.85921 -0.000000302 -0.000000096 +19336 18.86017 -0.000000302 -0.000000096 +19337 18.86114 -0.000000302 -0.000000096 +19338 18.86210 -0.000000302 -0.000000096 +19339 18.86307 -0.000000302 -0.000000096 +19340 18.86403 -0.000000302 -0.000000096 +19341 18.86500 -0.000000302 -0.000000096 +19342 18.86596 -0.000000302 -0.000000096 +19343 18.86693 -0.000000302 -0.000000096 +19344 18.86789 -0.000000302 -0.000000096 +19345 18.86886 -0.000000301 -0.000000096 +19346 18.86982 -0.000000301 -0.000000096 +19347 18.87079 -0.000000301 -0.000000096 +19348 18.87175 -0.000000301 -0.000000096 +19349 18.87272 -0.000000301 -0.000000096 +19350 18.87368 -0.000000301 -0.000000096 +19351 18.87465 -0.000000301 -0.000000096 +19352 18.87561 -0.000000301 -0.000000096 +19353 18.87658 -0.000000301 -0.000000096 +19354 18.87754 -0.000000301 -0.000000096 +19355 18.87851 -0.000000301 -0.000000096 +19356 18.87947 -0.000000300 -0.000000095 +19357 18.88044 -0.000000300 -0.000000095 +19358 18.88140 -0.000000300 -0.000000095 +19359 18.88237 -0.000000300 -0.000000095 +19360 18.88333 -0.000000300 -0.000000095 +19361 18.88430 -0.000000300 -0.000000095 +19362 18.88526 -0.000000300 -0.000000095 +19363 18.88623 -0.000000300 -0.000000095 +19364 18.88719 -0.000000300 -0.000000095 +19365 18.88816 -0.000000300 -0.000000095 +19366 18.88912 -0.000000300 -0.000000095 +19367 18.89009 -0.000000299 -0.000000095 +19368 18.89105 -0.000000299 -0.000000095 +19369 18.89202 -0.000000299 -0.000000095 +19370 18.89298 -0.000000299 -0.000000095 +19371 18.89395 -0.000000299 -0.000000095 +19372 18.89491 -0.000000299 -0.000000095 +19373 18.89588 -0.000000299 -0.000000095 +19374 18.89684 -0.000000299 -0.000000095 +19375 18.89781 -0.000000299 -0.000000095 +19376 18.89877 -0.000000299 -0.000000095 +19377 18.89974 -0.000000299 -0.000000095 +19378 18.90071 -0.000000298 -0.000000095 +19379 18.90167 -0.000000298 -0.000000095 +19380 18.90264 -0.000000298 -0.000000095 +19381 18.90360 -0.000000298 -0.000000095 +19382 18.90457 -0.000000298 -0.000000095 +19383 18.90553 -0.000000298 -0.000000095 +19384 18.90650 -0.000000298 -0.000000095 +19385 18.90746 -0.000000298 -0.000000094 +19386 18.90843 -0.000000298 -0.000000094 +19387 18.90939 -0.000000298 -0.000000094 +19388 18.91036 -0.000000297 -0.000000094 +19389 18.91132 -0.000000297 -0.000000094 +19390 18.91229 -0.000000297 -0.000000094 +19391 18.91325 -0.000000297 -0.000000094 +19392 18.91422 -0.000000297 -0.000000094 +19393 18.91518 -0.000000297 -0.000000094 +19394 18.91615 -0.000000297 -0.000000094 +19395 18.91711 -0.000000297 -0.000000094 +19396 18.91808 -0.000000297 -0.000000094 +19397 18.91904 -0.000000297 -0.000000094 +19398 18.92001 -0.000000297 -0.000000094 +19399 18.92097 -0.000000296 -0.000000094 +19400 18.92194 -0.000000296 -0.000000094 +19401 18.92290 -0.000000296 -0.000000094 +19402 18.92387 -0.000000296 -0.000000094 +19403 18.92483 -0.000000296 -0.000000094 +19404 18.92580 -0.000000296 -0.000000094 +19405 18.92676 -0.000000296 -0.000000094 +19406 18.92773 -0.000000296 -0.000000094 +19407 18.92869 -0.000000296 -0.000000094 +19408 18.92966 -0.000000296 -0.000000094 +19409 18.93062 -0.000000296 -0.000000094 +19410 18.93159 -0.000000296 -0.000000094 +19411 18.93255 -0.000000295 -0.000000094 +19412 18.93352 -0.000000295 -0.000000094 +19413 18.93448 -0.000000295 -0.000000094 +19414 18.93545 -0.000000295 -0.000000094 +19415 18.93641 -0.000000295 -0.000000093 +19416 18.93738 -0.000000295 -0.000000093 +19417 18.93834 -0.000000295 -0.000000093 +19418 18.93931 -0.000000295 -0.000000093 +19419 18.94027 -0.000000295 -0.000000093 +19420 18.94124 -0.000000295 -0.000000093 +19421 18.94220 -0.000000295 -0.000000093 +19422 18.94317 -0.000000294 -0.000000093 +19423 18.94413 -0.000000294 -0.000000093 +19424 18.94510 -0.000000294 -0.000000093 +19425 18.94606 -0.000000294 -0.000000093 +19426 18.94703 -0.000000294 -0.000000093 +19427 18.94799 -0.000000294 -0.000000093 +19428 18.94896 -0.000000294 -0.000000093 +19429 18.94992 -0.000000294 -0.000000093 +19430 18.95089 -0.000000294 -0.000000093 +19431 18.95185 -0.000000294 -0.000000093 +19432 18.95282 -0.000000294 -0.000000093 +19433 18.95378 -0.000000293 -0.000000093 +19434 18.95475 -0.000000293 -0.000000093 +19435 18.95571 -0.000000293 -0.000000093 +19436 18.95668 -0.000000293 -0.000000093 +19437 18.95764 -0.000000293 -0.000000093 +19438 18.95861 -0.000000293 -0.000000093 +19439 18.95957 -0.000000293 -0.000000093 +19440 18.96054 -0.000000293 -0.000000093 +19441 18.96150 -0.000000293 -0.000000093 +19442 18.96247 -0.000000293 -0.000000093 +19443 18.96343 -0.000000293 -0.000000093 +19444 18.96440 -0.000000292 -0.000000093 +19445 18.96536 -0.000000292 -0.000000092 +19446 18.96633 -0.000000292 -0.000000092 +19447 18.96729 -0.000000292 -0.000000092 +19448 18.96826 -0.000000292 -0.000000092 +19449 18.96922 -0.000000292 -0.000000092 +19450 18.97019 -0.000000292 -0.000000092 +19451 18.97115 -0.000000292 -0.000000092 +19452 18.97212 -0.000000292 -0.000000092 +19453 18.97308 -0.000000292 -0.000000092 +19454 18.97405 -0.000000292 -0.000000092 +19455 18.97501 -0.000000291 -0.000000092 +19456 18.97598 -0.000000291 -0.000000092 +19457 18.97694 -0.000000291 -0.000000092 +19458 18.97791 -0.000000291 -0.000000092 +19459 18.97887 -0.000000291 -0.000000092 +19460 18.97984 -0.000000291 -0.000000092 +19461 18.98080 -0.000000291 -0.000000092 +19462 18.98177 -0.000000291 -0.000000092 +19463 18.98273 -0.000000291 -0.000000092 +19464 18.98370 -0.000000291 -0.000000092 +19465 18.98466 -0.000000291 -0.000000092 +19466 18.98563 -0.000000290 -0.000000092 +19467 18.98659 -0.000000290 -0.000000092 +19468 18.98756 -0.000000290 -0.000000092 +19469 18.98852 -0.000000290 -0.000000092 +19470 18.98949 -0.000000290 -0.000000092 +19471 18.99045 -0.000000290 -0.000000092 +19472 18.99142 -0.000000290 -0.000000092 +19473 18.99238 -0.000000290 -0.000000092 +19474 18.99335 -0.000000290 -0.000000092 +19475 18.99431 -0.000000290 -0.000000092 +19476 18.99528 -0.000000290 -0.000000091 +19477 18.99624 -0.000000290 -0.000000091 +19478 18.99721 -0.000000289 -0.000000091 +19479 18.99817 -0.000000289 -0.000000091 +19480 18.99914 -0.000000289 -0.000000091 +19481 19.00011 -0.000000289 -0.000000091 +19482 19.00107 -0.000000289 -0.000000091 +19483 19.00204 -0.000000289 -0.000000091 +19484 19.00300 -0.000000289 -0.000000091 +19485 19.00397 -0.000000289 -0.000000091 +19486 19.00493 -0.000000289 -0.000000091 +19487 19.00590 -0.000000289 -0.000000091 +19488 19.00686 -0.000000289 -0.000000091 +19489 19.00783 -0.000000288 -0.000000091 +19490 19.00879 -0.000000288 -0.000000091 +19491 19.00976 -0.000000288 -0.000000091 +19492 19.01072 -0.000000288 -0.000000091 +19493 19.01169 -0.000000288 -0.000000091 +19494 19.01265 -0.000000288 -0.000000091 +19495 19.01362 -0.000000288 -0.000000091 +19496 19.01458 -0.000000288 -0.000000091 +19497 19.01555 -0.000000288 -0.000000091 +19498 19.01651 -0.000000288 -0.000000091 +19499 19.01748 -0.000000288 -0.000000091 +19500 19.01844 -0.000000287 -0.000000091 +19501 19.01941 -0.000000287 -0.000000091 +19502 19.02037 -0.000000287 -0.000000091 +19503 19.02134 -0.000000287 -0.000000091 +19504 19.02230 -0.000000287 -0.000000091 +19505 19.02327 -0.000000287 -0.000000091 +19506 19.02423 -0.000000287 -0.000000091 +19507 19.02520 -0.000000287 -0.000000090 +19508 19.02616 -0.000000287 -0.000000090 +19509 19.02713 -0.000000287 -0.000000090 +19510 19.02809 -0.000000287 -0.000000090 +19511 19.02906 -0.000000287 -0.000000090 +19512 19.03002 -0.000000286 -0.000000090 +19513 19.03099 -0.000000286 -0.000000090 +19514 19.03195 -0.000000286 -0.000000090 +19515 19.03292 -0.000000286 -0.000000090 +19516 19.03388 -0.000000286 -0.000000090 +19517 19.03485 -0.000000286 -0.000000090 +19518 19.03581 -0.000000286 -0.000000090 +19519 19.03678 -0.000000286 -0.000000090 +19520 19.03774 -0.000000286 -0.000000090 +19521 19.03871 -0.000000286 -0.000000090 +19522 19.03967 -0.000000286 -0.000000090 +19523 19.04064 -0.000000285 -0.000000090 +19524 19.04160 -0.000000285 -0.000000090 +19525 19.04257 -0.000000285 -0.000000090 +19526 19.04353 -0.000000285 -0.000000090 +19527 19.04450 -0.000000285 -0.000000090 +19528 19.04546 -0.000000285 -0.000000090 +19529 19.04643 -0.000000285 -0.000000090 +19530 19.04739 -0.000000285 -0.000000090 +19531 19.04836 -0.000000285 -0.000000090 +19532 19.04932 -0.000000285 -0.000000090 +19533 19.05029 -0.000000285 -0.000000090 +19534 19.05125 -0.000000285 -0.000000090 +19535 19.05222 -0.000000284 -0.000000090 +19536 19.05318 -0.000000284 -0.000000090 +19537 19.05415 -0.000000284 -0.000000090 +19538 19.05511 -0.000000284 -0.000000089 +19539 19.05608 -0.000000284 -0.000000089 +19540 19.05704 -0.000000284 -0.000000089 +19541 19.05801 -0.000000284 -0.000000089 +19542 19.05897 -0.000000284 -0.000000089 +19543 19.05994 -0.000000284 -0.000000089 +19544 19.06090 -0.000000284 -0.000000089 +19545 19.06187 -0.000000284 -0.000000089 +19546 19.06283 -0.000000284 -0.000000089 +19547 19.06380 -0.000000283 -0.000000089 +19548 19.06476 -0.000000283 -0.000000089 +19549 19.06573 -0.000000283 -0.000000089 +19550 19.06669 -0.000000283 -0.000000089 +19551 19.06766 -0.000000283 -0.000000089 +19552 19.06862 -0.000000283 -0.000000089 +19553 19.06959 -0.000000283 -0.000000089 +19554 19.07055 -0.000000283 -0.000000089 +19555 19.07152 -0.000000283 -0.000000089 +19556 19.07248 -0.000000283 -0.000000089 +19557 19.07345 -0.000000283 -0.000000089 +19558 19.07441 -0.000000282 -0.000000089 +19559 19.07538 -0.000000282 -0.000000089 +19560 19.07634 -0.000000282 -0.000000089 +19561 19.07731 -0.000000282 -0.000000089 +19562 19.07827 -0.000000282 -0.000000089 +19563 19.07924 -0.000000282 -0.000000089 +19564 19.08020 -0.000000282 -0.000000089 +19565 19.08117 -0.000000282 -0.000000089 +19566 19.08213 -0.000000282 -0.000000089 +19567 19.08310 -0.000000282 -0.000000089 +19568 19.08406 -0.000000282 -0.000000089 +19569 19.08503 -0.000000282 -0.000000089 +19570 19.08599 -0.000000281 -0.000000088 +19571 19.08696 -0.000000281 -0.000000088 +19572 19.08792 -0.000000281 -0.000000088 +19573 19.08889 -0.000000281 -0.000000088 +19574 19.08985 -0.000000281 -0.000000088 +19575 19.09082 -0.000000281 -0.000000088 +19576 19.09178 -0.000000281 -0.000000088 +19577 19.09275 -0.000000281 -0.000000088 +19578 19.09371 -0.000000281 -0.000000088 +19579 19.09468 -0.000000281 -0.000000088 +19580 19.09564 -0.000000281 -0.000000088 +19581 19.09661 -0.000000281 -0.000000088 +19582 19.09757 -0.000000280 -0.000000088 +19583 19.09854 -0.000000280 -0.000000088 +19584 19.09950 -0.000000280 -0.000000088 +19585 19.10047 -0.000000280 -0.000000088 +19586 19.10144 -0.000000280 -0.000000088 +19587 19.10240 -0.000000280 -0.000000088 +19588 19.10337 -0.000000280 -0.000000088 +19589 19.10433 -0.000000280 -0.000000088 +19590 19.10530 -0.000000280 -0.000000088 +19591 19.10626 -0.000000280 -0.000000088 +19592 19.10723 -0.000000280 -0.000000088 +19593 19.10819 -0.000000279 -0.000000088 +19594 19.10916 -0.000000279 -0.000000088 +19595 19.11012 -0.000000279 -0.000000088 +19596 19.11109 -0.000000279 -0.000000088 +19597 19.11205 -0.000000279 -0.000000088 +19598 19.11302 -0.000000279 -0.000000088 +19599 19.11398 -0.000000279 -0.000000088 +19600 19.11495 -0.000000279 -0.000000088 +19601 19.11591 -0.000000279 -0.000000088 +19602 19.11688 -0.000000279 -0.000000087 +19603 19.11784 -0.000000279 -0.000000087 +19604 19.11881 -0.000000279 -0.000000087 +19605 19.11977 -0.000000278 -0.000000087 +19606 19.12074 -0.000000278 -0.000000087 +19607 19.12170 -0.000000278 -0.000000087 +19608 19.12267 -0.000000278 -0.000000087 +19609 19.12363 -0.000000278 -0.000000087 +19610 19.12460 -0.000000278 -0.000000087 +19611 19.12556 -0.000000278 -0.000000087 +19612 19.12653 -0.000000278 -0.000000087 +19613 19.12749 -0.000000278 -0.000000087 +19614 19.12846 -0.000000278 -0.000000087 +19615 19.12942 -0.000000278 -0.000000087 +19616 19.13039 -0.000000278 -0.000000087 +19617 19.13135 -0.000000277 -0.000000087 +19618 19.13232 -0.000000277 -0.000000087 +19619 19.13328 -0.000000277 -0.000000087 +19620 19.13425 -0.000000277 -0.000000087 +19621 19.13521 -0.000000277 -0.000000087 +19622 19.13618 -0.000000277 -0.000000087 +19623 19.13714 -0.000000277 -0.000000087 +19624 19.13811 -0.000000277 -0.000000087 +19625 19.13907 -0.000000277 -0.000000087 +19626 19.14004 -0.000000277 -0.000000087 +19627 19.14100 -0.000000277 -0.000000087 +19628 19.14197 -0.000000277 -0.000000087 +19629 19.14293 -0.000000276 -0.000000087 +19630 19.14390 -0.000000276 -0.000000087 +19631 19.14486 -0.000000276 -0.000000087 +19632 19.14583 -0.000000276 -0.000000087 +19633 19.14679 -0.000000276 -0.000000087 +19634 19.14776 -0.000000276 -0.000000086 +19635 19.14872 -0.000000276 -0.000000086 +19636 19.14969 -0.000000276 -0.000000086 +19637 19.15065 -0.000000276 -0.000000086 +19638 19.15162 -0.000000276 -0.000000086 +19639 19.15258 -0.000000276 -0.000000086 +19640 19.15355 -0.000000276 -0.000000086 +19641 19.15451 -0.000000275 -0.000000086 +19642 19.15548 -0.000000275 -0.000000086 +19643 19.15644 -0.000000275 -0.000000086 +19644 19.15741 -0.000000275 -0.000000086 +19645 19.15837 -0.000000275 -0.000000086 +19646 19.15934 -0.000000275 -0.000000086 +19647 19.16030 -0.000000275 -0.000000086 +19648 19.16127 -0.000000275 -0.000000086 +19649 19.16223 -0.000000275 -0.000000086 +19650 19.16320 -0.000000275 -0.000000086 +19651 19.16416 -0.000000275 -0.000000086 +19652 19.16513 -0.000000275 -0.000000086 +19653 19.16609 -0.000000274 -0.000000086 +19654 19.16706 -0.000000274 -0.000000086 +19655 19.16802 -0.000000274 -0.000000086 +19656 19.16899 -0.000000274 -0.000000086 +19657 19.16995 -0.000000274 -0.000000086 +19658 19.17092 -0.000000274 -0.000000086 +19659 19.17188 -0.000000274 -0.000000086 +19660 19.17285 -0.000000274 -0.000000086 +19661 19.17381 -0.000000274 -0.000000086 +19662 19.17478 -0.000000274 -0.000000086 +19663 19.17574 -0.000000274 -0.000000086 +19664 19.17671 -0.000000274 -0.000000086 +19665 19.17767 -0.000000273 -0.000000086 +19666 19.17864 -0.000000273 -0.000000086 +19667 19.17960 -0.000000273 -0.000000085 +19668 19.18057 -0.000000273 -0.000000085 +19669 19.18153 -0.000000273 -0.000000085 +19670 19.18250 -0.000000273 -0.000000085 +19671 19.18346 -0.000000273 -0.000000085 +19672 19.18443 -0.000000273 -0.000000085 +19673 19.18539 -0.000000273 -0.000000085 +19674 19.18636 -0.000000273 -0.000000085 +19675 19.18732 -0.000000273 -0.000000085 +19676 19.18829 -0.000000273 -0.000000085 +19677 19.18925 -0.000000272 -0.000000085 +19678 19.19022 -0.000000272 -0.000000085 +19679 19.19118 -0.000000272 -0.000000085 +19680 19.19215 -0.000000272 -0.000000085 +19681 19.19311 -0.000000272 -0.000000085 +19682 19.19408 -0.000000272 -0.000000085 +19683 19.19504 -0.000000272 -0.000000085 +19684 19.19601 -0.000000272 -0.000000085 +19685 19.19697 -0.000000272 -0.000000085 +19686 19.19794 -0.000000272 -0.000000085 +19687 19.19890 -0.000000272 -0.000000085 +19688 19.19987 -0.000000272 -0.000000085 +19689 19.20084 -0.000000271 -0.000000085 +19690 19.20180 -0.000000271 -0.000000085 +19691 19.20277 -0.000000271 -0.000000085 +19692 19.20373 -0.000000271 -0.000000085 +19693 19.20470 -0.000000271 -0.000000085 +19694 19.20566 -0.000000271 -0.000000085 +19695 19.20663 -0.000000271 -0.000000085 +19696 19.20759 -0.000000271 -0.000000085 +19697 19.20856 -0.000000271 -0.000000085 +19698 19.20952 -0.000000271 -0.000000085 +19699 19.21049 -0.000000271 -0.000000085 +19700 19.21145 -0.000000271 -0.000000085 +19701 19.21242 -0.000000271 -0.000000084 +19702 19.21338 -0.000000270 -0.000000084 +19703 19.21435 -0.000000270 -0.000000084 +19704 19.21531 -0.000000270 -0.000000084 +19705 19.21628 -0.000000270 -0.000000084 +19706 19.21724 -0.000000270 -0.000000084 +19707 19.21821 -0.000000270 -0.000000084 +19708 19.21917 -0.000000270 -0.000000084 +19709 19.22014 -0.000000270 -0.000000084 +19710 19.22110 -0.000000270 -0.000000084 +19711 19.22207 -0.000000270 -0.000000084 +19712 19.22303 -0.000000270 -0.000000084 +19713 19.22400 -0.000000270 -0.000000084 +19714 19.22496 -0.000000269 -0.000000084 +19715 19.22593 -0.000000269 -0.000000084 +19716 19.22689 -0.000000269 -0.000000084 +19717 19.22786 -0.000000269 -0.000000084 +19718 19.22882 -0.000000269 -0.000000084 +19719 19.22979 -0.000000269 -0.000000084 +19720 19.23075 -0.000000269 -0.000000084 +19721 19.23172 -0.000000269 -0.000000084 +19722 19.23268 -0.000000269 -0.000000084 +19723 19.23365 -0.000000269 -0.000000084 +19724 19.23461 -0.000000269 -0.000000084 +19725 19.23558 -0.000000269 -0.000000084 +19726 19.23654 -0.000000268 -0.000000084 +19727 19.23751 -0.000000268 -0.000000084 +19728 19.23847 -0.000000268 -0.000000084 +19729 19.23944 -0.000000268 -0.000000084 +19730 19.24040 -0.000000268 -0.000000084 +19731 19.24137 -0.000000268 -0.000000084 +19732 19.24233 -0.000000268 -0.000000084 +19733 19.24330 -0.000000268 -0.000000084 +19734 19.24426 -0.000000268 -0.000000084 +19735 19.24523 -0.000000268 -0.000000083 +19736 19.24619 -0.000000268 -0.000000083 +19737 19.24716 -0.000000268 -0.000000083 +19738 19.24812 -0.000000268 -0.000000083 +19739 19.24909 -0.000000267 -0.000000083 +19740 19.25005 -0.000000267 -0.000000083 +19741 19.25102 -0.000000267 -0.000000083 +19742 19.25198 -0.000000267 -0.000000083 +19743 19.25295 -0.000000267 -0.000000083 +19744 19.25391 -0.000000267 -0.000000083 +19745 19.25488 -0.000000267 -0.000000083 +19746 19.25584 -0.000000267 -0.000000083 +19747 19.25681 -0.000000267 -0.000000083 +19748 19.25777 -0.000000267 -0.000000083 +19749 19.25874 -0.000000267 -0.000000083 +19750 19.25970 -0.000000267 -0.000000083 +19751 19.26067 -0.000000266 -0.000000083 +19752 19.26163 -0.000000266 -0.000000083 +19753 19.26260 -0.000000266 -0.000000083 +19754 19.26356 -0.000000266 -0.000000083 +19755 19.26453 -0.000000266 -0.000000083 +19756 19.26549 -0.000000266 -0.000000083 +19757 19.26646 -0.000000266 -0.000000083 +19758 19.26742 -0.000000266 -0.000000083 +19759 19.26839 -0.000000266 -0.000000083 +19760 19.26935 -0.000000266 -0.000000083 +19761 19.27032 -0.000000266 -0.000000083 +19762 19.27128 -0.000000266 -0.000000083 +19763 19.27225 -0.000000266 -0.000000083 +19764 19.27321 -0.000000265 -0.000000083 +19765 19.27418 -0.000000265 -0.000000083 +19766 19.27514 -0.000000265 -0.000000083 +19767 19.27611 -0.000000265 -0.000000083 +19768 19.27707 -0.000000265 -0.000000083 +19769 19.27804 -0.000000265 -0.000000082 +19770 19.27900 -0.000000265 -0.000000082 +19771 19.27997 -0.000000265 -0.000000082 +19772 19.28093 -0.000000265 -0.000000082 +19773 19.28190 -0.000000265 -0.000000082 +19774 19.28286 -0.000000265 -0.000000082 +19775 19.28383 -0.000000265 -0.000000082 +19776 19.28479 -0.000000264 -0.000000082 +19777 19.28576 -0.000000264 -0.000000082 +19778 19.28672 -0.000000264 -0.000000082 +19779 19.28769 -0.000000264 -0.000000082 +19780 19.28865 -0.000000264 -0.000000082 +19781 19.28962 -0.000000264 -0.000000082 +19782 19.29058 -0.000000264 -0.000000082 +19783 19.29155 -0.000000264 -0.000000082 +19784 19.29251 -0.000000264 -0.000000082 +19785 19.29348 -0.000000264 -0.000000082 +19786 19.29444 -0.000000264 -0.000000082 +19787 19.29541 -0.000000264 -0.000000082 +19788 19.29637 -0.000000264 -0.000000082 +19789 19.29734 -0.000000263 -0.000000082 +19790 19.29830 -0.000000263 -0.000000082 +19791 19.29927 -0.000000263 -0.000000082 +19792 19.30024 -0.000000263 -0.000000082 +19793 19.30120 -0.000000263 -0.000000082 +19794 19.30217 -0.000000263 -0.000000082 +19795 19.30313 -0.000000263 -0.000000082 +19796 19.30410 -0.000000263 -0.000000082 +19797 19.30506 -0.000000263 -0.000000082 +19798 19.30603 -0.000000263 -0.000000082 +19799 19.30699 -0.000000263 -0.000000082 +19800 19.30796 -0.000000263 -0.000000082 +19801 19.30892 -0.000000263 -0.000000082 +19802 19.30989 -0.000000262 -0.000000082 +19803 19.31085 -0.000000262 -0.000000082 +19804 19.31182 -0.000000262 -0.000000081 +19805 19.31278 -0.000000262 -0.000000081 +19806 19.31375 -0.000000262 -0.000000081 +19807 19.31471 -0.000000262 -0.000000081 +19808 19.31568 -0.000000262 -0.000000081 +19809 19.31664 -0.000000262 -0.000000081 +19810 19.31761 -0.000000262 -0.000000081 +19811 19.31857 -0.000000262 -0.000000081 +19812 19.31954 -0.000000262 -0.000000081 +19813 19.32050 -0.000000262 -0.000000081 +19814 19.32147 -0.000000261 -0.000000081 +19815 19.32243 -0.000000261 -0.000000081 +19816 19.32340 -0.000000261 -0.000000081 +19817 19.32436 -0.000000261 -0.000000081 +19818 19.32533 -0.000000261 -0.000000081 +19819 19.32629 -0.000000261 -0.000000081 +19820 19.32726 -0.000000261 -0.000000081 +19821 19.32822 -0.000000261 -0.000000081 +19822 19.32919 -0.000000261 -0.000000081 +19823 19.33015 -0.000000261 -0.000000081 +19824 19.33112 -0.000000261 -0.000000081 +19825 19.33208 -0.000000261 -0.000000081 +19826 19.33305 -0.000000261 -0.000000081 +19827 19.33401 -0.000000260 -0.000000081 +19828 19.33498 -0.000000260 -0.000000081 +19829 19.33594 -0.000000260 -0.000000081 +19830 19.33691 -0.000000260 -0.000000081 +19831 19.33787 -0.000000260 -0.000000081 +19832 19.33884 -0.000000260 -0.000000081 +19833 19.33980 -0.000000260 -0.000000081 +19834 19.34077 -0.000000260 -0.000000081 +19835 19.34173 -0.000000260 -0.000000081 +19836 19.34270 -0.000000260 -0.000000081 +19837 19.34366 -0.000000260 -0.000000081 +19838 19.34463 -0.000000260 -0.000000081 +19839 19.34559 -0.000000260 -0.000000080 +19840 19.34656 -0.000000259 -0.000000080 +19841 19.34752 -0.000000259 -0.000000080 +19842 19.34849 -0.000000259 -0.000000080 +19843 19.34945 -0.000000259 -0.000000080 +19844 19.35042 -0.000000259 -0.000000080 +19845 19.35138 -0.000000259 -0.000000080 +19846 19.35235 -0.000000259 -0.000000080 +19847 19.35331 -0.000000259 -0.000000080 +19848 19.35428 -0.000000259 -0.000000080 +19849 19.35524 -0.000000259 -0.000000080 +19850 19.35621 -0.000000259 -0.000000080 +19851 19.35717 -0.000000259 -0.000000080 +19852 19.35814 -0.000000259 -0.000000080 +19853 19.35910 -0.000000258 -0.000000080 +19854 19.36007 -0.000000258 -0.000000080 +19855 19.36103 -0.000000258 -0.000000080 +19856 19.36200 -0.000000258 -0.000000080 +19857 19.36296 -0.000000258 -0.000000080 +19858 19.36393 -0.000000258 -0.000000080 +19859 19.36489 -0.000000258 -0.000000080 +19860 19.36586 -0.000000258 -0.000000080 +19861 19.36682 -0.000000258 -0.000000080 +19862 19.36779 -0.000000258 -0.000000080 +19863 19.36875 -0.000000258 -0.000000080 +19864 19.36972 -0.000000258 -0.000000080 +19865 19.37068 -0.000000258 -0.000000080 +19866 19.37165 -0.000000257 -0.000000080 +19867 19.37261 -0.000000257 -0.000000080 +19868 19.37358 -0.000000257 -0.000000080 +19869 19.37454 -0.000000257 -0.000000080 +19870 19.37551 -0.000000257 -0.000000080 +19871 19.37647 -0.000000257 -0.000000080 +19872 19.37744 -0.000000257 -0.000000080 +19873 19.37840 -0.000000257 -0.000000080 +19874 19.37937 -0.000000257 -0.000000080 +19875 19.38033 -0.000000257 -0.000000079 +19876 19.38130 -0.000000257 -0.000000079 +19877 19.38226 -0.000000257 -0.000000079 +19878 19.38323 -0.000000257 -0.000000079 +19879 19.38419 -0.000000256 -0.000000079 +19880 19.38516 -0.000000256 -0.000000079 +19881 19.38612 -0.000000256 -0.000000079 +19882 19.38709 -0.000000256 -0.000000079 +19883 19.38805 -0.000000256 -0.000000079 +19884 19.38902 -0.000000256 -0.000000079 +19885 19.38998 -0.000000256 -0.000000079 +19886 19.39095 -0.000000256 -0.000000079 +19887 19.39191 -0.000000256 -0.000000079 +19888 19.39288 -0.000000256 -0.000000079 +19889 19.39384 -0.000000256 -0.000000079 +19890 19.39481 -0.000000256 -0.000000079 +19891 19.39577 -0.000000256 -0.000000079 +19892 19.39674 -0.000000255 -0.000000079 +19893 19.39770 -0.000000255 -0.000000079 +19894 19.39867 -0.000000255 -0.000000079 +19895 19.39963 -0.000000255 -0.000000079 +19896 19.40060 -0.000000255 -0.000000079 +19897 19.40157 -0.000000255 -0.000000079 +19898 19.40253 -0.000000255 -0.000000079 +19899 19.40350 -0.000000255 -0.000000079 +19900 19.40446 -0.000000255 -0.000000079 +19901 19.40543 -0.000000255 -0.000000079 +19902 19.40639 -0.000000255 -0.000000079 +19903 19.40736 -0.000000255 -0.000000079 +19904 19.40832 -0.000000255 -0.000000079 +19905 19.40929 -0.000000254 -0.000000079 +19906 19.41025 -0.000000254 -0.000000079 +19907 19.41122 -0.000000254 -0.000000079 +19908 19.41218 -0.000000254 -0.000000079 +19909 19.41315 -0.000000254 -0.000000079 +19910 19.41411 -0.000000254 -0.000000079 +19911 19.41508 -0.000000254 -0.000000078 +19912 19.41604 -0.000000254 -0.000000078 +19913 19.41701 -0.000000254 -0.000000078 +19914 19.41797 -0.000000254 -0.000000078 +19915 19.41894 -0.000000254 -0.000000078 +19916 19.41990 -0.000000254 -0.000000078 +19917 19.42087 -0.000000254 -0.000000078 +19918 19.42183 -0.000000253 -0.000000078 +19919 19.42280 -0.000000253 -0.000000078 +19920 19.42376 -0.000000253 -0.000000078 +19921 19.42473 -0.000000253 -0.000000078 +19922 19.42569 -0.000000253 -0.000000078 +19923 19.42666 -0.000000253 -0.000000078 +19924 19.42762 -0.000000253 -0.000000078 +19925 19.42859 -0.000000253 -0.000000078 +19926 19.42955 -0.000000253 -0.000000078 +19927 19.43052 -0.000000253 -0.000000078 +19928 19.43148 -0.000000253 -0.000000078 +19929 19.43245 -0.000000253 -0.000000078 +19930 19.43341 -0.000000253 -0.000000078 +19931 19.43438 -0.000000252 -0.000000078 +19932 19.43534 -0.000000252 -0.000000078 +19933 19.43631 -0.000000252 -0.000000078 +19934 19.43727 -0.000000252 -0.000000078 +19935 19.43824 -0.000000252 -0.000000078 +19936 19.43920 -0.000000252 -0.000000078 +19937 19.44017 -0.000000252 -0.000000078 +19938 19.44113 -0.000000252 -0.000000078 +19939 19.44210 -0.000000252 -0.000000078 +19940 19.44306 -0.000000252 -0.000000078 +19941 19.44403 -0.000000252 -0.000000078 +19942 19.44499 -0.000000252 -0.000000078 +19943 19.44596 -0.000000252 -0.000000078 +19944 19.44692 -0.000000252 -0.000000078 +19945 19.44789 -0.000000251 -0.000000078 +19946 19.44885 -0.000000251 -0.000000078 +19947 19.44982 -0.000000251 -0.000000078 +19948 19.45078 -0.000000251 -0.000000077 +19949 19.45175 -0.000000251 -0.000000077 +19950 19.45271 -0.000000251 -0.000000077 +19951 19.45368 -0.000000251 -0.000000077 +19952 19.45464 -0.000000251 -0.000000077 +19953 19.45561 -0.000000251 -0.000000077 +19954 19.45657 -0.000000251 -0.000000077 +19955 19.45754 -0.000000251 -0.000000077 +19956 19.45850 -0.000000251 -0.000000077 +19957 19.45947 -0.000000251 -0.000000077 +19958 19.46043 -0.000000250 -0.000000077 +19959 19.46140 -0.000000250 -0.000000077 +19960 19.46236 -0.000000250 -0.000000077 +19961 19.46333 -0.000000250 -0.000000077 +19962 19.46429 -0.000000250 -0.000000077 +19963 19.46526 -0.000000250 -0.000000077 +19964 19.46622 -0.000000250 -0.000000077 +19965 19.46719 -0.000000250 -0.000000077 +19966 19.46815 -0.000000250 -0.000000077 +19967 19.46912 -0.000000250 -0.000000077 +19968 19.47008 -0.000000250 -0.000000077 +19969 19.47105 -0.000000250 -0.000000077 +19970 19.47201 -0.000000250 -0.000000077 +19971 19.47298 -0.000000250 -0.000000077 +19972 19.47394 -0.000000249 -0.000000077 +19973 19.47491 -0.000000249 -0.000000077 +19974 19.47587 -0.000000249 -0.000000077 +19975 19.47684 -0.000000249 -0.000000077 +19976 19.47780 -0.000000249 -0.000000077 +19977 19.47877 -0.000000249 -0.000000077 +19978 19.47973 -0.000000249 -0.000000077 +19979 19.48070 -0.000000249 -0.000000077 +19980 19.48166 -0.000000249 -0.000000077 +19981 19.48263 -0.000000249 -0.000000077 +19982 19.48359 -0.000000249 -0.000000077 +19983 19.48456 -0.000000249 -0.000000077 +19984 19.48552 -0.000000249 -0.000000077 +19985 19.48649 -0.000000248 -0.000000077 +19986 19.48745 -0.000000248 -0.000000076 +19987 19.48842 -0.000000248 -0.000000076 +19988 19.48938 -0.000000248 -0.000000076 +19989 19.49035 -0.000000248 -0.000000076 +19990 19.49131 -0.000000248 -0.000000076 +19991 19.49228 -0.000000248 -0.000000076 +19992 19.49324 -0.000000248 -0.000000076 +19993 19.49421 -0.000000248 -0.000000076 +19994 19.49517 -0.000000248 -0.000000076 +19995 19.49614 -0.000000248 -0.000000076 +19996 19.49710 -0.000000248 -0.000000076 +19997 19.49807 -0.000000248 -0.000000076 +19998 19.49903 -0.000000248 -0.000000076 +19999 19.50000 -0.000000247 -0.000000076 diff --git a/examples/PACKAGES/imd/deca-ala-solv_imd_v3.py b/examples/PACKAGES/imd/deca-ala-solv_imd_v3.py new file mode 100644 index 0000000000..176e15e811 --- /dev/null +++ b/examples/PACKAGES/imd/deca-ala-solv_imd_v3.py @@ -0,0 +1,13 @@ +""" +For use with 'in.deca-ala-solv_imd_v3'. + +Tested with imdclient v0.1.4 and MDAnalysis v2.8.0 +""" +from imdclient.IMD import IMDReader +import MDAnalysis as mda + +u = mda.Universe('data.deca-ala-solv', "imd://localhost:5678", topology_format='DATA') + +for ts in u.trajectory: + print(ts.time) + print(ts.velocities) \ No newline at end of file diff --git a/examples/PACKAGES/imd/in.deca-ala-solv_imd_v3 b/examples/PACKAGES/imd/in.deca-ala-solv_imd_v3 new file mode 100644 index 0000000000..a7e8244cdc --- /dev/null +++ b/examples/PACKAGES/imd/in.deca-ala-solv_imd_v3 @@ -0,0 +1,31 @@ +# +units real +neighbor 2.5 bin +neigh_modify delay 1 every 1 + +atom_style full +bond_style harmonic +angle_style charmm +dihedral_style charmm +improper_style harmonic + +pair_style lj/charmm/coul/long 8 10 +pair_modify mix arithmetic +special_bonds charmm +read_data data.deca-ala-solv + + +group peptide id <= 103 +fix rigidh all shake 1e-6 100 1000 t 1 2 3 4 5 a 23 + +thermo 100 +thermo_style multi +timestep 2.0 +kspace_style pppm 1e-5 + +fix ensemble all npt temp 300.0 300.0 100.0 iso 1.0 1.0 1000.0 drag 0.2 + +# IMD setup. Client code available in 'deca-ala-solv_imd_v3.py' +fix comm all imd 5678 unwrap on trate 10 version 3 time on box on coordinates on velocities on forces off + +run 5000000 \ No newline at end of file diff --git a/examples/PACKAGES/stressprofile/in.flat b/examples/PACKAGES/stressprofile/in.flat index 8b484a423f..a0aea07d83 100644 --- a/examples/PACKAGES/stressprofile/in.flat +++ b/examples/PACKAGES/stressprofile/in.flat @@ -32,7 +32,7 @@ fix 1 all nvt temp 0.7 0.7 0.2 #dump_modify 3 pad 3 fix 2 all recenter NULL NULL 15 units lattice -compute p1 all stress/cartesian z 0.5 +compute p1 all stress/cartesian z 0.5 NULL 0 fix 3 all ave/time 100 1 100 c_p1[*] file flat.out mode vector thermo 50 diff --git a/examples/README b/examples/README index 90831b49f0..633fbbc49d 100644 --- a/examples/README +++ b/examples/README @@ -112,6 +112,7 @@ snap: examples for using several bundled SNAP potentials srd: stochastic rotation dynamics (SRD) particles as solvent steinhardt: Steinhardt-Nelson Q_l and W_l parameters usng orientorder/atom streitz: Streitz-Mintmire potential for Al2O3 +stress_vcm: removing binned rigid body motion from binned stress profile tad: temperature-accelerated dynamics of vacancy diffusion in bulk Si template: examples for using atom_style template and comparing to atom style molecular tersoff: regression test input for Tersoff variants diff --git a/examples/comb/README b/examples/comb/README index 0fdacfaef0..cb0ae368a3 100644 --- a/examples/comb/README +++ b/examples/comb/README @@ -20,5 +20,3 @@ Examples: 4. in.comb.Cu2O.elastic: Cu2O crystal, qeq on, minimizes, then calculates elastic constants 5. in.comb.HfO2: HfO2 polymorphs: Monoclinic HfO2 NVT @ 300K -6. in.comb.CuaS: Metallic Cu and amorphous silica interface, qeq on, - five step NVE run diff --git a/examples/comb/data.CuaS b/examples/comb/data.CuaS deleted file mode 100644 index 6039bce56a..0000000000 --- a/examples/comb/data.CuaS +++ /dev/null @@ -1,4147 +0,0 @@ -Transfromed from dump file 240089.dump - - 4126 atoms - 0 bonds - 0 angles - 0 dihedrals - 0 impropers - - 3 atom types - 0 bond types - 0 angle types - 0 dihedral types - 0 improper types - - 0.000000000000000E+000 60.4000000000000 xlo xhi - 0.000000000000000E+000 29.7600000000000 ylo yhi - 0.000000000000000E+000 29.7600000000000 zlo zhi - 0.0000000 0.0000000 0.0000000 xy xz yz - - Atoms - - 1 2 -1.5368900 12.1717000 0.4080250 5.4646100 - 2 2 -1.7044300 0.5602200 14.3770000 13.8990000 - 3 2 -1.6215100 1.5055000 13.5219000 0.5516700 - 4 2 -1.4700300 8.9731600 0.9027140 8.1504600 - 5 1 2.7243100 4.1740200 7.9774300 1.3186600 - 6 2 -1.6113900 0.1464830 6.3396600 2.1301600 - 7 2 -1.6120200 4.2918800 2.1859800 5.1092400 - 8 1 2.1983400 1.2837900 7.3036900 1.5887700 - 9 2 -1.5718600 3.8048100 9.3568400 2.2242400 - 10 1 2.7279600 2.9510000 1.3594100 5.1666200 - 11 1 2.5423100 2.2505500 5.7225300 7.2319200 - 12 2 -1.5870300 2.3555800 1.3088300 8.4835200 - 13 1 2.5689100 2.1342400 8.6397300 5.7452200 - 14 2 -1.5394400 1.8088100 1.8138600 6.1646100 - 15 2 -1.6206300 3.5163700 0.3456770 10.4105000 - 16 1 2.6216800 9.2633800 7.3310300 13.9759000 - 17 2 -1.6309000 1.5601700 7.0550300 6.5198000 - 18 2 -1.6429500 1.5821400 4.2357700 7.6013900 - 19 2 -1.6247000 4.1661600 6.8881800 2.6601800 - 20 1 1.9449200 1.8836900 4.8823200 13.0377000 - 21 2 -1.6140300 2.2967900 6.2024200 9.0152700 - 22 1 2.6742600 12.2449000 14.8469000 2.5420300 - 23 2 -1.6627500 2.3707500 4.6051100 0.2986830 - 24 1 2.6864100 2.0339300 14.5581000 1.9695500 - 25 2 -1.6762200 1.7991000 8.8757800 4.1359600 - 26 1 2.6768000 2.5507500 10.0126000 2.9292800 - 27 1 2.4694900 2.2285600 13.7702000 7.0218500 - 28 2 -1.5807100 2.8132600 11.3877000 3.6897800 - 29 2 -1.5099300 3.7249900 12.3879000 14.0392000 - 30 2 -1.5816800 1.8983000 12.7376000 5.9525800 - 31 2 -1.6086100 3.2093400 10.3221000 9.0758300 - 32 2 -1.6655100 4.0574200 14.5371000 1.9830700 - 33 1 2.6472900 1.8625600 10.8911000 8.3730800 - 34 2 -1.5178900 7.6693000 2.1612200 0.8724280 - 35 2 -1.6151200 2.2094700 12.7205000 8.4444300 - 36 2 -1.6518800 0.4143860 7.3627400 14.1016000 - 37 2 -1.6662100 2.5940700 7.8480100 12.4540000 - 38 2 -1.6741800 0.3553690 10.2579000 9.6243400 - 39 2 -1.6703800 1.5515000 9.8580700 6.7018200 - 40 1 2.6446300 2.0765800 7.7415800 13.9914000 - 41 2 -1.5977400 2.7757300 0.5925440 12.5758000 - 42 2 -1.6446300 2.3687000 2.5209700 2.1816000 - 43 2 -1.6750000 1.3843200 10.3343000 1.2599200 - 44 1 2.7888000 7.7002100 6.3545400 2.5116900 - 45 1 2.7490000 6.4620300 2.1743400 1.8792400 - 46 2 -1.5838500 6.3979500 5.6547400 3.4494500 - 47 2 -1.5493100 9.1658400 0.9823320 3.3657800 - 48 2 -1.6336500 5.0991900 2.9611800 1.2105700 - 49 2 -1.4921000 7.2558600 7.5711600 1.6101500 - 50 1 2.7054700 14.4620000 0.5135390 7.6389500 - 51 1 2.6094100 6.4779600 4.1891100 4.1353100 - 52 1 2.7489600 5.1488000 6.9800400 3.9667000 - 53 1 2.7624100 5.8350600 5.6903700 9.6098800 - 54 1 2.5873400 4.9592900 3.2757900 6.2399100 - 55 1 2.7266800 2.2797700 2.6514800 7.5277900 - 56 1 2.7596600 5.4385500 7.0669700 6.9279300 - 57 2 -1.5608000 5.9936900 4.1853700 5.5884900 - 58 2 -1.6286000 4.5321500 6.6802400 5.4623900 - 59 2 -1.5741200 4.3569700 5.5828600 10.6531000 - 60 2 -1.5862400 6.3830000 1.8197500 8.4997700 - 61 2 -1.5883300 3.9748600 3.1741600 7.4861300 - 62 2 -1.5999200 5.3115000 5.8086900 8.0241300 - 63 2 -1.5464900 6.4841900 4.1916500 9.9214500 - 64 2 -1.5789600 7.1244000 2.9173300 3.3670500 - 65 2 -1.5088200 8.8403700 6.0606300 3.6249700 - 66 1 2.6902000 6.0902300 2.6988600 9.7338100 - 67 1 2.6040100 2.9574800 4.9176500 10.2503000 - 68 1 2.6778400 4.2006900 1.1910800 14.0163000 - 69 1 2.6410200 3.7077800 2.0774200 11.4335000 - 70 1 2.7874700 6.8496500 8.6557600 12.7521000 - 71 2 -1.6058500 9.9653800 5.4060600 10.3955000 - 72 2 -1.6139000 6.8893800 9.4187700 14.3385000 - 73 2 -1.6362800 1.6014200 5.0762200 11.4931000 - 74 2 -1.5882400 7.8325600 0.5264250 13.0271000 - 75 2 -1.6724400 3.9081700 2.5956400 13.0412000 - 76 2 -1.5850800 8.0393700 3.1640700 13.3327000 - 77 2 -1.5992200 3.0642100 3.1978500 10.4224000 - 78 2 -1.6198400 5.9791700 1.1844700 14.0119000 - 79 2 -1.6455400 5.6751600 2.1447000 11.2320000 - 80 2 -1.5754000 7.1647700 6.7204200 9.5814100 - 81 2 -1.6335000 9.9751300 10.1907000 14.7464000 - 82 1 2.7219600 7.6690000 14.0632000 2.1677400 - 83 1 2.7641400 7.0129500 11.4214000 1.9916300 - 84 2 -1.6092500 6.3324400 13.1465000 2.9286500 - 85 2 -1.5149300 8.8281200 8.8098600 4.4233700 - 86 2 -1.5781400 5.5681300 11.1632000 1.2556600 - 87 2 -1.6165800 7.4924300 13.3110000 0.7779300 - 88 1 2.8128700 7.3151800 8.8232900 5.2334800 - 89 1 2.6467300 4.9924100 13.8094000 3.5363100 - 90 1 2.7901600 5.7659300 13.8746000 8.6313300 - 91 1 2.7487700 4.6872700 9.3713300 8.9627000 - 92 1 2.7136600 3.2414800 11.9545000 5.1831600 - 93 2 -1.5421400 6.9145000 10.3031000 5.9506400 - 94 2 -1.5826500 4.4641400 13.4897000 9.6881200 - 95 2 -1.5308200 8.7115300 10.5203000 9.3858100 - 96 1 2.7645600 5.9955000 11.0282000 7.1062700 - 97 2 -1.5734900 6.2861500 12.5678000 7.6270000 - 98 2 -1.6414800 4.4094300 8.2374200 7.7122700 - 99 2 -1.5537900 4.2599600 13.2214000 4.7561000 - 100 2 -1.5937900 6.9187300 7.6328700 6.3924400 - 101 2 -1.5512000 5.9422000 8.3640800 4.1922300 - 102 2 -1.5430300 8.6484900 12.9035000 2.6840700 - 103 1 2.7916900 7.2640200 10.5693000 10.0173000 - 104 1 2.7013600 2.9464200 13.3433000 9.8916900 - 105 1 2.8097300 6.4981000 14.2301000 14.0513000 - 106 1 2.7766000 4.1749100 8.6872100 11.9668000 - 107 2 -1.5356700 1.7627000 14.0822000 10.4319000 - 108 2 -1.5087300 7.0703500 11.9777000 10.6920000 - 109 2 -1.5980300 10.9557000 14.7051000 9.4617700 - 110 2 -1.5570300 4.6421500 8.5688500 10.3544000 - 111 2 -1.5090800 6.5710800 13.0739000 12.9852000 - 112 2 -1.5068300 5.3773800 8.0647600 12.7595000 - 113 2 -1.5825400 7.0671400 9.5130100 11.3206000 - 114 2 -1.5961600 5.9933500 10.2320000 8.6328900 - 115 2 -1.5755900 6.9927500 14.7127000 9.3744800 - 116 1 2.8669700 3.2710500 10.9112000 13.9472000 - 117 2 -1.6175300 1.5791300 10.7165000 14.3736000 - 118 2 -1.5700000 3.6285500 10.2433000 12.3416000 - 119 2 -1.5693200 8.1681900 14.4178000 14.3170000 - 120 2 -1.5973600 12.5358000 0.6572410 14.1779000 - 121 2 -1.4875400 13.2459000 3.8594500 1.1080300 - 122 2 -1.5990800 1.1617000 14.2489000 3.3626300 - 123 2 -1.6374500 2.4118000 7.7822800 0.6301080 - 124 1 2.7659700 12.4387000 4.9965700 1.6468800 - 125 1 2.7626200 10.1360000 0.5699560 2.0559400 - 126 1 2.8130600 11.3061000 7.7493200 0.7889240 - 127 2 -1.5661100 10.8884000 4.9003300 2.1287500 - 128 2 -1.6372700 12.6527000 6.6792800 1.1302500 - 129 1 2.7234600 12.1213000 1.4373900 4.4121000 - 130 1 2.8080500 10.1852000 5.1692700 3.6313400 - 131 1 2.7410700 11.0862000 5.8083700 7.3848300 - 132 1 2.7997500 8.0760900 2.0665400 7.4370800 - 133 1 2.7863800 8.4966400 2.2679700 4.1117400 - 134 1 2.7582700 11.7062000 7.4540800 4.9182900 - 135 2 -1.5405100 13.4678000 3.4918400 7.5684400 - 136 2 -1.5308100 11.2965000 6.0646700 4.3396400 - 137 2 -1.5371600 9.5667300 6.1545400 7.8589500 - 138 2 -1.5770500 10.4244000 0.3483110 10.3436000 - 139 2 -1.5561400 7.6745100 1.8618200 5.7969300 - 140 2 -1.5895000 11.6612000 7.1977000 6.5792800 - 141 2 -1.5507400 11.4353000 4.4774000 6.5051800 - 142 2 -1.5778000 9.6190600 3.5142700 4.1963200 - 143 2 -1.6151200 11.8330000 0.8197470 2.8983400 - 144 2 -1.5673200 13.8801000 6.1965800 3.0215300 - 145 1 2.7272700 9.0576100 0.7419120 9.7121100 - 146 1 2.7451100 8.7255800 6.7315300 9.1452700 - 147 1 2.6000800 12.0961000 6.8994000 14.7773000 - 148 1 2.6001500 9.3190800 0.2738610 13.7808000 - 149 1 2.6050300 6.9547800 2.0165000 12.8077000 - 150 1 2.6574100 11.6190000 8.5534400 12.1486000 - 151 2 -1.6670600 8.9886900 2.4953500 10.6017000 - 152 1 2.5616600 11.5447000 5.9290800 10.3477000 - 153 2 -1.6477600 0.8376510 0.5059940 14.7868000 - 154 1 2.5830400 9.8430200 3.3333800 13.8908000 - 155 2 -1.6355900 10.9809000 7.4250200 13.6612000 - 156 2 -1.6787200 10.7759000 3.7792300 12.3687000 - 157 2 -1.5737400 8.1648000 7.4863200 12.9271000 - 158 2 -1.5058100 8.6226400 3.4322500 7.8319600 - 159 2 -1.6366000 12.1706000 5.4749400 8.9199000 - 160 2 -1.6212600 10.1999000 1.6733500 14.2537000 - 161 1 2.3217100 1.5050700 0.0351454 13.4107000 - 162 1 2.8338800 9.4287700 9.4571400 2.7839600 - 163 2 -1.6362600 10.9324000 13.5974000 2.8907300 - 164 2 -1.5618300 13.4017000 10.9431000 3.0234600 - 165 2 -1.5384900 7.9200500 10.2421000 2.4141200 - 166 1 2.8706200 10.5114000 12.3050000 1.3282900 - 167 2 -1.5445600 10.3790000 8.1598900 2.2393300 - 168 3 0.2986670 0.7934790 5.6953000 3.9373400 - 169 1 2.7909000 11.9192000 10.4505000 3.6510300 - 170 1 2.7520100 9.2319500 13.1807000 4.2384000 - 171 1 2.6792300 10.1934000 14.3089000 8.1655000 - 172 1 2.8102700 10.0113000 9.6948700 8.9792500 - 173 1 2.7874300 10.3880000 11.3498000 6.4376800 - 174 2 -1.5532200 11.8791000 11.1586000 5.2039100 - 175 2 -1.5169000 14.3426000 9.9337000 5.8703000 - 176 2 -1.5976300 10.1750000 12.8885000 7.4913700 - 177 2 -1.5735300 12.7160000 11.2426000 7.4433200 - 178 2 -1.5803100 9.9846500 9.9866000 7.2243900 - 179 2 -1.5465900 9.4101900 11.9226000 5.1351300 - 180 2 -1.5914200 12.2179000 8.7346800 4.1541400 - 181 2 -1.5813200 10.5652000 10.7222000 2.7662000 - 182 2 -1.5833800 13.6582000 14.1980000 1.5406900 - 183 1 2.7677000 10.8067000 13.3524000 12.4986000 - 184 1 2.6910000 9.6771600 10.3446000 13.1204000 - 185 1 2.8157700 7.7708600 12.4703000 12.0076000 - 186 2 -1.5693900 11.6032000 10.1032000 9.5255000 - 187 2 -1.6012900 10.6966000 13.2393000 14.1306000 - 188 2 -1.5473000 8.4666300 11.2399000 12.9084000 - 189 2 -1.6288300 11.8516000 12.1087000 11.8800000 - 190 2 -1.5540600 10.3530000 9.4628600 12.0815000 - 191 2 -1.5874800 9.2734400 13.3336000 11.7899000 - 192 2 -1.6339900 12.0455000 7.1314800 11.2333000 - 193 2 -1.5750700 9.5784400 8.0977400 9.5126200 - 194 2 -1.6454100 4.2124400 11.0442000 6.3274200 - 195 2 -1.5061000 11.6320000 9.3739100 0.0846117 - 196 1 2.7022500 14.4093000 7.3595000 1.9734600 - 197 1 2.6937100 14.8574000 5.5225800 4.3371500 - 198 1 2.7201800 12.4453000 0.9569840 10.1321000 - 199 1 2.7579800 12.9030000 3.8690100 6.0356600 - 200 2 -1.4290000 14.6134000 0.1584970 12.2093000 - 201 2 -1.5602100 13.9891000 5.0653000 5.5268800 - 202 2 -1.5840200 12.8061000 2.8537700 4.6820600 - 203 2 -1.6150900 13.1820000 0.6596580 8.6583100 - 204 1 2.5939800 12.6880000 3.4014500 12.0103000 - 205 1 2.6619800 13.6672000 4.6000300 8.6492000 - 206 1 2.8199600 14.0064000 0.6232000 13.5290000 - 207 1 2.5395700 9.6342400 3.9429200 10.8428000 - 208 2 -1.6557400 14.1541000 4.4577200 11.6615000 - 209 2 -1.5717400 13.9016000 2.5088700 12.7325000 - 210 2 -1.6468200 12.3445000 2.5864700 10.5520000 - 211 1 2.6670300 3.7190500 0.3329020 8.7703000 - 212 1 2.8093000 14.7839000 10.4800000 2.3165100 - 213 2 -1.5580000 6.5597400 0.5230030 2.4634700 - 214 2 -1.5390600 12.0866000 12.2895000 0.6829230 - 215 1 2.7709100 13.0826000 10.2806000 8.8724400 - 216 1 2.8038700 13.5277000 11.2828000 6.0795800 - 217 2 -1.5658200 13.9303000 9.0285000 8.2486100 - 218 2 -1.5733100 14.2778000 12.6307000 5.4219500 - 219 1 2.7488600 14.1453000 8.5047600 13.7188000 - 220 1 2.6955300 13.2574000 11.1457000 11.9598000 - 221 2 -1.6059700 12.9394000 9.4642100 12.6929000 - 222 2 -1.5723000 14.4068000 7.1699200 12.8234000 - 223 2 -1.6284400 14.2422000 12.0057000 13.1381000 - 224 2 -1.6312600 14.0110000 10.5590000 10.5599000 - 225 2 -1.5740300 14.9723000 5.3602000 8.8375800 - 226 2 -1.4287300 10.1562000 1.5055000 0.9203180 - 227 1 2.6770800 14.7868000 12.1337000 14.6510000 - 228 2 -1.5566800 14.3111000 8.8759600 1.4543600 - 229 1 2.4168700 1.3727200 3.2829600 3.2063700 - 230 2 -1.5916800 1.9552000 4.1794500 4.5401000 - 231 3 0.4197030 0.0086996 0.5612590 3.0259000 - 232 2 -1.5019000 8.1396000 5.7628100 1.0924900 - 233 1 2.0279400 1.8831900 7.6176700 9.5151400 - 234 3 0.3339330 0.8838550 9.0260400 8.3090800 - 235 3 0.3818910 0.3576280 2.0133300 4.6435100 - 236 2 -1.4733900 2.6363300 0.0816984 4.5217500 - 237 2 -1.6287000 15.0252000 1.4519500 6.1427400 - 238 2 -1.6457800 8.3225400 14.3493000 5.1547200 - 239 1 2.7591100 8.0673900 7.3147100 0.1367910 - 240 2 -1.5453000 9.9438200 13.5475000 0.2081590 - 241 1 2.7625500 8.8960600 1.1565600 0.1659920 - 242 1 2.7241900 3.2414800 3.1413400 0.8089070 - 243 3 0.2917570 0.0559674 4.2838200 2.3214300 - 244 2 -1.5552000 14.9432000 12.7378000 8.8501500 - 245 1 2.7191700 14.9454000 14.3011000 8.5981400 - 246 1 2.6251400 4.6669000 11.5090000 0.0678038 - 247 2 -1.5350000 9.5766700 10.9610000 0.0091379 - 248 1 2.8157400 15.0954000 8.8716200 6.9277600 - 249 1 2.5957200 4.6166500 14.5152000 19.0436000 - 250 2 -1.5819100 3.4635900 1.3201700 20.4201000 - 251 2 -1.5327700 8.6026000 0.5245280 18.8369000 - 252 2 -1.6350500 1.2652800 6.0073600 24.0408000 - 253 2 -1.6100300 1.3348100 1.2712900 22.4975000 - 254 2 -1.5790500 2.7330000 4.6706400 19.7101000 - 255 2 -1.5460600 3.3438200 1.4783800 15.4007000 - 256 1 2.0067700 0.6113760 0.6373720 16.8148000 - 257 1 2.6927700 1.7232200 5.5309100 18.9403000 - 258 2 -1.5742700 4.2893500 2.4242700 17.8681000 - 259 1 2.5249600 1.2691400 14.3796000 23.2498000 - 260 2 -1.5478900 2.1401400 0.9588500 17.1304000 - 261 2 -1.5146600 4.4799400 5.8969600 16.8919000 - 262 1 2.5574500 1.3645100 8.0208300 20.5824000 - 263 2 -1.6164100 2.3785700 11.8027000 25.0276000 - 264 2 -1.6771800 1.1279900 13.4405000 21.7923000 - 265 2 -1.5987600 3.1014200 14.4126000 18.6125000 - 266 1 2.5564200 2.1499200 12.0135000 21.0229000 - 267 2 -1.5741100 1.6708700 13.0296000 17.1055000 - 268 2 -1.6007100 1.4894700 9.2089400 19.5995000 - 269 2 -1.6607400 1.7095500 12.3128000 19.4964000 - 270 2 -1.6376100 0.6649770 13.7103000 24.6046000 - 271 1 2.5772400 1.8773700 13.1136000 25.9739000 - 272 1 2.7146900 14.1947000 4.7898800 25.2917000 - 273 1 2.4278900 1.5148600 13.9539000 18.8224000 - 274 1 2.7216000 7.4065500 0.2619710 20.2254000 - 275 1 2.7610500 3.7017100 4.7225400 21.2800000 - 276 1 2.8203700 7.0866200 5.1614700 23.8225000 - 277 1 2.5027300 2.0195900 0.7322930 23.9251000 - 278 1 2.6044900 2.4620300 1.9822600 21.4678000 - 279 1 2.7822700 5.8720900 7.9673600 22.9649000 - 280 2 -1.5429200 7.2836300 1.6577300 21.0197000 - 281 2 -1.5836300 3.2719300 6.1723900 22.1565000 - 282 2 -1.5447500 6.7035000 6.5852700 23.0372000 - 283 2 -1.5998500 6.8134300 3.5632600 23.1632000 - 284 2 -1.6080400 2.7366600 3.4141100 22.0539000 - 285 2 -1.5276500 5.9065500 0.4994620 19.3826000 - 286 2 -1.5064600 7.7988800 5.4731400 20.6378000 - 287 1 2.8098700 5.7047000 9.6118600 15.5901000 - 288 1 2.6768800 6.3850200 1.6319000 15.5103000 - 289 1 2.7120200 3.0627800 5.8348800 16.2962000 - 290 1 2.8189800 6.4272900 4.9959600 19.8737000 - 291 1 2.7058300 4.7046200 1.4888100 19.2460000 - 292 1 2.6795900 3.2052000 2.5255100 16.6338000 - 293 2 -1.5661800 2.5762600 11.1211000 22.1606000 - 294 2 -1.5749200 6.5235300 3.1111300 15.8983000 - 295 2 -1.5873100 5.1639900 4.3516300 20.9169000 - 296 2 -1.5874300 5.7742900 8.3700000 16.8093000 - 297 2 -1.5437800 5.7615900 6.4423000 19.0828000 - 298 2 -1.6106800 6.7083600 3.7539700 18.7682000 - 299 2 -1.6611700 2.5998600 4.0546700 15.8729000 - 300 2 -1.6177300 9.0946700 6.8989700 15.4528000 - 301 1 2.7890400 5.5567500 9.7356100 20.3182000 - 302 1 2.6477700 5.6899600 13.4687000 22.8632000 - 303 1 2.7338300 4.1732400 9.1371900 25.1647000 - 304 1 2.6805400 3.3622400 11.6126000 23.7185000 - 305 2 -1.5403800 6.0332300 10.9066000 21.2524000 - 306 2 -1.6705400 4.1041300 13.1732000 23.1183000 - 307 2 -1.5584900 4.3757500 10.4547000 24.1155000 - 308 2 -1.6090800 6.9262700 12.2948000 23.1092000 - 309 2 -1.5818400 4.8216000 7.9810500 24.2488000 - 310 2 -1.5933100 2.9972700 11.3530000 29.3330000 - 311 2 -1.5794500 5.1887600 8.4866500 21.4836000 - 312 2 -1.6094700 4.2052800 10.2699000 19.3469000 - 313 2 -1.5687300 7.9799200 12.3881000 19.5290000 - 314 1 2.6850600 2.6998900 14.1768000 16.2940000 - 315 1 2.7343100 5.3861300 12.6800000 16.6553000 - 316 1 2.7382300 7.0004300 13.5143000 19.1594000 - 317 1 2.7501300 3.0587200 9.4332100 18.5117000 - 318 1 2.6492200 2.5864700 11.7422000 17.1525000 - 319 2 -1.6382900 6.0926700 14.0787000 15.8178000 - 320 2 -1.6660900 5.7094000 14.1948000 20.3544000 - 321 2 -1.5699400 7.0947300 9.3818200 19.6062000 - 322 2 -1.5636200 2.4426300 10.2515000 17.2913000 - 323 2 -1.5747300 5.5725400 0.0867556 28.8017000 - 324 2 -1.5749500 5.5489700 13.1528000 18.3188000 - 325 2 -1.5681800 3.5717500 7.9156600 18.4642000 - 326 2 -1.5569900 3.7002800 12.8045000 16.6235000 - 327 2 -1.6099600 1.7395200 6.0958000 17.4120000 - 328 2 -1.6145800 3.6190600 4.9580900 24.8406000 - 329 1 2.7735400 14.5308000 0.4618240 19.9090000 - 330 1 2.8537200 11.7839000 1.6792800 20.7713000 - 331 1 2.8373600 9.3474600 4.9502800 21.1176000 - 332 1 2.6198000 13.1502000 6.6253500 23.5775000 - 333 2 -1.4704300 8.6143900 5.2261200 24.3506000 - 334 1 2.8117600 7.8992200 2.5119800 22.2894000 - 335 1 2.8106700 10.0913000 7.4717300 22.8471000 - 336 2 -1.5329200 12.4725000 2.9877800 21.5856000 - 337 2 -1.5671500 10.1331000 6.0017400 22.1480000 - 338 2 -1.5955600 10.4533000 3.1952600 24.5462000 - 339 2 -1.5454900 7.9940800 1.1716800 23.4517000 - 340 2 -1.6409200 1.2781500 12.8436000 27.6158000 - 341 2 -1.5541200 11.9338000 7.5240200 23.4260000 - 342 2 -1.5962200 9.0585500 3.4248800 21.7115000 - 343 2 -1.5381500 10.8800000 2.1541400 19.2182000 - 344 2 -1.5905400 12.6319000 6.3100400 19.7557000 - 345 2 -1.5732500 9.6163000 10.5670000 24.8617000 - 346 1 2.6974800 11.6181000 1.3122000 15.3761000 - 347 1 2.6799600 9.6005500 5.4756900 16.3801000 - 348 1 2.7689500 11.6349000 5.2031900 18.9696000 - 349 1 2.7213800 9.5019600 1.9045800 18.7499000 - 350 1 2.7332400 7.4165800 3.5565800 17.2752000 - 351 1 2.6794600 10.6588000 7.8123300 17.9003000 - 352 2 -1.5829700 12.1644000 2.6824100 15.9535000 - 353 2 -1.5613800 10.2571000 4.8395600 19.7469000 - 354 2 -1.4652600 12.9795000 0.8774730 20.0625000 - 355 2 -1.5828600 8.6935600 2.2211900 17.4263000 - 356 2 -1.6246000 10.8301000 6.2264800 17.4828000 - 357 2 -1.5362800 12.4504000 4.0880800 18.2498000 - 358 2 -1.5278000 8.3726900 4.8613500 17.1399000 - 359 1 2.7876700 11.1681000 9.7960000 20.9119000 - 360 1 2.7228500 9.6033900 12.2653000 19.8107000 - 361 1 2.7108200 12.0859000 13.9407000 24.1505000 - 362 1 2.7591900 8.4613100 9.3591900 24.5594000 - 363 1 2.6594300 7.0272000 10.8479000 22.3941000 - 364 2 -1.5321700 11.7591000 11.1199000 21.6342000 - 365 2 -1.6336300 9.5317800 13.4770000 21.3723000 - 366 2 -1.6292600 7.2377500 9.5009700 23.2228000 - 367 1 2.6502200 14.9311000 9.8786000 23.7716000 - 368 2 -1.5390100 12.3286000 12.3581000 23.8986000 - 369 2 -1.5044200 9.2164400 8.0001900 24.1516000 - 370 2 -1.5923500 9.7723400 10.5827000 19.8954000 - 371 2 -1.5771000 10.0682000 8.8411000 21.8123000 - 372 1 2.7014700 8.9927400 10.8574000 15.9968000 - 373 2 -1.6265000 13.2135000 14.0596000 19.8183000 - 374 1 2.6783700 10.3632000 14.8568000 21.5887000 - 375 1 2.7149000 13.2137000 8.9576900 16.4469000 - 376 1 2.7880900 9.3739300 13.8220000 15.1974000 - 377 1 2.7247700 11.7565000 14.3852000 18.9625000 - 378 1 2.7671400 8.6360700 9.4576000 19.0113000 - 379 2 -1.6049900 5.8953400 11.1921000 16.3966000 - 380 2 -1.6245100 13.3903000 10.6919000 15.9980000 - 381 2 -1.6056900 10.7054000 13.0256000 18.7918000 - 382 2 -1.5571500 12.3825000 9.2036800 19.9872000 - 383 2 -1.5826900 8.6492900 10.0055000 17.2465000 - 384 2 -1.5551200 9.5531600 8.1385700 18.8747000 - 385 2 -1.5953700 8.8731300 12.4232000 16.0418000 - 386 2 -1.6275200 11.6014000 8.8317000 17.0781000 - 387 1 2.8125700 5.0433100 7.1460500 17.7971000 - 388 1 2.6257600 2.0656700 4.8772000 24.8977000 - 389 1 2.7745500 14.0031000 6.1958200 20.7384000 - 390 1 2.7994200 13.6525000 3.6730800 22.4483000 - 391 2 -1.6302800 14.8707000 7.7498000 21.1479000 - 392 2 -1.6802200 14.4988000 6.5417700 24.6127000 - 393 2 -1.6183100 13.5100000 5.6233400 22.3209000 - 394 2 -1.6037500 0.8688830 6.5845200 19.9229000 - 395 1 2.7557500 14.4895000 5.4343500 15.8548000 - 396 3 0.3732260 0.6167470 4.4183500 17.6093000 - 397 1 2.7995500 13.2202000 2.9959300 17.2393000 - 398 2 -1.6387000 14.4722000 6.8527400 16.8829000 - 399 2 -1.5807100 13.5185000 1.3614000 17.8400000 - 400 2 -1.5491100 14.4357000 3.9858800 16.6377000 - 401 1 2.7170100 14.5014000 13.7422000 20.9378000 - 402 1 2.7941800 12.9445000 11.5084000 22.6497000 - 403 2 -1.5465900 13.6966000 10.1332000 23.0191000 - 404 1 2.4925800 2.0825700 9.9880700 29.5533000 - 405 2 -1.5641300 14.1883000 12.6835000 22.1211000 - 406 2 -1.5666700 14.9300000 10.5292000 20.0724000 - 407 1 2.7200100 13.5656000 14.4690000 16.2128000 - 408 1 2.7836800 13.7189000 9.9044300 19.2321000 - 409 1 2.6353200 12.9357000 11.8638000 17.0733000 - 410 2 -1.5845900 12.9962000 11.4552000 18.5137000 - 411 2 -1.6053800 14.2510000 8.9981200 17.8543000 - 412 2 -1.5602000 13.6461000 12.8736000 16.0588000 - 413 1 2.8152300 14.0966000 0.7310060 24.6664000 - 414 2 -1.6002300 13.6592000 3.6722200 24.0840000 - 415 2 -1.6110600 1.3007700 8.0993200 22.1991000 - 416 2 -1.6345400 1.2536400 3.8625300 25.8778000 - 417 2 -1.3844700 10.6431000 0.9951060 21.4710000 - 418 2 -1.5646100 14.8103000 9.5263200 25.3073000 - 419 1 2.6054400 2.2035300 6.9872800 23.0500000 - 420 2 -1.5500500 12.4892000 1.0024000 24.4957000 - 421 2 -1.5880300 6.5776000 14.8264000 23.3261000 - 422 2 -1.6187400 10.6113000 14.1706000 24.9280000 - 423 2 -1.5174100 6.7101100 0.5782860 16.5321000 - 424 2 -1.6384500 12.2594000 9.2570700 25.6247000 - 425 2 -1.6233800 1.5595900 1.3012600 25.3793000 - 426 2 -1.5716800 0.7653070 2.0824500 28.1046000 - 427 1 2.4439700 1.6097300 2.3414300 26.8312000 - 428 1 2.6147000 2.8170200 5.8292200 29.0824000 - 429 2 -1.4690500 6.0918300 2.0265600 27.2868000 - 430 1 2.5953800 13.7427000 8.5287800 25.8224000 - 431 1 2.5188900 2.1119500 8.0825600 27.2441000 - 432 2 -1.6504600 1.4810500 9.3146100 28.1910000 - 433 1 2.7144500 15.0064000 13.6868000 25.9453000 - 434 2 -1.6436900 1.8236400 6.5456500 27.8736000 - 435 2 -1.6203800 12.9349000 5.6416400 14.9712000 - 436 2 -1.6318100 2.4857100 8.4080100 25.7423000 - 437 2 -1.5231600 7.2848200 1.2640100 25.8505000 - 438 2 -1.6495300 5.1087700 0.3901280 25.8029000 - 439 1 2.5994000 5.8030100 1.8719000 25.4975000 - 440 1 2.7739900 4.8780500 5.2773700 25.9639000 - 441 2 -1.6057600 10.0749000 4.5485200 15.0045000 - 442 1 2.7463000 5.4187900 7.1064900 28.4127000 - 443 2 -1.5627800 8.8369200 1.7792200 28.3422000 - 444 2 -1.5711300 7.0625000 7.1945100 28.5868000 - 445 2 -1.5742200 6.6353000 3.7272400 28.5991000 - 446 1 2.8164900 5.2549000 3.1085200 28.4103000 - 447 2 -1.5890800 11.5211000 0.2305180 16.6638000 - 448 1 2.6985600 6.5083900 9.7582700 26.9342000 - 449 2 -1.5972600 7.5993900 14.7765000 18.1986000 - 450 2 -1.5673500 6.8633300 11.0997000 27.7081000 - 451 2 -1.6052100 5.0620600 12.6886000 28.8543000 - 452 2 -1.5928000 7.3008900 13.9681000 28.1062000 - 453 2 -1.5559300 4.2848900 6.4111400 29.4974000 - 454 1 2.6871400 6.2228600 12.6797000 27.5822000 - 455 2 -1.5621300 4.8558700 6.7635100 26.9487000 - 456 2 -1.4825300 3.8054100 2.7994200 29.0238000 - 457 2 -1.5168500 6.1657300 4.9318600 25.1980000 - 458 2 -1.6315500 5.4483800 13.1521000 26.1987000 - 459 1 2.7478100 8.5324700 0.8459950 24.8657000 - 460 2 -1.5372000 9.9667600 0.7409850 25.4138000 - 461 1 2.8166600 9.4913300 4.3148800 25.3641000 - 462 1 2.7434200 12.0124000 4.3307500 26.9565000 - 463 1 2.8371500 7.8641000 2.7728700 27.5949000 - 464 1 2.7906500 10.0825000 9.7491200 28.9306000 - 465 2 -1.5756600 11.2378000 7.4819600 28.6600000 - 466 2 -1.5767300 10.4370000 4.7887700 26.6463000 - 467 2 -1.5021900 8.3385900 3.5901000 26.3362000 - 468 2 -1.6195700 12.7086000 5.3999000 25.7976000 - 469 2 -1.5661100 11.9399000 2.7340300 26.7747000 - 470 2 -1.6668600 13.5937000 14.4002000 24.9525000 - 471 1 2.7801100 10.9743000 10.3980000 26.0919000 - 472 1 2.6977900 10.2100000 14.2843000 26.4747000 - 473 1 2.7423300 8.8117900 13.7622000 28.8620000 - 474 2 -1.5871300 12.0617000 11.5999000 26.9741000 - 475 2 -1.5733200 12.3487000 14.5235000 28.6225000 - 476 2 -1.4980600 13.9360000 12.9045000 28.7259000 - 477 2 -1.5810100 2.8728600 14.3608000 26.1371000 - 478 2 -1.5580200 9.3994700 13.3353000 27.3247000 - 479 2 -1.5384900 10.0235000 9.6589000 27.3247000 - 480 2 -1.5928300 4.9389500 9.6125100 26.6373000 - 481 1 2.7593300 12.5846000 0.0199036 17.6849000 - 482 1 2.6636400 12.2361000 7.0242800 27.6900000 - 483 2 -1.6187800 14.4472000 1.7180200 26.0746000 - 484 2 -1.6612600 4.9399900 8.5070300 29.4752000 - 485 2 -1.5740500 13.7885000 7.5957000 27.1251000 - 486 2 -1.6153700 2.9690900 6.8960400 15.0413000 - 487 1 2.6653000 14.7891000 9.4904200 29.5148000 - 488 1 2.7677400 13.5809000 11.7513000 27.7910000 - 489 2 -1.6102800 14.5617000 10.3803000 28.0953000 - 490 1 2.5730200 2.0305800 12.7304000 29.0291000 - 491 2 -1.5878200 14.7942000 12.0845000 26.5144000 - 492 2 -1.5733100 13.0097000 5.7263600 28.1399000 - 493 2 -1.5989800 4.4123700 4.0890000 27.0936000 - 494 1 2.8034200 11.3401000 1.8368300 25.2012000 - 495 2 -1.5463600 7.4374200 9.1129400 25.8795000 - 496 2 -1.6066100 13.0947000 8.1727400 14.9780000 - 497 2 -1.5568400 15.0019000 5.1525000 20.1077000 - 498 2 -1.5188500 15.0971000 3.3024300 22.1235000 - 499 1 2.6293200 4.3692200 14.8041000 26.1240000 - 500 1 2.8087900 12.9837000 13.5086000 29.7472000 - 501 2 -1.6037200 9.3996400 8.2012000 29.7553000 - 502 2 -1.6009000 4.1738200 9.7881200 14.8960000 - 503 2 -1.5393500 9.1246100 26.1052000 11.9543000 - 504 2 -1.9231200 5.6755600 17.6202000 2.2972500 - 505 1 2.4571000 1.7921800 17.6614000 0.9816150 - 506 2 -1.6739400 1.4887000 16.1520000 1.5435100 - 507 1 2.5801800 2.6073000 16.8684000 6.1195800 - 508 2 -1.5816000 3.1938800 15.8316000 12.6034000 - 509 1 2.4527300 1.7689800 22.3104000 1.8760300 - 510 2 -1.7299100 3.9767800 19.7571000 2.8246000 - 511 2 -1.5755100 1.7415200 23.9051000 1.4352000 - 512 1 2.4372800 5.0613700 16.9522000 3.6676300 - 513 1 2.6809600 2.2071400 19.1387000 3.6112100 - 514 1 2.7928100 1.1170100 23.7302000 7.1244500 - 515 2 -1.5921200 1.5868300 19.9830000 4.7865800 - 516 2 -1.5873400 1.1234000 18.4814000 8.7423500 - 517 2 -1.6013700 14.4800000 29.6434000 1.9820300 - 518 2 -1.6060300 1.2371300 21.9571000 6.7307500 - 519 2 -1.6422300 2.1965400 17.4411000 4.6141300 - 520 2 -1.6253200 1.4181900 22.3599000 3.3414700 - 521 1 2.7609300 1.8911300 19.5622000 9.6383500 - 522 1 2.5387600 2.0319100 22.9558000 13.1681000 - 523 1 2.4231100 0.5167890 24.7025000 1.3835400 - 524 2 -1.6230700 0.6380080 23.5051000 13.5477000 - 525 2 -1.6198800 2.2759200 21.5498000 12.4528000 - 526 2 -1.5217900 1.5860000 21.0417000 9.4213900 - 527 2 -1.5416300 2.6920300 24.2895000 7.1240700 - 528 2 -1.6047900 1.3238200 26.1509000 1.5739400 - 529 1 2.7192900 14.4667000 26.1330000 1.0001500 - 530 1 2.5853900 7.2833800 17.0422000 2.0167800 - 531 2 -1.6264100 7.5552800 15.5698000 2.4906600 - 532 1 2.6610300 5.4667000 16.0322000 6.3081400 - 533 2 -1.6366500 5.0984400 15.7156000 4.4792300 - 534 3 0.3645500 0.3270670 26.6758000 3.2821400 - 535 1 2.8238100 4.8512300 16.4461000 12.3922000 - 536 2 -1.5372800 11.6250000 25.3134000 10.6924000 - 537 2 -1.5686600 5.8471700 15.5295000 13.2882000 - 538 1 2.7995700 5.0539600 23.7306000 2.5331800 - 539 1 2.6969600 5.0582000 21.2531000 4.6442200 - 540 1 2.7500000 3.2480200 25.5637000 1.0849300 - 541 2 -1.5363300 4.3224800 22.6785000 3.5294500 - 542 2 -1.6111700 7.9929100 18.3320000 3.2907600 - 543 1 2.3188400 4.9528000 19.0599000 1.5260100 - 544 2 -1.4836100 4.1078400 24.2201000 1.4072500 - 545 2 -1.5920800 5.9762600 22.4479000 1.3892500 - 546 2 -1.6166100 2.1005400 21.5814000 0.5031990 - 547 1 2.7221700 7.1321700 19.1111000 5.5623900 - 548 1 2.7459100 2.7498100 23.0588000 4.3242700 - 549 1 2.8877300 6.0258800 22.6004000 7.5116400 - 550 1 2.8090300 4.7033000 18.9868000 7.4356900 - 551 1 2.7650900 2.3704200 20.9006000 6.0829200 - 552 2 -1.4446000 6.1778000 19.2039000 6.8808800 - 553 2 -1.4782200 5.2742300 24.2621000 7.5323900 - 554 2 -1.5544900 7.2190400 17.3759000 9.2620800 - 555 2 -1.5797400 3.3614300 19.7770000 6.8036800 - 556 2 -1.4984600 5.2702600 21.7496000 8.5665900 - 557 2 -1.6285100 4.1811400 17.2631000 6.5614100 - 558 2 -1.5189000 3.3402800 21.7827000 5.1913100 - 559 2 -1.6575200 6.9435900 16.7003000 6.0904000 - 560 2 -1.6424000 5.8335600 19.7447000 4.5987200 - 561 2 -1.4834200 5.8558100 22.0210000 5.8552000 - 562 1 2.8124000 5.8620800 18.1417000 10.1062000 - 563 1 2.7666500 4.0422200 25.1071000 7.1336800 - 564 1 2.7193800 5.3362500 22.4099000 12.0930000 - 565 1 2.6692000 4.1082300 18.1222000 14.4535000 - 566 1 2.5940600 2.5089400 19.8728000 12.6174000 - 567 1 2.6904700 4.2287500 25.3234000 11.5604000 - 568 2 -1.6042700 6.2477700 19.5994000 10.9297000 - 569 2 -1.6081800 4.4933200 23.7079000 11.4508000 - 570 2 -1.6481600 4.8065600 21.7106000 13.4677000 - 571 2 -1.5706900 7.4187000 17.9803000 13.1556000 - 572 2 -1.5994300 2.7786400 19.0830000 14.0784000 - 573 2 -1.6272600 5.2537600 26.4705000 10.9281000 - 574 2 -1.5550200 6.8452100 22.1879000 11.6759000 - 575 2 -1.5390100 4.7793100 17.9200000 13.1913000 - 576 2 -1.5382500 2.2565100 19.1237000 11.2418000 - 577 2 -1.5725800 5.3159500 16.7852000 10.8972000 - 578 2 -1.5279100 4.6575100 18.4972000 8.9754600 - 579 2 -1.4874900 7.7123100 22.9663000 7.8700500 - 580 1 2.6563400 7.2948000 19.4467000 14.0471000 - 581 1 2.8086100 10.0978000 20.5681000 13.5728000 - 582 2 -1.5255500 2.7648600 24.6355000 4.4128200 - 583 2 -1.5786000 12.2702000 29.4756000 11.1837000 - 584 1 2.6764700 10.5413000 16.5840000 0.6823800 - 585 1 2.6916600 8.4439300 15.7081000 6.1720500 - 586 1 2.7104100 13.0555000 15.3692000 12.9350000 - 587 1 2.7223100 7.9399000 16.0010000 9.4063900 - 588 1 2.7458700 10.9092000 15.8840000 10.7224000 - 589 2 -1.6131000 8.9534700 15.4197000 7.8528000 - 590 1 2.7274300 11.8797000 26.5011000 9.6611700 - 591 2 -0.6578570 8.6301400 19.4542000 10.8421000 - 592 1 2.7913100 11.1555000 21.1904000 1.1045400 - 593 1 2.6858600 9.0110400 19.4194000 2.4081200 - 594 1 2.7691600 10.0335000 23.9477000 1.7249100 - 595 2 -1.6387000 7.9534900 20.5679000 1.4034300 - 596 2 -1.5676700 10.5170000 21.8539000 14.4709000 - 597 2 -1.5191900 10.8084000 22.7925000 0.8497250 - 598 2 -1.6091600 8.3841400 17.8671000 1.1763100 - 599 1 2.6995800 11.6034000 17.6000000 3.5554000 - 600 1 2.8063800 7.5972200 22.0667000 2.0131400 - 601 1 2.8318000 11.9652000 22.7528000 6.9372900 - 602 1 2.7882200 10.0106000 18.2551000 6.0579500 - 603 1 2.7925800 12.3647000 23.8807000 4.0527300 - 604 2 -1.5973300 11.4125000 18.4944000 4.9956700 - 605 2 -1.4058900 7.7095500 22.0211000 3.4704900 - 606 2 -1.5781400 10.6619000 22.9516000 8.1183900 - 607 2 -1.4830100 13.0462000 17.7691000 8.0691300 - 608 2 -1.5238700 8.7273200 19.2476000 5.7601400 - 609 2 -1.5490000 12.1192000 23.9966000 5.6965700 - 610 2 -1.5122300 11.6074000 21.3857000 6.0322000 - 611 2 -1.5789100 9.6294100 16.6456000 5.6670700 - 612 2 -1.5561900 8.4397000 23.3628000 1.4026200 - 613 2 -1.6022400 12.3241000 16.0411000 3.8081400 - 614 2 -1.5816700 10.6099000 17.6651000 2.3431200 - 615 2 -1.5216000 12.8189000 21.2099000 1.5129300 - 616 1 2.8010200 11.7744000 18.3558000 8.8429000 - 617 1 2.8332300 9.0299400 22.8032000 9.0048600 - 618 1 2.8051700 11.1718000 23.3159000 14.0207000 - 619 1 2.6939200 8.4694400 16.8733000 12.5215000 - 620 1 2.7866000 7.8026000 20.7151000 11.1180000 - 621 1 2.8061700 10.0442000 24.7041000 11.3844000 - 622 2 -1.4946100 12.0988000 19.9328000 9.3192400 - 623 2 -1.5084100 9.2128700 24.0446000 10.1360000 - 624 2 -1.4545500 11.7115000 18.0919000 12.8287000 - 625 2 -1.5875500 8.2966800 20.3923000 13.1174000 - 626 2 -1.5542400 10.2530000 23.8567000 12.7656000 - 627 2 -1.4249900 10.8566000 20.5509000 12.2744000 - 628 2 -1.6540400 9.0380600 16.2006000 10.9464000 - 629 2 -1.4753300 8.6698300 21.4258000 9.7455200 - 630 2 -1.5130800 11.5049000 17.2465000 10.1593000 - 631 2 -1.5422900 10.5365000 18.3970000 7.6320900 - 632 2 -1.5366300 13.5524000 22.8220000 7.6632600 - 633 1 2.8093300 10.8615000 17.7235000 14.0920000 - 634 2 -1.5640800 9.6918800 16.6794000 13.4088000 - 635 2 -1.5847600 12.8676000 23.2813000 13.7917000 - 636 2 -1.5443600 10.3468000 20.3752000 2.3930400 - 637 2 -1.5706300 10.7727000 23.9345000 3.2012900 - 638 2 -1.5506600 13.4380000 22.6851000 3.6877000 - 639 2 -0.6419120 11.9213000 21.9244000 10.9331000 - 640 2 -1.5467600 10.2885000 19.1451000 14.5097000 - 641 2 -1.5423800 5.0096300 25.5696000 0.0615873 - 642 2 -1.7053700 14.6117000 15.4058000 9.8358400 - 643 1 2.7119800 14.7648000 18.0451000 0.4488230 - 644 2 -1.5408100 13.1408000 18.3046000 0.9600250 - 645 1 2.8160500 13.7487000 21.1621000 2.8724300 - 646 1 2.8676000 14.3212000 18.6139000 7.6964600 - 647 1 2.8103900 12.5539000 20.0740000 5.4751600 - 648 2 -1.6246300 11.6725000 15.3739000 1.0443300 - 649 2 -1.4777700 13.6276000 19.7351000 6.5408800 - 650 2 -1.6272100 0.7520550 26.6685000 14.6244000 - 651 2 -1.5543800 13.1929000 20.1470000 4.0349500 - 652 1 2.8299400 14.4713000 22.9388000 9.0958600 - 653 1 2.8201200 13.1621000 18.6018000 12.7253000 - 654 1 2.8098500 12.7673000 20.8734000 10.5985000 - 655 2 -1.5722900 14.4656000 24.4525000 9.7640400 - 656 2 -1.5012800 13.4249000 19.6575000 11.5785000 - 657 2 -1.5820600 13.8614000 16.6730000 12.3087000 - 658 2 -1.4865700 14.0391000 21.8201000 10.1214000 - 659 2 -1.5982800 13.7616000 19.1827000 14.2373000 - 660 2 -1.5003000 12.9539000 25.3089000 3.4956700 - 661 1 2.7115300 13.2407000 15.7973000 5.2985500 - 662 2 -1.6195100 9.9970800 25.6552000 1.0218000 - 663 2 -1.6949400 0.3839280 24.2471000 8.7811500 - 664 2 -1.6647700 1.1969400 17.3420000 13.8954000 - 665 1 2.4399400 4.6751300 15.4551000 0.4146770 - 666 2 -1.6792700 1.3264100 18.8787000 2.1809000 - 667 2 -1.6759200 2.6838500 25.7265000 12.0780000 - 668 2 -1.5689400 5.6687400 25.1138000 3.5283100 - 669 2 -1.5266000 3.3088900 27.1189000 1.3051000 - 670 2 -1.5898500 4.5543000 26.5261000 6.3082700 - 671 2 -1.6046300 11.2582000 27.3538000 3.2940800 - 672 1 2.7195600 1.9795000 15.5698000 13.7434000 - 673 2 -1.6249400 1.1365500 27.5065000 7.4771600 - 674 2 -1.5706800 3.5801700 26.5335000 8.2534600 - 675 2 -1.5612500 14.5383000 27.0851000 10.9825000 - 676 2 -1.5640000 4.1514400 29.4145000 14.8620000 - 677 2 -1.9292300 3.8913200 17.9006000 0.3014590 - 678 2 -1.3291900 10.1801000 29.0364000 1.3717400 - 679 2 -1.6388500 13.0964000 27.1403000 5.5008500 - 680 1 2.6860900 12.5598000 26.6386000 4.0774400 - 681 1 2.7223400 14.0224000 26.9994000 7.0324400 - 682 1 2.6158100 13.3790000 28.9063000 0.8110540 - 683 1 2.5924800 14.5367000 29.3545000 3.6388300 - 684 2 -1.6092600 9.8986000 28.6307000 12.8575000 - 685 1 2.7087200 10.7242000 28.7916000 11.2183000 - 686 2 -1.5632000 7.4254700 27.6518000 13.5716000 - 687 2 -1.5452500 10.7868000 27.5838000 10.1619000 - 688 1 2.6802600 6.7331700 29.0302000 13.2938000 - 689 2 -0.6858040 14.0199000 28.5098000 13.2839000 - 690 1 2.7794500 8.9566600 26.9386000 13.2867000 - 691 1 2.6524200 2.2336500 26.7367000 13.4511000 - 692 2 -1.4436300 6.6616600 27.3115000 5.1543900 - 693 2 -1.6268600 11.6800000 14.9203000 12.0789000 - 694 1 2.7030000 10.0449000 27.5344000 2.1122500 - 695 2 -1.6364500 0.0462051 28.5128000 3.9423300 - 696 1 2.8090100 5.4826300 26.6339000 4.4344300 - 697 2 -1.5763100 14.4026000 16.8589000 4.9468200 - 698 1 2.7893500 1.5973300 28.9932000 3.8617100 - 699 3 0.4951330 0.6026780 28.7504000 5.8229400 - 700 2 -1.5986800 5.0606900 28.6096000 9.0585300 - 701 2 -1.6320400 1.8037800 28.3045000 12.7008000 - 702 2 -1.3558600 9.3213100 26.0858000 14.5734000 - 703 2 -1.5841400 5.7023000 28.9275000 11.9967000 - 704 1 2.5071700 2.7628400 28.7277000 11.2450000 - 705 1 2.4675200 2.0904700 26.9501000 8.6207900 - 706 1 2.7048100 6.0361300 28.3857000 10.4724000 - 707 2 -1.6522200 1.8060600 27.9115000 10.1032000 - 708 2 -1.4541900 6.0153500 27.9210000 1.6388400 - 709 2 -1.4857500 4.4222500 27.6340000 3.6399200 - 710 2 -1.5442100 3.8751300 29.2255000 7.3649700 - 711 2 -1.6640100 7.6824800 29.1583000 10.2235000 - 712 2 -1.5103600 3.5178600 26.8023000 14.3082000 - 713 3 0.4499670 1.9410100 27.0224000 4.8143100 - 714 1 2.7232200 4.5557600 27.7137000 7.5448600 - 715 1 2.7326100 7.1984500 28.8190000 1.9678700 - 716 2 -1.4900800 4.1714400 29.5574000 1.2464600 - 717 2 -1.5530400 8.4781600 27.7217000 2.5530200 - 718 2 -1.5902100 13.1038000 27.4178000 1.1794800 - 719 1 2.8285700 4.4550100 28.1220000 2.0393200 - 720 2 -1.4879800 2.0375500 28.5627000 2.5169000 - 721 1 2.7115700 1.0686800 27.7987000 1.6991400 - 722 2 -1.6047200 12.7733000 26.7744000 8.3137700 - 723 2 -1.6009800 12.5096000 15.1563000 6.4163600 - 724 2 -1.5709500 8.3018200 29.4642000 0.4800630 - 725 2 -1.6767900 1.8351600 15.4965000 6.8596100 - 726 2 -1.6344800 4.4968900 24.2190000 14.4550000 - 727 3 0.4875510 0.3149100 19.9737000 13.5647000 - 728 1 2.7752800 14.1882000 22.9038000 14.7598000 - 729 2 -1.5778200 0.0949927 24.4103000 5.7561200 - 730 1 2.7149300 14.9975000 28.2635000 12.0225000 - 731 2 -1.6075700 15.0326000 28.5710000 7.3461000 - 732 2 -1.6817400 13.3737000 14.9411000 14.4424000 - 733 2 -1.6065500 5.0237800 14.9408000 7.3966700 - 734 2 -1.6005400 0.0197857 28.3108000 0.7083360 - 735 1 2.8321800 4.3593900 26.1816000 19.8970000 - 736 2 -1.5745300 8.8260600 26.2837000 22.3329000 - 737 1 2.6921700 4.2361500 16.9755000 17.3502000 - 738 1 2.6319000 1.3124300 16.8392000 17.9717000 - 739 1 2.6562500 2.0180400 17.1304000 21.9348000 - 740 2 -1.6065500 1.8297000 18.6732000 22.4439000 - 741 1 2.5521700 1.9160200 23.3455000 23.1759000 - 742 2 -1.6378200 2.7872300 21.1485000 24.2073000 - 743 2 -1.6282500 0.5935710 17.0939000 19.9046000 - 744 2 -1.5643700 3.6400900 21.8882000 19.6559000 - 745 1 2.8026700 8.9626900 26.6658000 18.1593000 - 746 1 2.7952400 15.0546000 24.7156000 28.5041000 - 747 1 2.7618700 1.9959600 21.9833000 19.3676000 - 748 2 -1.6070000 1.2795800 20.4185000 15.3670000 - 749 2 -1.6284800 1.0214700 21.5930000 20.7205000 - 750 2 -1.5947900 1.3163000 15.9891000 23.0820000 - 751 2 -1.6041200 1.7788500 23.5368000 18.6596000 - 752 2 -1.5882400 1.5936700 21.0575000 17.9910000 - 753 2 -1.5508900 2.8663800 17.3338000 18.0426000 - 754 2 -1.6256100 0.4019430 17.6118000 16.6249000 - 755 2 -1.5570400 2.8480900 25.2490000 16.7314000 - 756 2 -1.5739700 1.4278000 24.4641000 22.1141000 - 757 1 2.6596600 5.1682900 17.1049000 21.3409000 - 758 2 -1.6752100 3.5060900 16.5585000 21.4007000 - 759 2 -1.6460700 14.9925000 16.6649000 29.4142000 - 760 2 -1.6437000 10.8556000 15.3646000 20.0487000 - 761 1 2.4381600 8.0824400 15.1111000 23.5205000 - 762 1 2.7669900 14.1303000 21.9385000 24.7457000 - 763 2 -1.6573200 1.2211500 15.2167000 17.6941000 - 764 1 2.7608200 6.8326300 15.5257000 16.9010000 - 765 1 2.7733700 6.8756000 18.4808000 19.2141000 - 766 1 2.7776500 4.6510600 21.4798000 20.8629000 - 767 1 2.6751300 4.9242600 17.4292000 24.2875000 - 768 1 2.6779000 3.1209000 19.6940000 23.0055000 - 769 1 2.7035400 5.7246600 23.5846000 22.9535000 - 770 2 -1.5451500 7.5212800 19.0564000 20.7198000 - 771 2 -1.6041000 4.7030500 22.6173000 22.1834000 - 772 2 -1.6653500 3.6983500 18.4843000 24.1625000 - 773 2 -1.6413800 6.8454000 23.0146000 24.2906000 - 774 2 -1.5680300 7.7538600 20.2582000 23.2313000 - 775 2 -1.6300000 5.8129400 17.4151000 22.8511000 - 776 2 -1.6013500 4.0690400 20.0346000 21.7395000 - 777 2 -1.5483600 5.8516000 17.2378000 20.0188000 - 778 2 -1.6031400 5.8231900 19.3744000 18.3306000 - 779 2 -1.6072200 8.8079100 22.0779000 20.1613000 - 780 2 -1.6049300 12.0108000 20.3977000 21.2357000 - 781 1 2.7760100 4.6907900 22.8168000 15.2284000 - 782 1 2.7652200 7.2507600 22.0460000 19.3823000 - 783 1 2.7689500 5.4736000 20.4423000 16.8820000 - 784 1 2.7259000 2.4284600 21.0572000 16.5575000 - 785 2 -1.6439400 7.0773700 20.1628000 15.4934000 - 786 2 -1.5592000 5.6924100 21.9959000 16.1675000 - 787 2 -1.5427000 6.2182800 21.3996000 20.4029000 - 788 2 -1.5313400 8.3204300 17.9722000 18.5318000 - 789 2 -1.6794900 3.8669200 20.1272000 16.7109000 - 790 2 -1.5753000 6.9223700 23.6670000 19.2433000 - 791 2 -1.5051800 7.2778500 21.4912000 17.8417000 - 792 2 -1.6054300 5.1417600 15.7385000 17.8048000 - 793 2 -1.5270700 7.0847600 16.8056000 16.1494000 - 794 2 -1.5728800 4.3399400 17.6194000 15.8863000 - 795 2 -1.5851900 8.3412800 23.4807000 15.6575000 - 796 1 2.7607500 6.9340100 25.0437000 20.0911000 - 797 2 -1.6033000 4.8373400 24.5583000 23.9162000 - 798 2 -1.6279100 0.9379400 23.1662000 24.4031000 - 799 1 2.8383100 4.1299400 26.3464000 15.8921000 - 800 2 -1.5908400 3.1119400 22.4161000 15.9551000 - 801 1 2.6155600 12.9604000 16.2336000 22.0890000 - 802 1 2.5625000 7.5736800 19.5815000 24.5427000 - 803 1 2.7900900 11.0184000 15.9831000 16.5461000 - 804 1 2.6370100 12.1208000 18.7100000 29.4768000 - 805 1 2.7510900 12.1531000 18.8693000 20.3291000 - 806 1 2.6351500 9.8095600 23.2667000 20.3474000 - 807 1 2.5913500 10.0359000 17.4834000 23.7101000 - 808 1 2.6757700 8.3837100 19.4946000 21.9002000 - 809 1 2.6603000 11.2125000 24.4249000 22.7083000 - 810 2 -1.6288200 10.8359000 21.4375000 23.5969000 - 811 2 -1.6090200 9.7667100 23.9869000 21.9353000 - 812 2 -1.6292800 14.3705000 17.3545000 24.6602000 - 813 2 -1.6287100 9.6413500 18.5533000 22.4818000 - 814 2 -1.5832900 10.9361000 24.2341000 24.1885000 - 815 2 -1.6030000 9.3029200 15.8324000 22.8960000 - 816 1 2.5533400 11.7917000 20.7057000 22.6671000 - 817 2 -1.6278600 12.4370000 17.6816000 21.7588000 - 818 2 -1.5561800 10.5581000 18.7633000 19.7837000 - 819 2 -1.5256600 13.7077000 23.2088000 20.0856000 - 820 2 -1.6463900 11.3769000 17.2081000 24.5768000 - 821 2 -1.5941900 8.3152400 25.8179000 19.5437000 - 822 2 -1.5489500 10.9828000 24.2977000 15.3273000 - 823 1 2.8246700 12.9486000 25.2125000 16.6230000 - 824 1 2.7891400 12.4440000 23.2218000 18.9341000 - 825 1 2.7478300 9.8711400 17.9801000 18.6390000 - 826 1 2.7682700 8.2139900 21.9417000 16.4664000 - 827 1 2.7548000 9.5234900 24.8615000 15.6664000 - 828 2 -1.5181700 12.0543000 19.9845000 16.5333000 - 829 1 2.7421600 2.9293300 24.5539000 18.1102000 - 830 2 -1.5878000 10.9667000 23.9199000 19.5931000 - 831 2 -1.5409100 13.0730000 18.3715000 19.2010000 - 832 2 -1.5176000 12.6220000 23.9572000 17.4864000 - 833 2 -1.5623300 11.8597000 21.6584000 18.5441000 - 834 2 -1.5472100 10.7055000 17.1283000 17.6628000 - 835 2 -1.6102300 9.6370600 21.2440000 17.0925000 - 836 2 -1.5795000 11.5722000 16.8193000 15.2109000 - 837 1 2.6328000 8.4782500 24.7992000 22.4391000 - 838 2 -1.4268400 12.6399000 26.5595000 16.0356000 - 839 2 -1.5587000 12.3651000 23.6497000 22.1625000 - 840 2 -1.5765700 7.0982100 24.3819000 21.8827000 - 841 2 -1.5179100 4.1578300 24.9527000 18.8234000 - 842 2 -1.5794300 13.3238000 26.4570000 18.5052000 - 843 2 -1.5167700 9.4970600 25.5089000 17.1937000 - 844 1 2.5044400 1.4118100 18.6735000 14.9185000 - 845 1 2.8267100 13.9324000 23.4723000 21.7590000 - 846 2 -1.5501700 14.8558000 24.7176000 22.2162000 - 847 2 -1.6104100 14.2578000 20.1968000 24.3825000 - 848 2 -1.6126600 14.3546000 22.3560000 23.1108000 - 849 1 2.7603800 1.8564000 25.9454000 21.1877000 - 850 2 -1.6397100 4.7034500 16.1587000 25.2398000 - 851 1 2.8043100 13.6306000 19.9168000 15.8997000 - 852 3 0.3372550 0.1770650 16.2066000 21.4413000 - 853 1 2.7624200 14.7963000 18.7580000 18.5733000 - 854 1 2.7579400 11.2136000 20.9846000 17.3093000 - 855 2 -1.5877200 14.4371000 24.6095000 15.7490000 - 856 2 -1.5175300 14.6315000 19.0786000 16.8943000 - 857 2 -1.6251800 14.4990000 15.6526000 22.1227000 - 858 2 -1.5641300 13.8442000 21.6557000 15.7713000 - 859 1 2.7724700 13.3776000 27.2531000 25.1270000 - 860 2 -1.6593900 8.6914500 18.2219000 25.0089000 - 861 2 -1.2879200 5.4683000 25.9693000 16.2867000 - 862 2 -1.5038700 5.8385600 26.0526000 20.4284000 - 863 2 -1.6231800 14.4707000 25.9947000 25.3299000 - 864 2 -1.5387500 3.3156000 25.5466000 20.9813000 - 865 2 -1.5843000 11.8079000 26.9106000 21.0484000 - 866 2 -1.5457800 12.5140000 26.9303000 23.8532000 - 867 2 -1.9297200 6.2525400 16.3982000 29.5194000 - 868 2 -1.6293500 8.2108400 25.4373000 25.5650000 - 869 2 -1.6202600 2.8753600 25.1851000 29.0047000 - 870 1 2.4573800 2.4253500 20.8716000 25.7185000 - 871 1 2.5600900 1.5812900 18.2701000 26.8298000 - 872 1 2.4583100 1.4105600 24.6924000 28.1522000 - 873 2 -1.9229400 1.8594600 18.6074000 28.9578000 - 874 2 -1.9299700 3.4904000 19.4029000 27.1195000 - 875 2 -1.6028300 3.5482500 15.6803000 29.2320000 - 876 2 -1.6046200 0.7576950 25.7615000 26.9632000 - 877 2 -1.6095400 0.9441120 16.9727000 26.6954000 - 878 1 2.6144200 6.7997800 15.3863000 27.7695000 - 879 2 -1.9238800 5.3796700 21.4629000 26.3233000 - 880 1 2.6188000 7.9732600 17.5229000 26.3633000 - 881 1 2.5347000 5.3278100 19.7128000 26.7828000 - 882 1 2.4318000 4.3500300 16.8824000 28.6031000 - 883 1 2.5001600 3.5556600 19.2600000 28.9335000 - 884 1 2.7302300 4.6167700 24.5374000 28.6990000 - 885 2 -1.9269800 7.7746600 18.3278000 28.0008000 - 886 2 -1.6402000 4.3240800 22.7269000 28.9064000 - 887 1 2.7250000 8.1416200 29.5839000 17.0885000 - 888 2 -1.6024500 5.1465200 15.6028000 27.6570000 - 889 2 -1.6881900 5.0142000 20.0809000 29.3115000 - 890 2 -1.6329600 7.6096200 16.0264000 26.4367000 - 891 2 -1.6374200 7.4830200 23.1686000 27.0619000 - 892 1 2.7230800 11.1243000 22.7805000 24.9401000 - 893 2 -1.5817200 5.3172100 25.0878000 27.4437000 - 894 1 2.8031900 11.9519000 15.9392000 28.3208000 - 895 2 -1.6273200 10.5313000 15.6534000 27.1091000 - 896 2 -1.5494500 9.8814000 22.7793000 26.0418000 - 897 1 2.6985900 8.9446600 23.9410000 26.7746000 - 898 1 2.7123300 4.2191500 29.5922000 29.4944000 - 899 2 -1.5804400 12.5506000 18.8908000 26.3598000 - 900 2 -1.6136100 9.7016000 24.6818000 28.0944000 - 901 2 -1.5168900 11.4174000 17.2311000 29.2568000 - 902 2 -1.5463100 13.0198000 16.6160000 27.1150000 - 903 2 -1.5781600 12.4486000 27.5764000 26.7174000 - 904 1 2.7008800 13.6061000 19.6170000 27.3251000 - 905 2 -1.6255000 11.8605000 14.9755000 22.5441000 - 906 2 -1.5423400 14.4572000 23.1452000 28.3996000 - 907 2 -1.4842700 13.6044000 19.0335000 28.7178000 - 908 2 -1.5526000 9.8642200 14.9301000 16.2219000 - 909 2 -1.5153400 14.2380000 20.9346000 26.9812000 - 910 1 2.7280200 12.7667000 17.2634000 25.6705000 - 911 2 -1.5627800 14.0421000 25.9155000 29.1202000 - 912 2 -1.6857400 2.8086500 15.3371000 15.2290000 - 913 1 2.6511300 6.5324800 22.8735000 25.8327000 - 914 2 -1.6339600 1.8505100 23.0407000 27.8370000 - 915 2 -1.9309800 6.3499400 18.7028000 25.5933000 - 916 2 -1.6305500 1.1393800 19.8749000 26.2230000 - 917 2 -1.5725300 12.6255000 22.4421000 25.3188000 - 918 2 -1.6350000 14.3195000 28.8310000 25.4807000 - 919 1 2.6532500 8.0572000 17.5335000 29.5411000 - 920 2 -1.5768600 8.3004100 26.7791000 29.1004000 - 921 2 -1.6193200 13.1100000 29.3620000 29.0694000 - 922 1 2.6742600 12.0153000 28.4695000 27.9552000 - 923 1 2.8417200 14.8104000 27.3579000 18.5519000 - 924 2 -1.4611900 7.9053300 29.1153000 27.4865000 - 925 2 -1.6355100 10.5056000 28.3240000 28.4559000 - 926 1 2.7565600 8.3473700 28.4122000 28.7869000 - 927 2 -1.5773600 5.7412200 26.5154000 25.3630000 - 928 2 -1.5697300 3.7656700 27.2873000 27.1691000 - 929 2 -1.6195100 3.0273000 29.5553000 28.3624000 - 930 2 -1.5591700 6.2157400 27.8696000 27.5329000 - 931 1 2.6581100 2.7816500 28.4983000 27.1590000 - 932 1 2.7365000 5.3055800 26.6326000 26.9779000 - 933 2 -1.6233600 1.5809000 28.2751000 26.1332000 - 934 1 2.7174500 1.2551900 26.7020000 25.3507000 - 935 1 2.7879100 11.9489000 27.3586000 19.4158000 - 936 2 -1.4792300 3.5536600 27.9035000 16.7242000 - 937 2 -1.6387900 2.7918000 26.0946000 24.7979000 - 938 1 2.6817900 7.3119200 26.5784000 25.0007000 - 939 1 2.2951700 5.4259900 21.8329000 29.6574000 - 940 1 2.6317200 2.8253700 29.2863000 16.0864000 - 941 2 -1.6026300 8.8841300 29.6104000 15.4290000 - 942 2 -1.4384100 12.9302000 28.6080000 18.8822000 - 943 2 -1.4152200 12.9955000 28.9263000 22.2818000 - 944 1 2.8435300 11.9278000 27.8874000 22.5051000 - 945 2 -1.5630600 8.0591700 27.9565000 17.5181000 - 946 3 0.2212210 0.3714930 25.8055000 29.6027000 - 947 1 2.7889300 6.2294300 29.5067000 27.2511000 - 948 2 -1.6874500 11.0871000 20.0103000 29.4996000 - 949 2 -1.6499300 0.5845090 26.1376000 19.8899000 - 950 2 -1.5951700 1.9863300 27.5720000 21.9786000 - 951 2 -1.5795300 7.7563100 27.7723000 24.0498000 - 952 2 -1.4912100 10.2778000 28.2446000 22.9354000 - 953 1 2.8160900 8.6807100 27.8960000 22.4879000 - 954 2 -1.5520300 10.4590000 27.5492000 18.8070000 - 955 1 2.1152000 0.6497460 27.2220000 16.1206000 - 956 2 -1.4840800 3.8541400 27.6453000 19.6222000 - 957 2 -1.6145600 1.0583600 28.6737000 16.5218000 - 958 2 -1.4661200 7.7495800 28.7066000 21.2409000 - 959 2 -1.5673100 4.5215100 27.2163000 23.0778000 - 960 1 2.6713100 3.3466500 28.1434000 22.5848000 - 961 1 2.7131600 4.4436700 26.2009000 24.2981000 - 962 1 2.2810300 2.7586700 22.1384000 28.8492000 - 963 1 2.7122500 9.3762300 25.6272000 29.3048000 - 964 2 -1.6153800 3.2355900 29.5621000 23.4789000 - 965 2 -1.5465700 14.8359000 29.1262000 18.5863000 - 966 2 -1.5682200 14.7705000 14.8808000 17.0492000 - 967 2 -1.6189400 9.1580400 16.1870000 29.5215000 - 968 2 -1.5882000 12.1617000 14.9229000 17.2521000 - 969 1 2.6721500 17.7053000 12.3497000 0.8333460 - 970 2 -1.6720000 20.6773000 0.2615210 14.7541000 - 971 2 -1.6569500 28.7780000 8.0499300 1.3100100 - 972 2 -1.6051500 16.0357000 0.4430960 8.3613400 - 973 2 -1.5409400 28.7330000 1.7128300 14.3475000 - 974 1 2.6670800 15.8635000 1.1135900 1.8545100 - 975 2 -1.5457100 18.1967000 14.2190000 9.8448100 - 976 2 -1.6507700 15.4540000 14.2689000 3.5039000 - 977 1 2.7947800 29.4870000 14.2818000 14.7672000 - 978 2 -1.6270100 26.8453000 14.7342000 1.2763400 - 979 2 -1.5630700 26.0867000 10.3858000 14.5698000 - 980 2 -1.5474200 16.5882000 7.9930100 4.1977400 - 981 2 -1.6303200 25.7898000 0.5184430 4.7540700 - 982 2 -1.5369600 18.5363000 5.2808800 14.3651000 - 983 2 -1.5890200 21.0903000 14.6571000 11.7830000 - 984 2 -1.6027400 15.4893000 0.9825010 3.6124600 - 985 1 2.7795800 17.1200000 5.5248400 2.3854500 - 986 2 -1.5885900 16.0246000 4.8035500 3.5618800 - 987 2 -1.5977500 15.9866000 6.6988000 1.6795200 - 988 2 -1.4933900 17.8540000 4.9867700 0.8996230 - 989 1 2.6692700 15.9552000 1.9147000 4.8874900 - 990 1 2.8082000 16.1935000 5.9723200 7.6789200 - 991 2 -1.5212000 17.2634000 2.6903300 4.8879500 - 992 2 -1.5661100 25.5931000 13.1040000 14.5815000 - 993 2 -1.6109100 18.2701000 0.9782160 9.5773100 - 994 2 -1.5830500 15.4181000 7.1858500 6.8434600 - 995 2 -1.5802400 16.6866000 4.4656700 6.7207100 - 996 2 -1.5722100 26.3020000 5.6206800 2.2606400 - 997 2 -1.5697900 18.5486000 5.6633700 3.3021700 - 998 1 2.7339300 16.5888000 0.9166860 10.0378000 - 999 1 2.8289700 16.9981000 5.1046000 13.6630000 - 1000 1 2.7717900 15.3142000 6.7124100 11.6386000 - 1001 2 -1.6151500 15.8967000 2.3321800 10.6665000 - 1002 2 -1.5427300 20.4175000 0.2197870 1.5114400 - 1003 2 -1.5159600 16.6569000 6.0100500 12.1936000 - 1004 2 -1.5298700 16.4393000 3.5180400 13.1662000 - 1005 1 2.7442600 15.3193000 3.1664300 12.1043000 - 1006 2 -1.5457100 17.5560000 6.3483200 8.4880000 - 1007 1 2.7143700 15.6690000 14.3673000 1.7716200 - 1008 2 -1.5746700 18.4609000 8.8826900 2.3566700 - 1009 2 -1.5581700 16.2814000 12.9808000 1.2314400 - 1010 1 2.8125700 17.3111000 9.2951100 3.7497400 - 1011 1 2.7383400 15.6229000 13.0867000 4.6591300 - 1012 2 -1.6032500 18.9376000 6.9321900 6.0303000 - 1013 2 -1.5445500 16.5688000 9.5551800 7.0968100 - 1014 1 2.7510900 18.3298000 13.2782000 10.9790000 - 1015 2 -1.6060700 18.4348000 10.2819000 4.7135200 - 1016 2 -1.6386000 16.1730000 10.4101000 3.1876700 - 1017 1 2.7928600 20.1373000 10.5420000 4.9866700 - 1018 1 2.7922800 15.5944000 9.7278200 10.7163000 - 1019 2 -1.5769000 18.3749000 10.7153000 0.3594170 - 1020 1 2.7574900 16.9661000 14.1154000 13.5832000 - 1021 2 -1.5621200 16.3281000 10.3185000 12.0216000 - 1022 2 -1.5155400 18.0909000 8.3629700 14.4300000 - 1023 2 -1.5661800 17.4643000 12.9159000 12.3378000 - 1024 2 -1.5605700 15.3166000 8.0073700 10.6943000 - 1025 2 -1.6462100 15.4539000 9.1633800 14.3149000 - 1026 2 -1.5506000 18.5586000 13.0218000 14.7028000 - 1027 2 -1.5270900 16.9688000 12.6849000 5.3162300 - 1028 2 -1.5777300 18.6510000 0.2876910 2.8590300 - 1029 1 2.7834900 19.8793000 1.5148300 2.4356000 - 1030 1 2.7960400 19.4443000 4.1840700 0.6809990 - 1031 2 -1.6385000 19.2689000 2.7017800 1.3838500 - 1032 1 2.7648700 19.2753000 5.8387600 4.7523700 - 1033 1 2.7523700 18.8360000 2.4920800 8.8619500 - 1034 1 2.7485400 17.9278000 3.4556300 6.1561400 - 1035 2 -1.5859300 18.4194000 2.3955900 7.2731000 - 1036 2 -1.5899500 19.3698000 4.4989500 5.7682400 - 1037 1 2.8088800 18.8185000 5.4670700 9.1786300 - 1038 1 2.7587200 19.7886000 0.7425400 13.2492000 - 1039 2 -1.4483900 19.8181000 2.0603400 12.5754000 - 1040 2 -1.4616400 19.4982000 6.0245600 10.4719000 - 1041 2 -1.5466500 18.3026000 3.9061900 9.3437500 - 1042 1 2.6806100 19.0696000 9.5210800 1.1126300 - 1043 2 -1.5823600 18.7946000 12.8310000 2.1491500 - 1044 1 2.8259100 17.5873000 10.1015000 8.3028500 - 1045 1 2.7829900 17.8979000 12.6852000 6.6470500 - 1046 2 -1.5532000 18.9545000 9.0167500 8.0677900 - 1047 2 -1.5136400 18.2739000 14.1634000 6.8595800 - 1048 1 2.7212200 19.3726000 8.2549500 6.7799600 - 1049 1 2.8459900 19.3401000 9.0292700 13.5319000 - 1050 1 2.8093700 17.6893000 11.1799000 12.3007000 - 1051 2 -1.4250900 18.6319000 10.5249000 13.2546000 - 1052 2 -1.4662200 19.3238000 8.2716500 11.9354000 - 1053 2 -1.4669600 18.6225000 11.7810000 10.9747000 - 1054 2 -1.5212900 16.6510000 10.0657000 9.5719300 - 1055 2 -1.6172000 15.3617000 0.5848450 14.6271000 - 1056 1 2.7759900 17.0275000 0.3290490 14.7155000 - 1057 2 -1.5719100 18.0573000 11.6276000 7.7928000 - 1058 2 -1.5445800 16.0033000 2.6493400 1.3899000 - 1059 1 2.7475100 25.7614000 11.7619000 13.8449000 - 1060 2 -1.5834400 21.1741000 4.1055200 0.6064320 - 1061 2 -1.5621400 25.6050000 6.9195900 5.0659300 - 1062 1 2.6473500 22.0916000 1.2400400 4.1514500 - 1063 2 -1.6308000 27.2199000 9.4524800 10.7220000 - 1064 2 -1.6435300 25.2623000 5.2601900 8.4660100 - 1065 2 -1.5982500 16.5722000 0.9657100 0.2575230 - 1066 2 -1.5325800 20.6117000 1.8726300 3.9267300 - 1067 1 2.7624600 22.4623000 11.3962000 0.6832750 - 1068 2 -1.5881400 23.1276000 10.9418000 6.1264100 - 1069 1 2.7751400 23.7529000 14.1977000 8.4779400 - 1070 1 2.7681300 21.9684000 6.0372900 2.9517800 - 1071 2 -1.5855000 20.9521000 6.2682000 4.2792800 - 1072 2 -1.5765300 21.6802000 6.7396000 1.4673300 - 1073 2 -1.6208700 22.5896000 4.3167200 2.9079200 - 1074 1 2.6987400 20.6814000 4.5436000 6.7769400 - 1075 1 2.5618600 24.8638000 1.6886800 4.1844800 - 1076 1 2.4644500 22.9342000 7.4370500 6.4414200 - 1077 2 -1.6031300 22.7945000 1.0975200 8.2998300 - 1078 2 -1.6547800 22.6885000 6.0086300 7.1348600 - 1079 2 -1.6234600 21.9228000 3.6097900 6.3493800 - 1080 2 -1.5192800 24.8355000 7.7635300 1.0969500 - 1081 2 -1.5879400 23.5004000 6.5002600 3.5823800 - 1082 1 2.7558700 21.6084000 1.2255900 9.4761300 - 1083 1 2.8540200 21.8597000 5.3805500 13.2275000 - 1084 1 2.8377400 20.3027000 7.2214800 11.1726000 - 1085 2 -1.6119400 22.4582000 1.3849000 10.9048000 - 1086 1 2.7710500 19.9670000 13.9081000 14.6445000 - 1087 2 -1.5555000 23.0795000 1.3706500 14.1688000 - 1088 2 -1.4838900 21.2296000 6.6155100 12.3582000 - 1089 2 -1.4987500 21.7819000 4.1353600 12.0294000 - 1090 2 -1.5500800 20.5370000 2.3386600 8.9553000 - 1091 1 2.7074600 23.0806000 7.7749000 10.1385000 - 1092 2 -1.4978100 23.3989000 5.6223100 13.5835000 - 1093 1 2.8083300 20.4464000 12.5861000 2.6795100 - 1094 2 -1.5545700 20.6626000 11.1047000 3.3641800 - 1095 2 -1.5237300 21.1467000 12.3922000 1.0628500 - 1096 1 2.7711900 24.2281000 9.2803900 1.8612600 - 1097 1 2.8137400 22.2522000 12.3916000 6.2774800 - 1098 2 -1.6166300 23.8647000 9.4207900 3.6232300 - 1099 2 -1.5487100 23.7175000 7.4337800 11.4890000 - 1100 2 -1.4935100 23.0914000 13.2714000 5.2032900 - 1101 2 -1.6641900 24.3742000 7.9498000 9.0751000 - 1102 2 -1.5609600 22.8204000 9.9636200 1.1358600 - 1103 2 -1.5062600 21.4324000 13.6844000 3.2435800 - 1104 1 2.8172900 22.2075000 13.6939000 12.6760000 - 1105 2 -1.4256200 21.5546000 10.0448000 11.5252000 - 1106 2 -1.4767200 21.1847000 13.2334000 13.8332000 - 1107 2 -1.5030400 23.2721000 8.6018800 14.0844000 - 1108 2 -1.5813800 20.6912000 11.7803000 5.8796300 - 1109 2 -1.5598700 22.8469000 12.5099000 11.6864000 - 1110 2 -1.5521900 21.6233000 7.6323700 9.7867100 - 1111 2 -0.6334960 20.5713000 8.3904400 13.9833000 - 1112 2 -1.5529500 21.2709000 5.1371300 14.7518000 - 1113 2 -1.5699100 23.4672000 14.6267000 13.4688000 - 1114 2 -1.5160100 24.6058000 0.5316340 12.0237000 - 1115 2 -1.6482200 24.9578000 2.6035800 2.8400200 - 1116 1 2.6854500 23.8551000 3.7716600 2.1949900 - 1117 1 2.6759900 24.5314000 5.9707000 4.6809300 - 1118 1 2.7370800 24.3987000 1.0867100 8.0457900 - 1119 1 2.6458400 23.4257000 3.1287000 6.0787300 - 1120 2 -1.6541300 24.5232000 2.4898900 7.1457400 - 1121 2 -1.6364300 24.5187000 4.5094700 5.4068100 - 1122 2 -1.5835500 23.4460000 1.9552200 4.7979100 - 1123 1 2.4570000 23.5516000 5.0735600 8.5027100 - 1124 1 2.8436800 24.5234000 1.7403200 13.1782000 - 1125 1 2.7744500 22.8694000 3.2096900 11.4577000 - 1126 2 -1.4818000 24.1808000 3.0187000 12.3142000 - 1127 2 -1.6137300 23.3368000 4.0990500 9.7597600 - 1128 2 -1.5791600 23.8418000 12.3311000 0.2159250 - 1129 2 -1.6125700 20.4848000 9.0950300 5.8393600 - 1130 2 -1.5557400 24.1571000 13.1219000 2.7107700 - 1131 2 -1.6169700 24.0552000 8.5540200 6.5018500 - 1132 1 2.8149900 23.0118000 14.0591000 3.8914200 - 1133 1 2.6286400 25.3487000 8.6411500 7.7369200 - 1134 1 2.6429100 24.1764000 10.0995000 5.1609500 - 1135 2 -1.5889700 24.8575000 12.8127000 9.1220700 - 1136 2 -1.5791500 22.4493000 13.2774000 7.7340500 - 1137 1 2.6891700 24.5318000 11.3226000 8.8143400 - 1138 1 2.8035300 24.1760000 8.4908700 12.8333000 - 1139 1 2.8237900 22.7786000 10.8602000 11.2792000 - 1140 2 -1.5912300 24.0795000 10.0054000 11.9564000 - 1141 2 -1.5637600 23.2711000 10.7651000 9.5427800 - 1142 2 -1.6571800 25.7021000 10.1834000 8.3426800 - 1143 2 -1.5551200 27.7651000 0.3753310 11.8758000 - 1144 2 -1.6448500 25.6504000 1.1570500 9.1698800 - 1145 1 2.7852100 25.2686000 6.1348700 1.0061400 - 1146 2 -1.5912300 25.3984000 10.2447000 1.1581200 - 1147 2 -1.5673200 29.0014000 4.0234000 11.1071000 - 1148 2 -1.5962100 20.2727000 5.2761800 8.1069800 - 1149 1 2.7717300 29.7767000 0.4897570 13.3879000 - 1150 2 -1.5834700 25.7069000 10.6420000 5.2709800 - 1151 1 2.6424400 24.6659000 13.4345000 1.2170500 - 1152 2 -1.5779800 25.9664000 11.8526000 12.2007000 - 1153 1 2.5898300 24.8138000 0.8276360 0.3384170 - 1154 2 -1.6156900 29.8743000 11.1761000 1.9430300 - 1155 2 -1.5656500 29.9110000 13.5010000 12.3478000 - 1156 2 -1.6067800 29.5629000 8.5908200 12.6920000 - 1157 2 -1.5786000 28.7261000 11.2166000 13.2955000 - 1158 1 2.6325300 28.9822000 12.3338000 12.2411000 - 1159 1 2.7201200 29.4004000 9.9476200 14.0025000 - 1160 2 -1.6179200 27.7299000 12.4042000 3.9207300 - 1161 2 -1.6416900 29.6880000 8.1686400 6.8898700 - 1162 2 -1.5609400 28.3850000 10.4901000 5.8350400 - 1163 1 2.7699400 27.1785000 10.6789000 4.5883300 - 1164 1 2.5893300 29.4581000 9.7410300 6.6229600 - 1165 1 2.6559500 28.7999000 13.5728000 3.5954200 - 1166 2 -1.5418200 27.4172000 11.8531000 0.5029320 - 1167 2 -1.6363900 29.5257000 14.0401000 1.5475600 - 1168 2 -1.6727600 22.1459000 1.0484700 2.3869700 - 1169 1 2.6711500 28.5310000 11.1013000 1.1783500 - 1170 2 -1.5855700 15.7714000 14.8277000 7.3379100 - 1171 1 2.5207300 26.2042000 5.6270300 9.7907100 - 1172 2 -1.6276100 29.5504000 6.3280100 9.9366500 - 1173 1 2.6872700 29.8050000 7.0757300 13.1692000 - 1174 1 2.6975800 27.7900000 3.5552700 12.0739000 - 1175 1 2.6541700 29.0323000 4.6625500 9.5501800 - 1176 2 -1.5638000 29.6783000 3.3057700 5.3678600 - 1177 2 -1.6454500 28.1105000 1.7789300 7.1373400 - 1178 2 -1.5902600 30.0519000 4.3356400 8.2954900 - 1179 1 2.6560700 28.8888000 2.0368700 5.7746900 - 1180 1 2.7148500 28.7071000 0.6789080 8.2313500 - 1181 1 2.7571200 29.6532000 4.5196300 4.3109100 - 1182 2 -1.6380400 28.4847000 0.9998070 4.5917800 - 1183 2 -1.6305400 28.2033000 11.8894000 10.7935000 - 1184 1 2.5233700 26.7198000 11.2121000 11.0125000 - 1185 2 -1.5718600 28.1231000 7.1386700 4.2974000 - 1186 2 -1.6763700 26.7723000 7.8004200 7.2882300 - 1187 1 2.0473400 28.2819000 9.0712900 9.5989400 - 1188 2 -1.5789700 27.3204000 9.4803600 3.4916300 - 1189 1 2.7795900 28.4339000 8.3645700 3.0726700 - 1190 2 -1.5813000 29.9249000 9.0444100 3.4881000 - 1191 2 -1.5095500 25.7853000 8.2914900 12.8778000 - 1192 2 -1.5438800 25.7767000 1.7988600 14.2292000 - 1193 2 -1.5038100 24.4512000 4.6014000 1.0782500 - 1194 2 -1.5938200 27.3757000 4.6853800 9.0416400 - 1195 2 -1.5644900 27.3902000 4.1642200 13.4607000 - 1196 2 -1.5901000 28.4902000 6.4086100 13.6081000 - 1197 2 -1.5350600 25.8807000 6.2526200 14.3397000 - 1198 2 -1.6437600 26.7693000 6.7198800 10.8082000 - 1199 2 -1.6114900 26.7895000 2.8798300 10.9559000 - 1200 1 2.5657400 26.7063000 8.1601400 11.7241000 - 1201 1 2.7692800 27.2977000 5.6038000 14.4379000 - 1202 1 2.7748800 26.8778000 1.4359400 10.2367000 - 1203 2 -1.5592000 28.0580000 4.2142900 3.6884700 - 1204 2 -1.5976400 27.3261000 5.2239700 6.3547600 - 1205 2 -1.5960500 28.0233000 0.5626790 9.6775100 - 1206 1 2.7258400 27.0765000 6.7757400 5.6208300 - 1207 1 2.5135600 26.7361000 4.5013800 7.5563100 - 1208 1 2.7408400 27.4155000 4.5490400 2.2970900 - 1209 2 -1.5837700 27.8154000 3.7382800 0.8897930 - 1210 2 -1.6028500 29.9901000 2.4331200 0.9437030 - 1211 1 2.6238300 29.8764000 1.0697800 1.6256100 - 1212 2 -1.5338500 15.1565000 10.6944000 0.7782400 - 1213 2 -1.5553400 20.1179000 8.7410500 0.1667200 - 1214 2 -1.6038800 29.1804000 14.6496000 4.7612300 - 1215 1 2.6369100 28.2891000 14.8354000 0.2990260 - 1216 2 -1.4509700 20.6083000 10.1121000 14.7352000 - 1217 1 2.7748500 24.2817000 14.3136000 14.8609000 - 1218 2 -1.6302500 18.1980000 0.0146118 13.2201000 - 1219 2 -1.5562900 24.8432000 14.8205000 7.4622100 - 1220 2 -1.5071900 22.9647000 10.9786000 14.8568000 - 1221 1 2.7828300 24.3343000 5.8547100 14.8305000 - 1222 1 2.7716800 27.2165000 1.2650200 14.8806000 - 1223 2 -1.6095300 16.2825000 9.4732700 23.2329000 - 1224 2 -1.5912400 18.9027000 8.7664400 24.1655000 - 1225 1 2.7104200 26.4218000 0.3940430 20.5054000 - 1226 1 2.8432300 15.8817000 1.9363600 22.1275000 - 1227 1 2.6719500 16.5272000 6.2690300 24.1162000 - 1228 1 2.7524100 16.4423000 8.2695400 21.6454000 - 1229 2 -1.4967300 17.3860000 2.3274000 22.5530000 - 1230 2 -1.6041400 20.6259000 6.6728100 24.5858000 - 1231 2 -1.5638200 16.8322000 7.1023300 22.7851000 - 1232 2 -1.6065800 16.9756000 4.5902000 23.9999000 - 1233 2 -1.5563200 15.5850000 1.3457300 20.5731000 - 1234 2 -1.5149900 17.6779000 5.3915000 20.8793000 - 1235 2 -1.5870700 17.7232000 6.8244400 25.2719000 - 1236 2 -1.5723800 23.9792000 14.6940000 25.3235000 - 1237 2 -1.4913200 18.4692000 0.9312680 27.8292000 - 1238 1 2.8371500 16.6304000 5.1313300 19.6037000 - 1239 1 2.7835600 15.7261000 7.4436500 17.7249000 - 1240 2 -1.5366500 17.7721000 1.2186300 15.6755000 - 1241 2 -1.6249000 16.1757000 12.9655000 14.9397000 - 1242 2 -1.5513500 16.5478000 6.6497500 18.8127000 - 1243 2 -1.5674000 17.5567000 4.0422300 18.7281000 - 1244 1 2.7796500 16.5655000 10.8168000 20.4630000 - 1245 2 -1.5442900 16.9827000 11.7844000 21.7302000 - 1246 2 -1.5701900 17.2193000 9.3079300 20.7882000 - 1247 2 -1.5422100 18.9741000 13.8657000 19.4755000 - 1248 1 2.5852700 17.7583000 14.5188000 24.6204000 - 1249 1 2.8139700 16.9400000 9.0179700 15.3066000 - 1250 1 2.8401700 18.1673000 12.8335000 18.4900000 - 1251 2 -1.5535500 17.1990000 10.6218000 15.7391000 - 1252 2 -1.5034500 17.2583000 13.1863000 17.1424000 - 1253 2 -1.6828800 15.9960000 14.1712000 20.5819000 - 1254 2 -1.5513000 17.3885000 11.5790000 19.2753000 - 1255 2 -1.4980500 16.7366000 8.2574200 16.7677000 - 1256 1 2.8264700 19.2621000 5.6341100 20.9424000 - 1257 1 2.7395600 17.8546000 3.2066100 23.6550000 - 1258 2 -1.5651100 18.7621000 2.5301900 24.8634000 - 1259 1 2.8438300 19.7567000 5.5079800 15.5418000 - 1260 1 2.8290600 19.3304000 3.8691000 18.3594000 - 1261 2 -1.5387500 19.9893000 4.6709900 19.6986000 - 1262 2 -1.5440400 20.0501000 2.1561100 18.5360000 - 1263 2 -1.5056200 19.5624000 4.5089400 16.9253000 - 1264 1 2.6874800 17.6976000 9.8748200 24.2821000 - 1265 1 2.6870000 18.3981000 12.0101000 22.2540000 - 1266 2 -1.6132500 18.4350000 11.2684000 23.6963000 - 1267 2 -1.5700500 19.8293000 5.1859500 22.4770000 - 1268 2 -1.6427900 19.4972000 13.1231000 21.9513000 - 1269 1 2.8417900 17.9784000 12.0461000 15.8696000 - 1270 2 -1.5230300 19.0921000 11.9587000 17.3512000 - 1271 2 -1.4996600 20.0784000 9.3926800 17.6309000 - 1272 2 -1.6321000 16.5326000 13.5884000 24.9596000 - 1273 1 2.8139100 22.1586000 9.5482600 14.9115000 - 1274 1 2.6399400 19.1966000 7.3113900 24.7867000 - 1275 1 2.7970700 23.5219000 12.4537000 28.2570000 - 1276 2 -1.5524100 22.7847000 0.0663426 19.0498000 - 1277 1 2.5910800 15.1394000 6.6267200 28.1584000 - 1278 1 2.7053500 20.8707000 7.2259100 29.5777000 - 1279 1 2.5991800 28.7191000 8.6672700 29.1961000 - 1280 1 2.7933200 25.0199000 14.1115000 26.5474000 - 1281 1 2.6628700 16.3308000 2.5580400 29.6204000 - 1282 2 -1.4614000 20.4397000 0.9105840 20.9110000 - 1283 2 -1.6068500 19.0929000 14.8691000 25.4881000 - 1284 2 -1.5548500 16.8207000 1.7615300 26.1558000 - 1285 1 2.7709400 17.8388000 5.6294600 28.9877000 - 1286 1 2.7542300 15.5333000 2.7285600 26.6661000 - 1287 2 -1.6033200 16.4903000 5.7742200 28.2122000 - 1288 2 -1.5970900 19.5808000 12.0845000 26.5063000 - 1289 2 -1.5316300 19.4987000 10.1083000 27.4731000 - 1290 1 2.6823700 16.0147000 10.8651000 26.9110000 - 1291 2 -1.5797500 17.0868000 11.5681000 28.2203000 - 1292 2 -1.4961600 24.3492000 6.6461300 28.8790000 - 1293 2 -1.5656900 19.0361000 12.5619000 29.2219000 - 1294 2 -1.6290200 17.1756000 10.1666000 25.8559000 - 1295 1 2.7489400 18.3130000 1.4051600 26.0090000 - 1296 2 -1.5059900 20.1918000 0.8021750 26.8650000 - 1297 1 2.6925800 20.1656000 13.7398000 26.3661000 - 1298 1 2.6770600 20.2468000 10.6951000 26.1206000 - 1299 1 2.7479800 18.6699000 11.2573000 28.3228000 - 1300 2 -1.5613500 18.7965000 4.1092800 28.7669000 - 1301 2 -1.5725200 19.3888000 6.1770100 29.5648000 - 1302 1 2.7758800 25.4371000 3.0829800 25.0999000 - 1303 2 -1.5437300 23.4891000 0.4296600 21.5636000 - 1304 2 -1.5769000 24.0836000 2.6456300 24.3112000 - 1305 1 2.6287700 25.1020000 8.7999900 21.2756000 - 1306 2 -1.5563400 24.6792000 13.5064000 18.4375000 - 1307 2 -1.5316100 22.0995000 4.3156100 23.8001000 - 1308 1 2.7209500 23.4589000 1.7478700 20.8571000 - 1309 1 2.7915600 24.0270000 6.3528000 22.4979000 - 1310 1 2.8475600 21.4758000 7.5334900 20.3481000 - 1311 2 -1.5485000 23.1971000 3.0141200 21.6768000 - 1312 2 -1.5343200 19.9360000 7.1471300 20.7108000 - 1313 2 -1.5809100 22.3546000 6.4351200 22.2059000 - 1314 2 -1.5401000 21.8914000 11.6503000 17.1843000 - 1315 1 2.7090700 21.2388000 5.5890200 23.3331000 - 1316 2 -1.4109900 22.6109000 1.7259000 19.6148000 - 1317 2 -1.5747200 24.7677000 4.3480700 15.5846000 - 1318 2 -1.6122100 24.5556000 7.2911000 23.8738000 - 1319 1 2.7431800 22.1231000 0.8151480 15.3651000 - 1320 1 2.8851200 20.9244000 8.0185200 17.3002000 - 1321 2 -1.5366300 22.2537000 2.0941300 16.7982000 - 1322 2 -1.5284800 20.0502000 6.9582400 16.4408000 - 1323 2 -1.4979300 21.5691000 7.4341800 18.7881000 - 1324 2 -1.5502300 24.5699000 2.5886400 17.7507000 - 1325 1 2.8178000 22.1484000 10.6429000 20.2527000 - 1326 1 2.7969500 20.3858000 14.1378000 20.3333000 - 1327 1 2.8069900 22.8741000 13.8966000 24.3940000 - 1328 2 -1.5505900 22.5045000 11.4819000 21.6949000 - 1329 1 2.5993400 28.8955000 3.0691500 29.6969000 - 1330 2 -1.5447600 23.5980000 12.3404000 23.9136000 - 1331 2 -1.6098400 23.6779000 5.4074800 19.2982000 - 1332 2 -1.5619100 20.7914000 11.0165000 19.4340000 - 1333 2 -1.7026400 15.5232000 3.0214400 28.3677000 - 1334 2 -1.5653800 22.1022000 9.0021500 20.8167000 - 1335 1 2.8237300 23.1686000 14.2648000 18.9524000 - 1336 1 2.8426200 20.4631000 10.9942000 17.7789000 - 1337 2 -1.7083800 20.4100000 14.3964000 16.2066000 - 1338 2 -1.5520200 21.7330000 13.6435000 19.5984000 - 1339 2 -1.5866600 23.6415000 10.8531000 19.5639000 - 1340 2 -1.5423200 22.1402000 8.8015500 16.4517000 - 1341 1 2.7203900 25.1227000 5.8526700 19.4861000 - 1342 1 2.7882400 23.4884000 3.6394700 23.1608000 - 1343 2 -1.6003300 24.8374000 7.1174500 21.1073000 - 1344 2 -1.5108600 24.5095000 4.9057900 22.7679000 - 1345 1 2.7164800 23.2914000 1.5239000 18.0120000 - 1346 1 2.7679200 24.8986000 7.7220300 17.1226000 - 1347 2 -1.5427600 24.0699000 6.7694500 16.1843000 - 1348 2 -1.6574900 25.5431000 1.6507100 20.5277000 - 1349 1 2.7456800 25.0839000 14.3952000 21.3361000 - 1350 1 2.7807800 24.8718000 8.8275000 24.3110000 - 1351 1 2.7434100 23.8148000 11.5741000 22.5573000 - 1352 2 -1.6369100 24.4518000 9.6666200 22.5621000 - 1353 2 -1.6265100 25.2104000 12.6448000 21.8916000 - 1354 2 -1.5212900 15.3418000 4.2524000 26.0996000 - 1355 1 2.7086800 22.0871000 3.4429000 29.3160000 - 1356 1 2.8160600 24.7356000 10.4337000 18.3945000 - 1357 1 2.8753500 23.1693000 11.8299000 16.1836000 - 1358 2 -1.5179500 24.4974000 11.0620000 16.9503000 - 1359 2 -1.5879700 24.5532000 8.8133600 18.2786000 - 1360 2 -1.4414400 23.3892000 13.3730000 15.7792000 - 1361 1 2.6260300 24.8972000 3.8440400 16.9431000 - 1362 2 -1.5366500 24.1591000 9.1450400 25.7169000 - 1363 1 2.5554000 29.6811000 13.2830000 20.4257000 - 1364 1 2.8152200 25.7439000 7.3257600 28.4940000 - 1365 2 -1.5521500 25.2346000 2.0339700 26.5050000 - 1366 2 -1.5473500 25.6680000 8.9868300 28.5891000 - 1367 2 -1.5129600 25.9418000 13.0034000 25.6162000 - 1368 2 -1.6227400 26.3256000 10.5317000 19.0910000 - 1369 2 -1.6368700 28.1289000 6.2454100 25.2679000 - 1370 2 -1.4850600 28.8331000 13.9731000 28.6613000 - 1371 1 2.5252000 30.0117000 5.9035500 28.8041000 - 1372 1 2.6333400 28.0097000 12.0875000 28.0719000 - 1373 2 -1.6668200 28.8135000 7.0750100 28.4156000 - 1374 1 2.6917400 26.8145000 5.5877900 26.2507000 - 1375 2 -1.6589700 29.1016000 9.4078800 25.8041000 - 1376 2 -1.6118700 29.6989000 4.3013400 28.5896000 - 1377 2 -1.6560300 28.7344000 1.3844500 28.5394000 - 1378 1 2.5760400 29.4575000 4.0191600 26.8847000 - 1379 2 -1.5897300 27.8757000 4.1873300 26.7465000 - 1380 2 -1.6385600 28.5239000 12.0855000 26.3559000 - 1381 1 2.6209500 29.6500000 10.9043000 25.6856000 - 1382 1 2.6352600 28.8861000 7.5755000 24.8193000 - 1383 2 -1.5086100 26.1644000 6.8574000 27.0439000 - 1384 2 -1.6415800 27.9120000 1.0270900 26.1391000 - 1385 1 2.6474800 26.3032000 0.8811020 26.5681000 - 1386 1 2.5150300 29.2070000 1.2821200 27.0435000 - 1387 2 -1.5888500 21.4475000 13.4862000 25.2919000 - 1388 1 2.7526200 27.3755000 12.9560000 25.0052000 - 1389 2 -1.5013800 26.4361000 11.4822000 28.1897000 - 1390 1 2.7697400 26.0490000 10.2816000 29.4461000 - 1391 2 -1.6099400 29.0758000 0.6108710 18.0123000 - 1392 1 2.7244700 20.9819000 1.3228600 19.6158000 - 1393 2 -1.5548000 28.6392000 12.9284000 15.5305000 - 1394 2 -1.6173300 28.1508000 14.5060000 25.0687000 - 1395 2 -1.4485400 29.9129000 11.5899000 17.7081000 - 1396 1 2.8185900 28.4819000 12.0593000 16.9454000 - 1397 2 -1.5400300 29.2674000 11.1842000 29.0440000 - 1398 2 -1.6234100 30.1519000 2.4293900 26.6376000 - 1399 2 -1.6112600 23.9160000 1.9726800 29.3954000 - 1400 2 -1.6006900 28.6366000 12.3792000 21.4201000 - 1401 1 2.5439600 27.6911000 9.0632000 22.8479000 - 1402 1 2.7774700 20.0540000 1.2878000 28.4466000 - 1403 1 2.6947400 29.2179000 11.4253000 23.0334000 - 1404 1 2.6935300 23.0083000 7.1137800 28.4706000 - 1405 2 -1.5696700 21.4602000 2.0862100 28.7141000 - 1406 2 -1.5025700 21.7035000 6.4472600 28.4908000 - 1407 2 -1.5431600 23.0278000 3.5010600 27.7064000 - 1408 2 -1.5833200 29.3198000 3.4868500 17.0796000 - 1409 2 -1.5953400 28.3637000 2.4120500 19.2630000 - 1410 2 -1.5690400 29.5419000 6.4530000 16.8406000 - 1411 1 2.7312300 28.3076000 2.2311700 17.5328000 - 1412 1 2.6267500 29.6277000 1.1046600 19.5312000 - 1413 1 2.6423500 29.1234000 6.8571800 19.8536000 - 1414 1 2.7579800 29.4035000 4.8655700 16.1463000 - 1415 2 -1.4991500 25.6490000 4.5880100 25.5369000 - 1416 2 -1.6415100 29.6047000 2.0249300 23.5174000 - 1417 1 2.7659500 23.1259000 9.8602200 26.8019000 - 1418 2 -1.5531400 23.8081000 11.1621000 27.4215000 - 1419 1 2.6914300 20.4360000 13.1654000 29.2741000 - 1420 2 -1.5718300 21.9797000 12.0726000 28.8711000 - 1421 2 -1.6502000 22.4861000 8.5803800 28.1040000 - 1422 2 -1.5961000 20.5498000 14.1401000 27.9186000 - 1423 2 -1.5289900 21.6615000 10.1980000 25.8318000 - 1424 2 -1.6307800 15.5337000 8.0335700 28.9007000 - 1425 2 -1.6237700 25.8368000 6.6400100 18.0603000 - 1426 2 -1.5980300 26.5727000 8.1801700 16.5997000 - 1427 2 -1.6527900 27.6407000 12.9618000 18.1753000 - 1428 2 -1.5056800 28.9866000 10.1414000 19.6472000 - 1429 2 -1.5585600 26.4429000 13.6904000 20.4508000 - 1430 2 -1.5271200 27.5696000 10.7202000 16.7091000 - 1431 1 2.7479800 26.2846000 13.7161000 18.6782000 - 1432 1 2.7882400 27.1329000 9.6473000 15.6751000 - 1433 2 -1.4855900 24.0027000 13.7630000 27.7123000 - 1434 2 -1.6378800 26.7070000 9.0477100 21.1635000 - 1435 2 -1.6690200 15.4670000 14.7981000 27.0414000 - 1436 2 -1.6061100 27.7469000 12.2803000 23.5064000 - 1437 1 2.6322900 26.9116000 12.4943000 21.4971000 - 1438 1 2.7144900 27.6022000 9.4993000 19.4967000 - 1439 2 -1.6159900 27.7670000 5.4831000 16.0269000 - 1440 1 2.6976200 27.6004000 4.0371200 20.2777000 - 1441 2 -1.6588400 28.9004000 0.4032970 20.9447000 - 1442 2 -1.6635800 28.6293000 5.3302300 19.7522000 - 1443 2 -1.5632200 27.0997000 1.7340400 16.6739000 - 1444 1 2.6811200 27.5354000 7.0407200 17.3027000 - 1445 2 -1.6378900 28.0480000 7.7591900 18.9442000 - 1446 2 -1.6084400 27.9986000 3.8191700 21.9749000 - 1447 2 -1.5803400 28.4130000 7.7303500 23.2962000 - 1448 2 -1.5874400 26.7430000 2.4666000 23.9913000 - 1449 1 2.6332700 28.0402000 2.8317600 23.2773000 - 1450 1 2.7336900 23.6173000 1.8963800 27.3953000 - 1451 2 -1.5382200 28.8868000 10.0252000 22.5711000 - 1452 2 -1.5289700 29.7383000 14.7949000 21.0714000 - 1453 2 -1.6207400 26.5180000 9.3826900 23.9969000 - 1454 2 -1.6021400 26.2227000 4.5396500 19.5919000 - 1455 2 -1.6160600 26.3066000 0.6876380 29.6688000 - 1456 2 -1.5545600 28.0665000 9.9948400 29.3844000 - 1457 2 -1.5082400 28.3959000 9.1233000 14.9295000 - 1458 2 -1.4961700 26.4309000 6.3335700 29.5794000 - 1459 2 -1.5316100 15.1822000 1.0873000 23.4158000 - 1460 2 -1.5189500 23.3266000 0.3257770 27.4639000 - 1461 2 -1.6952400 30.1198000 8.9417200 17.4341000 - 1462 2 -1.5964700 15.8769000 5.6356800 14.9131000 - 1463 2 -1.6038200 30.1780000 11.3765000 24.2463000 - 1464 1 2.7288300 20.0565000 25.9652000 0.6946430 - 1465 2 -1.5588000 16.7586000 25.8750000 9.8688700 - 1466 1 2.7083100 18.0679000 26.5806000 9.3274100 - 1467 2 -0.5976980 20.4323000 22.3096000 14.5702000 - 1468 2 -1.5744000 15.5621000 18.0090000 6.9219500 - 1469 2 -0.6475310 15.7117000 22.6060000 1.0777300 - 1470 2 -1.5620900 20.1902000 29.1130000 12.1703000 - 1471 2 -1.5261900 19.8045000 17.6697000 6.8887400 - 1472 2 -1.6634000 26.8323000 29.6610000 2.5212600 - 1473 1 2.8133600 17.2732000 21.7800000 10.2833000 - 1474 2 -1.6242500 18.0470000 26.3351000 6.0839200 - 1475 2 -1.5231800 16.3156000 19.0899000 4.6411300 - 1476 2 -1.5833500 16.4732000 15.5300000 0.9460870 - 1477 1 2.8055000 17.4320000 15.6052000 7.0206900 - 1478 1 2.7804800 15.9183000 17.5823000 5.3843100 - 1479 1 2.7718500 25.1352000 26.6258000 5.7920900 - 1480 1 2.8901900 30.1603000 21.8026000 2.4002400 - 1481 2 -1.5016500 17.1029000 16.5119000 5.5828400 - 1482 1 2.7640100 15.3327000 16.2093000 11.2783000 - 1483 2 -1.5566500 15.4708000 18.2373000 2.0901100 - 1484 2 -1.5208900 17.8209000 16.3039000 8.3736900 - 1485 1 2.7169200 16.7586000 23.4633000 0.8954730 - 1486 2 -1.5337300 15.3958000 20.7498000 2.6226700 - 1487 2 -1.6216100 23.0996000 27.1385000 2.6431400 - 1488 1 2.8566700 16.3774000 19.4159000 2.9491000 - 1489 1 2.8052600 17.0671000 22.4667000 7.3100400 - 1490 1 2.6923800 19.5053000 26.1221000 5.4000800 - 1491 2 -1.5148700 17.9372000 18.7903000 2.7436700 - 1492 2 -1.5868500 16.3453000 22.4105000 8.8187800 - 1493 2 -1.4892200 16.4761000 21.3399000 6.3610900 - 1494 1 2.7554400 29.6463000 25.0036000 1.5314100 - 1495 2 -1.5633100 17.6168000 23.3230000 2.4698800 - 1496 1 2.8581200 16.4190000 18.9443000 9.8603100 - 1497 1 2.8521900 16.5432000 23.6342000 12.7884000 - 1498 1 2.8158900 15.2949000 25.6228000 10.6451000 - 1499 2 -1.5871200 17.8888000 18.9388000 8.7692200 - 1500 2 -1.5996100 15.5557000 22.7519000 13.9113000 - 1501 2 -1.5277400 15.7599000 24.9618000 12.1901000 - 1502 2 -1.5107800 17.0024000 22.8894000 11.4028000 - 1503 2 -1.5420200 16.1920000 17.5563000 10.9353000 - 1504 2 -1.5734800 29.9719000 16.8136000 2.4954300 - 1505 2 -1.5709400 18.7188000 22.2388000 7.4399700 - 1506 2 -1.5582800 17.9657000 23.8384000 13.6949000 - 1507 2 -1.5321800 16.1285000 25.5911000 4.3932300 - 1508 2 -1.4469900 18.4796000 18.1734000 0.3008800 - 1509 1 2.8237900 20.0356000 15.8624000 12.2322000 - 1510 1 2.8814700 18.9922000 19.0139000 1.5551300 - 1511 2 -1.5076200 19.9479000 22.1037000 2.6595900 - 1512 2 -1.6122300 18.1888000 20.2814000 0.6296350 - 1513 1 2.7978900 18.4370000 22.1173000 3.0573400 - 1514 1 2.8270600 19.0496000 18.9559000 7.4885200 - 1515 1 2.8248300 17.3029000 20.3936000 5.4342200 - 1516 2 -1.5855100 19.8681000 24.7374000 4.6900300 - 1517 2 -1.4912600 18.5683000 19.7673000 6.1715500 - 1518 2 -1.5704300 17.6353000 20.9871000 3.8902500 - 1519 1 2.7893300 19.7937000 21.8033000 8.5929800 - 1520 2 -0.6572500 19.3641000 19.1932000 12.6074000 - 1521 2 -1.4922100 19.0173000 16.2638000 10.9892000 - 1522 2 -1.5240100 18.9154000 21.6953000 9.9000900 - 1523 1 2.8118700 18.8971000 15.7935000 9.4061000 - 1524 2 -1.5442700 17.0055000 23.8521000 6.3918400 - 1525 1 2.7819200 16.6295000 25.3516000 5.9460800 - 1526 2 -1.4743700 16.7727000 20.3422000 10.7310000 - 1527 2 -1.5687900 19.0068000 25.3909000 8.6380700 - 1528 2 -0.6246890 20.1254000 23.5725000 0.6600390 - 1529 2 -1.5882500 18.7944000 27.9072000 10.3882000 - 1530 1 2.6506700 19.0962000 29.2226000 9.2738400 - 1531 1 2.6973100 18.8421000 28.1699000 12.0444000 - 1532 2 -1.6168000 20.7369000 27.2590000 4.7648100 - 1533 1 2.8606100 18.6316000 28.3078000 1.9924700 - 1534 2 -1.5932000 19.2730000 29.1660000 6.1392400 - 1535 2 -1.5600200 16.0500000 27.5159000 13.0975000 - 1536 1 2.7524400 21.8822000 27.9543000 9.0358500 - 1537 1 2.7698200 17.2220000 26.9885000 13.9936000 - 1538 1 2.6220500 29.8726000 21.1942000 8.9297000 - 1539 2 -1.6580800 16.3208000 29.1784000 10.6626000 - 1540 2 -1.5657800 18.5554000 26.9717000 13.0266000 - 1541 2 -1.5648400 17.9165000 28.1349000 8.4076900 - 1542 1 2.6220700 16.6152000 28.6859000 7.6709600 - 1543 2 -1.6087100 15.2304000 28.1657000 4.2372900 - 1544 2 -1.5956000 15.4382000 26.8051000 2.0124800 - 1545 2 -1.5552800 17.7212000 27.5804000 3.3170300 - 1546 2 -1.5732200 20.0868000 27.4181000 1.9176800 - 1547 2 -1.5224900 17.5252000 28.5696000 0.7543700 - 1548 1 2.7140700 16.2705000 26.8883000 3.4019900 - 1549 2 -0.6501390 18.7680000 26.1858000 1.2362400 - 1550 1 2.8380300 28.5278000 20.2176000 0.0551573 - 1551 2 -1.5339300 25.2497000 23.3023000 10.7806000 - 1552 1 2.8118400 21.9880000 16.3723000 8.7424000 - 1553 1 2.8670600 20.9876000 16.7093000 5.8579300 - 1554 2 -1.3898700 20.3283000 15.4611000 5.3702800 - 1555 2 -1.5477000 20.4697000 15.7160000 8.9334300 - 1556 2 -1.5354400 22.2324000 16.6241000 7.1109400 - 1557 2 -1.5793600 23.2497000 15.5218000 9.3654000 - 1558 2 -1.5337500 24.8784000 18.5948000 0.4315710 - 1559 1 2.8084600 20.9329000 22.5777000 1.2178700 - 1560 2 -1.5135100 23.0109000 18.2870000 2.3350900 - 1561 2 -0.6330400 20.7880000 25.0717000 1.3506700 - 1562 2 -1.4788300 21.1137000 21.1308000 0.3711060 - 1563 1 2.8686900 21.5704000 17.6718000 2.8421700 - 1564 1 2.7834600 23.5383000 24.4319000 7.3227200 - 1565 1 2.7978000 21.3384000 23.7825000 5.2899900 - 1566 1 2.7237200 21.9236000 19.5164000 8.3583400 - 1567 2 -1.6113800 20.3923000 20.1126000 8.2053500 - 1568 2 -1.5764800 23.0441000 19.9026000 7.3504800 - 1569 2 -1.4000600 21.3372000 22.3801000 5.5038500 - 1570 2 -1.5021900 24.1821000 23.2713000 6.4654300 - 1571 2 -1.4924800 21.4420000 17.6563000 4.5534000 - 1572 2 -1.4335000 20.4721000 18.9074000 2.1501600 - 1573 2 -1.4954500 22.3852000 23.2166000 2.1198000 - 1574 2 -1.5776500 24.0588000 19.1457000 11.8697000 - 1575 1 2.7207900 22.7877000 23.7063000 13.1086000 - 1576 1 2.7613900 20.7515000 18.9097000 12.7960000 - 1577 1 2.7658900 20.5449000 24.8473000 8.4626600 - 1578 2 -1.5217200 21.5702000 19.8664000 11.6524000 - 1579 2 -1.5609500 20.8015000 23.2065000 8.7896700 - 1580 2 -1.6127400 21.8985000 24.9192000 13.9370000 - 1581 2 -1.5070900 22.7750000 23.8985000 11.5651000 - 1582 2 -1.6365900 15.5778000 24.8177000 0.3700820 - 1583 2 -1.6097000 22.1281000 18.1029000 9.2442000 - 1584 2 -1.5077200 23.5709000 24.0220000 8.8921600 - 1585 2 -1.5116300 20.9967000 17.2441000 12.4491000 - 1586 2 -1.5565200 21.3196000 19.0560000 14.2920000 - 1587 2 -1.5455100 24.2943000 25.8553000 6.9974500 - 1588 2 -1.6727800 21.1376000 25.0014000 6.7786000 - 1589 2 -1.6358600 22.5665000 24.7850000 4.3459500 - 1590 2 -1.6076400 21.3531000 26.4024000 9.3873500 - 1591 2 -1.5486700 24.6353000 15.0019000 1.0189200 - 1592 1 2.8295800 24.6013000 16.8024000 4.5264100 - 1593 2 -1.5439500 23.5782000 15.5832000 4.1140000 - 1594 1 2.8506400 24.0092000 19.3552000 1.6812600 - 1595 2 -1.4533000 23.4051000 20.5109000 0.7707100 - 1596 2 -1.3208000 21.2487000 16.4031000 1.9917000 - 1597 1 2.8199900 23.6392000 23.6509000 3.0696600 - 1598 1 2.8113900 24.3077000 19.3015000 6.3386600 - 1599 1 2.8604300 25.2034000 22.3927000 5.3121200 - 1600 2 -1.5338100 24.9272000 24.5615000 2.5922200 - 1601 2 -1.5177200 25.1198000 20.6633000 5.9154100 - 1602 2 -1.4842100 24.0469000 18.2171000 5.0500300 - 1603 2 -1.5155000 24.2905000 22.4982000 3.9791500 - 1604 1 2.8243500 23.6869000 23.1580000 10.2749000 - 1605 1 2.7618800 22.9405000 20.6393000 11.6910000 - 1606 2 -1.5715800 23.2341000 21.5077000 10.3134000 - 1607 2 -1.5925800 23.1617000 21.9365000 13.0257000 - 1608 2 -1.5252100 25.8477000 16.3845000 5.5711900 - 1609 2 -1.5733800 25.1136000 25.6901000 12.3725000 - 1610 2 -1.5089600 25.4109000 25.6685000 4.6407200 - 1611 1 2.6712500 22.3756000 26.5905000 4.1363200 - 1612 2 -1.6170800 28.3005000 28.8315000 7.7523800 - 1613 2 -1.5264900 29.3965000 28.4964000 12.5728000 - 1614 1 2.7870700 28.0877000 28.6461000 11.5743000 - 1615 1 2.7353200 29.6573000 26.9479000 13.4724000 - 1616 2 -1.6083000 28.5657000 27.4234000 9.8155900 - 1617 1 2.6822400 29.3337000 27.8313000 8.4865200 - 1618 2 -1.5695400 23.2887000 28.5107000 10.0626000 - 1619 2 -1.5100000 25.7179000 27.6471000 14.2938000 - 1620 2 -1.5602000 26.3030000 27.6683000 6.1603300 - 1621 2 -1.5130400 28.0974000 26.7255000 14.1509000 - 1622 2 -1.6591600 27.7603000 29.4103000 14.6250000 - 1623 2 -1.6013700 26.7972000 27.6069000 11.4972000 - 1624 1 2.7803500 25.2595000 27.2786000 12.6929000 - 1625 1 2.6904100 26.3626000 28.3047000 7.5747200 - 1626 1 2.7251600 27.0814000 27.9988000 1.9478100 - 1627 2 -1.6075300 27.8702000 27.9940000 3.5315700 - 1628 1 2.7118900 28.7816000 26.6038000 3.9552900 - 1629 2 -1.6122900 29.6340000 26.4151000 2.4786600 - 1630 1 2.7984200 27.2741000 26.2917000 9.6312100 - 1631 1 2.7687000 29.9619000 17.1400000 10.7576000 - 1632 2 -1.5561800 26.2307000 27.2065000 8.6913000 - 1633 2 -1.6245800 24.6986000 29.3728000 7.1752600 - 1634 2 -1.5586800 29.3503000 22.4762000 12.6187000 - 1635 2 -1.5521800 25.3957000 19.7222000 2.6712200 - 1636 2 -1.5701700 29.6992000 15.5304000 10.5347000 - 1637 2 -1.5749600 25.3416000 18.3294000 7.3070000 - 1638 2 -1.6274900 15.9993000 14.9251000 12.2303000 - 1639 2 -1.5958200 28.0398000 25.1045000 8.0875100 - 1640 2 -1.4649700 28.9016000 19.1028000 1.1492700 - 1641 1 2.5007100 24.2937000 17.7528000 11.3529000 - 1642 1 2.8109400 28.3514000 23.5478000 4.3460100 - 1643 2 -1.5824800 28.7583000 20.9195000 10.0819000 - 1644 1 2.6580700 27.1825000 20.6250000 10.6534000 - 1645 2 -1.5668800 25.8239000 27.0126000 2.2463200 - 1646 2 -1.5660700 29.6490000 19.5719000 5.9129300 - 1647 1 2.7731000 28.4800000 20.3533000 4.6955400 - 1648 1 2.7285400 29.1871000 18.5082000 7.1385200 - 1649 2 -1.6555900 21.9274000 29.5755000 5.1211500 - 1650 1 2.7786200 28.8398000 17.5516000 1.4260900 - 1651 2 -1.5890700 29.4134000 16.9974000 6.7295700 - 1652 1 2.7430500 29.0120000 15.3968000 6.6627100 - 1653 2 -1.5884500 29.3475000 22.4789000 8.0594800 - 1654 2 -1.5392600 27.1163000 25.0641000 10.6714000 - 1655 2 -1.6005800 27.6230000 25.2799000 3.7884000 - 1656 1 2.6910900 27.3272000 29.7016000 4.1619500 - 1657 2 -1.5760200 27.0664000 21.1838000 12.1620000 - 1658 2 -1.6008000 29.8748000 26.8318000 5.1790000 - 1659 2 -1.5980300 28.3081000 17.6129000 10.5151000 - 1660 2 -1.6107200 25.5896000 16.8367000 11.2609000 - 1661 2 -0.6492430 27.9023000 20.9765000 5.9142500 - 1662 2 -1.5516400 26.7880000 23.6581000 13.0201000 - 1663 2 -1.5448700 28.8263000 18.2363000 14.2747000 - 1664 1 2.7771100 26.0554000 24.4464000 11.7194000 - 1665 1 2.6609600 27.7956000 22.5400000 12.8351000 - 1666 1 2.7375200 26.4263000 18.8167000 8.3879300 - 1667 2 -1.4838800 29.0309000 23.7449000 2.4109100 - 1668 2 -1.4255900 28.3931000 21.8900000 3.8179400 - 1669 2 -1.5515800 27.9339000 19.0820000 7.8957200 - 1670 2 -1.5392900 29.0728000 24.1013000 5.7296200 - 1671 2 -1.5718800 27.5511000 19.0338000 4.2324200 - 1672 1 2.7705900 26.0856000 25.4214000 3.2322700 - 1673 1 2.8375400 29.2187000 24.2054000 7.3772600 - 1674 1 2.7796000 26.5288000 18.5928000 3.0614800 - 1675 2 -1.5994000 27.1953000 17.5072000 1.9735600 - 1676 2 -1.5970000 27.7169000 27.9845000 0.5947180 - 1677 1 2.6094300 26.9383000 16.7874000 10.2475000 - 1678 1 2.7287200 25.7676000 16.0142000 1.7372700 - 1679 2 -1.5826400 27.5215000 15.1793000 7.3461300 - 1680 2 -1.5398900 26.4346000 17.0972000 8.7950300 - 1681 1 2.7778100 25.9658000 15.9468000 7.1811700 - 1682 2 -1.6240500 26.1716000 19.7754000 9.7095800 - 1683 2 -1.5811900 29.9708000 25.7977000 12.2254000 - 1684 2 -1.4603000 26.5707000 23.1201000 5.2441500 - 1685 2 -1.5178700 29.3425000 21.6057000 0.8175430 - 1686 2 -1.5848400 25.4619000 16.9646000 3.0748000 - 1687 1 2.7659000 23.5091000 29.3379000 11.4032000 - 1688 2 -1.5623500 23.6608000 27.4940000 5.2119100 - 1689 1 2.6644300 23.2792000 28.6676000 6.2398600 - 1690 2 -1.6818400 23.8058000 29.2880000 0.6326190 - 1691 2 -1.5303300 30.1231000 20.4179000 3.3202100 - 1692 1 2.7343900 24.3916000 27.3755000 1.5324600 - 1693 2 -1.5134500 23.9308000 28.1103000 12.4986000 - 1694 1 2.6373500 20.3508000 28.7861000 5.1436000 - 1695 2 -1.5643300 22.2699000 28.0108000 7.4372700 - 1696 2 -1.6191100 20.7812000 29.3594000 9.2062900 - 1697 2 -1.5438400 28.4860000 24.9218000 0.1441480 - 1698 2 -1.5950200 15.3144000 25.9521000 7.0561500 - 1699 1 2.6588900 17.5102000 16.7816000 0.1345940 - 1700 2 -1.4290900 24.4646000 27.3008000 0.0127936 - 1701 2 -1.5355400 29.5412000 29.5330000 0.5814070 - 1702 1 2.7351900 22.0308000 29.6567000 1.0469600 - 1703 2 -1.4592600 28.4193000 14.9715000 13.7506000 - 1704 2 -1.6024100 19.1656000 14.9492000 13.4588000 - 1705 2 -1.5631000 15.1187000 19.2133000 8.9756200 - 1706 2 -1.5901300 23.9316000 23.7575000 14.8738000 - 1707 2 -1.4221100 25.3473000 23.2329000 0.0271609 - 1708 1 2.7234100 24.4494000 26.0610000 18.9594000 - 1709 1 2.7437800 23.0430000 26.8255000 21.7989000 - 1710 2 -1.5506700 17.6299000 23.3324000 29.4037000 - 1711 2 -1.6198100 21.9868000 27.1098000 15.9963000 - 1712 2 -1.5494500 22.7694000 15.2742000 17.7913000 - 1713 1 2.7225700 26.5886000 22.5327000 29.6399000 - 1714 1 2.7937500 19.3379000 26.6224000 19.0017000 - 1715 2 -1.5901900 18.2998000 17.4973000 25.3077000 - 1716 2 -1.5801700 25.3121000 28.9050000 20.4226000 - 1717 1 2.6387400 15.6527000 16.4066000 23.3154000 - 1718 1 2.8325100 17.3000000 16.9492000 18.8887000 - 1719 2 -1.6259200 16.9073000 15.3715000 23.0945000 - 1720 2 -1.4665500 17.2438000 15.8157000 17.6676000 - 1721 1 2.6779200 16.9298000 15.2541000 21.4344000 - 1722 2 -1.5536900 17.0281000 25.7796000 14.9671000 - 1723 1 2.7916000 16.4646000 24.7876000 22.6119000 - 1724 2 -1.4815600 19.5153000 22.2425000 22.2230000 - 1725 2 -1.5293500 16.5240000 24.2795000 24.4044000 - 1726 2 -1.5134600 17.4602000 16.3924000 20.5674000 - 1727 2 -1.5486400 16.0136000 17.9227000 23.2845000 - 1728 2 -1.4574900 15.7182000 17.4461000 18.9642000 - 1729 2 -1.5073900 22.9366000 29.6366000 16.5399000 - 1730 2 -1.4764300 18.9113000 23.7048000 19.8628000 - 1731 2 -1.5398700 17.7205000 21.6859000 24.3608000 - 1732 2 -1.3640600 19.2002000 17.8245000 15.2444000 - 1733 1 2.8051800 17.6741000 24.0280000 18.3757000 - 1734 1 2.7679000 15.8773000 25.3911000 16.1427000 - 1735 2 -0.6431340 17.8116000 22.5968000 17.9743000 - 1736 2 -1.4254200 16.3252000 24.3094000 18.9730000 - 1737 2 -1.4749800 18.4568000 18.0666000 18.5543000 - 1738 2 -1.5172100 17.1496000 24.3064000 16.4640000 - 1739 2 -1.4959900 17.3832000 23.6580000 22.0464000 - 1740 1 2.7105300 16.7731000 22.8206000 24.9102000 - 1741 2 -1.6089000 15.6401000 26.7563000 17.0355000 - 1742 1 2.7407200 15.1384000 21.7936000 27.9472000 - 1743 2 -1.4888100 16.9950000 26.2185000 22.3266000 - 1744 2 -1.6423400 29.6347000 15.3556000 16.1389000 - 1745 1 2.8502300 18.9755000 23.6286000 21.4231000 - 1746 2 -1.5645500 19.8702000 24.9908000 22.0878000 - 1747 1 2.8190500 19.4502000 21.9725000 23.8715000 - 1748 2 -1.5664300 26.3636000 14.9023000 27.1220000 - 1749 1 2.8357300 20.0736000 17.9806000 18.4157000 - 1750 2 -1.5556200 19.6549000 25.4192000 15.5288000 - 1751 2 -1.5491800 21.9293000 28.8384000 29.4296000 - 1752 2 -0.6023630 19.7859000 22.6031000 15.7315000 - 1753 2 -1.4687600 18.5805000 25.5660000 18.2086000 - 1754 1 2.7535300 19.7160000 16.6493000 25.3098000 - 1755 2 -1.2615000 19.7500000 23.2534000 24.4643000 - 1756 2 -1.5905200 17.5246000 26.3335000 25.3894000 - 1757 1 2.7817700 16.0400000 25.1622000 25.5963000 - 1758 2 -1.4872500 15.1975000 19.9753000 19.4442000 - 1759 2 -1.5394900 16.0305000 24.9366000 27.1917000 - 1760 1 2.7807300 15.8105000 15.9487000 28.2658000 - 1761 2 -1.5105400 17.7628000 15.6896000 28.9249000 - 1762 2 -1.6178200 15.4524000 22.6604000 25.6839000 - 1763 1 2.7466800 17.3689000 18.1454000 26.5362000 - 1764 1 2.7498000 15.2003000 18.8629000 24.5856000 - 1765 2 -1.5500000 16.0788000 20.7345000 28.7343000 - 1766 2 -1.5497000 16.7881000 17.0675000 27.4830000 - 1767 2 -1.5776600 16.1766000 19.0406000 25.8122000 - 1768 2 -1.4598900 19.5701000 21.4844000 26.9576000 - 1769 1 2.7211000 19.8987000 20.1623000 27.4293000 - 1770 1 2.7550200 17.3983000 19.6643000 29.1111000 - 1771 2 -1.5742100 18.3281000 19.4133000 27.6999000 - 1772 1 2.6716000 15.7127000 28.6770000 26.2989000 - 1773 2 -1.4353300 16.4432000 18.1206000 29.4736000 - 1774 1 2.7969000 19.7305000 28.7349000 27.1244000 - 1775 2 -1.6389600 16.2032000 28.7356000 27.8287000 - 1776 2 -1.5654700 18.6955000 29.1025000 25.8488000 - 1777 2 -1.4988500 16.6269000 28.5134000 24.9280000 - 1778 2 -1.5381700 18.5814000 27.1906000 20.2243000 - 1779 2 -1.5624300 19.3356000 28.0850000 17.6411000 - 1780 1 2.6259200 18.5201000 28.0917000 16.4433000 - 1781 2 -1.2874600 16.9394000 28.7880000 21.5721000 - 1782 1 2.8079400 18.0921000 27.8717000 24.7131000 - 1783 2 -1.5730000 17.5445000 28.2762000 15.2978000 - 1784 2 -1.5810500 15.7945000 27.0594000 19.9228000 - 1785 1 2.8701900 17.0171000 27.3993000 21.0370000 - 1786 2 -1.5134100 19.3068000 27.5237000 23.6543000 - 1787 2 -1.5911400 22.0655000 26.2337000 22.9608000 - 1788 1 2.7382700 22.2929000 21.2797000 24.6216000 - 1789 2 -1.5693200 25.1567000 23.6442000 24.6059000 - 1790 2 -1.5690500 23.2076000 25.7750000 20.2207000 - 1791 1 2.7167200 25.5873000 16.5321000 23.5169000 - 1792 1 2.7856100 20.2835000 26.0463000 23.4478000 - 1793 2 -0.6276730 20.3460000 25.5230000 24.7059000 - 1794 2 -1.6318100 24.5195000 24.1965000 22.2431000 - 1795 2 -1.5852300 22.7286000 22.7202000 23.7850000 - 1796 1 2.8141500 21.4463000 16.1273000 22.4853000 - 1797 2 -1.5624900 20.5914000 15.5168000 21.2068000 - 1798 2 -1.6188200 23.6055000 21.5423000 16.6485000 - 1799 1 2.7949600 21.3636000 15.6706000 17.0158000 - 1800 2 -1.5424500 21.5802000 16.7514000 15.7691000 - 1801 1 2.7758200 22.1891000 18.2541000 20.5220000 - 1802 1 2.6685100 23.8440000 23.8033000 23.6226000 - 1803 2 -1.5783400 23.0005000 19.8176000 20.8410000 - 1804 2 -1.5447300 20.7617000 20.8122000 24.3625000 - 1805 2 -1.5952500 20.3317000 16.4804000 23.7357000 - 1806 2 -1.6238000 24.6338000 24.3342000 18.4077000 - 1807 2 -1.5149700 22.1901000 17.4942000 21.9474000 - 1808 2 -1.5225100 20.7795000 18.4989000 19.8599000 - 1809 2 -1.5522800 24.5781000 21.6111000 19.1943000 - 1810 1 2.8132400 20.6589000 18.1554000 15.4946000 - 1811 1 2.7220100 21.7864000 24.9429000 19.7185000 - 1812 2 -1.5167500 21.9169000 23.4803000 19.7124000 - 1813 2 -1.5481500 23.3702000 17.5667000 19.4464000 - 1814 2 -1.5106100 20.7464000 18.7764000 17.1385000 - 1815 2 -1.5119000 20.4717000 16.3960000 18.1852000 - 1816 2 -1.4915600 19.3421000 29.2751000 28.7286000 - 1817 1 2.7559400 21.2474000 25.3530000 15.4344000 - 1818 2 -1.6025000 25.9150000 15.3369000 22.4896000 - 1819 2 -1.5838400 25.1448000 15.6362000 15.5842000 - 1820 2 -1.5957800 20.9449000 26.1669000 19.1010000 - 1821 1 2.6477300 23.7926000 20.3993000 22.0795000 - 1822 2 -1.5811300 23.3987000 20.1221000 23.6340000 - 1823 2 -1.5857500 24.6742000 17.7720000 23.3579000 - 1824 2 -1.6199100 25.2325000 21.2791000 22.0245000 - 1825 2 -1.5874800 22.1431000 24.2504000 16.5013000 - 1826 1 2.6156600 23.0281000 22.9918000 16.0634000 - 1827 1 2.6346300 24.8035000 17.9335000 19.0367000 - 1828 1 2.5761000 24.0479000 20.6541000 18.0195000 - 1829 1 2.5965800 24.9644000 23.1437000 19.4245000 - 1830 2 -1.6110800 24.8054000 19.1890000 17.9618000 - 1831 1 2.7343200 24.1221000 18.7209000 24.5461000 - 1832 2 -1.5085300 25.9220000 26.3829000 27.1568000 - 1833 1 2.5926000 29.2234000 27.6324000 25.0770000 - 1834 2 -1.6054400 25.8262000 29.1517000 26.7751000 - 1835 2 -1.5548500 27.8464000 28.3431000 27.9036000 - 1836 1 2.7780300 26.7386000 27.7197000 26.9393000 - 1837 2 -1.5743600 27.6927000 27.7104000 25.2983000 - 1838 2 -1.4248100 21.9084000 24.8658000 27.0244000 - 1839 1 2.6739800 26.4925000 27.0682000 22.5948000 - 1840 2 -1.6028000 29.6489000 27.6886000 18.3826000 - 1841 1 2.7186300 28.4348000 28.7510000 17.7921000 - 1842 1 2.4066300 29.2464000 29.0151000 22.1333000 - 1843 2 -1.5512000 27.3190000 28.7672000 19.2261000 - 1844 2 -1.6171000 27.5456000 27.9576000 16.6699000 - 1845 2 -1.6067700 29.7769000 28.6816000 23.7754000 - 1846 2 -1.5689100 25.7610000 26.4068000 19.7140000 - 1847 2 -1.6624800 27.4182000 28.2962000 21.6930000 - 1848 1 2.7515400 27.0542000 27.3290000 20.0539000 - 1849 1 2.8004500 27.2379000 19.5612000 25.0325000 - 1850 1 2.6249400 27.8635000 29.2682000 29.3643000 - 1851 2 -1.5919900 25.6320000 19.0417000 25.2433000 - 1852 2 -1.5859900 29.5679000 19.8755000 18.9568000 - 1853 2 -1.6043400 16.5987000 15.3176000 15.0397000 - 1854 2 -1.4821900 28.0393000 24.9199000 27.0056000 - 1855 1 2.8465800 29.3744000 24.7208000 28.5189000 - 1856 1 2.6909900 29.9601000 18.3529000 19.2801000 - 1857 2 -1.5021800 22.5962000 28.0105000 20.9372000 - 1858 2 -1.5808400 28.2129000 16.7190000 26.8622000 - 1859 2 -1.4196900 27.5036000 18.9696000 28.7483000 - 1860 1 2.7480200 18.2979000 24.4462000 15.1534000 - 1861 2 -1.4810900 26.8127000 23.0379000 28.1001000 - 1862 2 -1.4519000 28.2648000 19.7489000 26.0691000 - 1863 1 2.8001500 25.7121000 24.9648000 27.9116000 - 1864 1 2.7504600 25.6185000 19.1452000 28.9080000 - 1865 1 2.7793800 28.4993000 24.6928000 25.6137000 - 1866 1 2.7968700 27.2009000 15.7797000 26.0384000 - 1867 2 -1.5614500 25.8659000 21.4923000 26.0493000 - 1868 1 2.7631000 27.7484000 17.5548000 28.1275000 - 1869 2 -1.4491500 29.7402000 20.5775000 28.6620000 - 1870 2 -1.5016100 25.9835000 17.7502000 28.1171000 - 1871 1 2.7337600 26.4017000 22.6610000 25.0665000 - 1872 2 -1.5166300 27.6861000 23.6142000 24.9010000 - 1873 2 -1.6029400 28.3676000 26.4757000 20.1857000 - 1874 1 2.7291100 29.6409000 26.4883000 19.3501000 - 1875 2 -1.5057400 24.2593000 24.2725000 27.5786000 - 1876 2 -1.4206700 19.8879000 26.2710000 28.9586000 - 1877 2 -1.6109600 26.9447000 25.4676000 22.5098000 - 1878 2 -1.6122400 23.9391000 27.1929000 17.8048000 - 1879 1 2.7447700 29.0969000 20.9056000 17.7346000 - 1880 2 -1.6115100 29.7328000 16.3178000 23.5258000 - 1881 1 2.4935300 24.9117000 24.8505000 15.2773000 - 1882 1 2.7704900 29.6673000 18.0631000 23.8988000 - 1883 1 2.7330500 17.4155000 29.4070000 29.1357000 - 1884 2 -1.6391300 24.8313000 27.1092000 22.6739000 - 1885 2 -1.5760500 28.6732000 23.5912000 15.5962000 - 1886 2 -1.5559800 27.1448000 17.4169000 16.4002000 - 1887 2 -1.6372300 27.6903000 21.8066000 18.3555000 - 1888 2 -1.5999800 27.8711000 24.3828000 18.0780000 - 1889 2 -1.5639700 28.7344000 17.8019000 20.3921000 - 1890 2 -1.5891800 26.3019000 23.4910000 20.2644000 - 1891 2 -1.5890900 28.5883000 19.9284000 16.3751000 - 1892 1 2.6614000 27.7114000 24.5173000 16.4616000 - 1893 1 2.6956400 27.7767000 23.0860000 19.3046000 - 1894 1 2.7259400 28.3368000 18.3813000 15.7647000 - 1895 2 -1.5939600 28.7605000 22.9766000 20.6896000 - 1896 2 -0.6393120 26.0485000 24.9582000 29.2617000 - 1897 2 -1.5848600 26.9833000 21.1376000 23.9467000 - 1898 2 -1.5885200 28.0747000 18.4875000 23.8507000 - 1899 2 -1.6412400 27.4808000 20.0296000 21.4115000 - 1900 1 2.4565900 26.1425000 24.1935000 21.7553000 - 1901 1 2.6066100 27.4172000 18.5455000 20.7908000 - 1902 2 -1.5962200 26.4796000 15.3269000 18.0171000 - 1903 2 -1.5749300 22.9872000 18.3367000 25.6630000 - 1904 1 2.7881600 21.9945000 17.9103000 27.0993000 - 1905 1 2.7390500 23.4668000 29.1575000 20.2769000 - 1906 2 -1.5932100 21.3566000 19.5932000 27.7491000 - 1907 1 2.7409200 27.2493000 27.8669000 14.9528000 - 1908 2 -1.5833100 22.4028000 22.6256000 28.9966000 - 1909 2 -1.3541000 22.6735000 17.2538000 28.1980000 - 1910 2 -1.5861100 20.6189000 17.1928000 26.4818000 - 1911 2 -1.5785400 23.1385000 21.5391000 26.0893000 - 1912 2 -1.5366100 26.3136000 16.3401000 24.8692000 - 1913 2 -1.5800000 29.4367000 25.8624000 24.6939000 - 1914 2 -1.4426700 22.0914000 25.5118000 28.9885000 - 1915 2 -1.5934600 26.1973000 25.0337000 16.0314000 - 1916 1 2.5791100 26.7665000 20.9603000 22.4308000 - 1917 2 -1.6332100 25.9734000 17.9660000 20.2702000 - 1918 1 2.6982600 26.3502000 15.9115000 16.5495000 - 1919 1 2.8317600 22.7861000 24.1025000 28.2105000 - 1920 1 2.7515700 23.2378000 28.5598000 28.4939000 - 1921 1 2.8550200 21.4710000 26.3401000 27.6197000 - 1922 2 -1.4521200 22.8784000 27.4462000 27.4599000 - 1923 2 -1.5408200 20.4716000 27.4224000 26.8188000 - 1924 1 2.7135700 24.5394000 21.3385000 26.9000000 - 1925 1 2.7784800 22.5958000 20.8765000 29.1314000 - 1926 2 -1.6258500 24.2723000 20.4589000 28.1661000 - 1927 1 2.7014300 22.6538000 28.1185000 17.1514000 - 1928 2 -1.5193500 29.9313000 23.2837000 28.0370000 - 1929 2 -1.6286000 30.1017000 23.7147000 22.8844000 - 1930 2 -1.5297100 30.1131000 26.0786000 27.7798000 - 1931 2 -1.6834200 30.1146000 17.3313000 18.0018000 - 1932 2 -1.5867300 22.5791000 14.9189000 23.1016000 - 1933 2 -1.5983200 26.8673000 20.8701000 29.7422000 - 1934 1 2.7865500 16.3230000 14.8887000 16.6060000 - 1935 1 2.7080200 29.7673000 14.8924000 24.8043000 - 1936 2 -1.6319200 28.4285000 16.5193000 29.7385000 - 1937 2 -1.5231200 23.9733000 14.8914000 20.4311000 - 1938 1 2.6090400 37.9965000 3.8393000 4.9080200 - 1939 2 -1.6389400 36.0954000 7.1795400 12.7777000 - 1940 2 -1.6428800 33.2335000 14.5415000 7.6825200 - 1941 2 -1.6264600 38.7765000 6.9239700 4.3574800 - 1942 2 -1.6510700 38.0821000 1.1906800 2.6537800 - 1943 2 -1.6703000 36.9293000 7.9142900 9.1975500 - 1944 1 2.5859900 34.5306000 2.4366300 12.8029000 - 1945 3 0.3114410 40.7420000 14.6228000 10.5978000 - 1946 2 -1.7088500 34.5258000 12.8462000 9.9588300 - 1947 1 2.6664400 32.4384000 9.0103600 11.7044000 - 1948 3 0.2513860 40.7052000 14.7368000 7.8220500 - 1949 2 -1.6375500 33.1006000 11.5976000 7.8204200 - 1950 2 -1.6094000 33.9608000 12.2287000 5.3365900 - 1951 1 2.7940800 32.6164000 13.2285000 4.4538600 - 1952 2 -1.5720300 34.9531000 13.8082000 2.1548300 - 1953 1 2.6966700 36.7235000 14.7692000 7.1664500 - 1954 2 -1.6146000 32.6519000 4.6132600 9.3082900 - 1955 2 -1.5905200 33.5509000 7.0203700 11.7173000 - 1956 2 -1.6163200 33.1199000 1.6599700 12.9482000 - 1957 1 2.7588900 31.6741000 2.3934100 11.9719000 - 1958 1 2.5623900 34.5868000 6.5810000 12.7268000 - 1959 2 -1.6420900 34.2216000 4.7171600 5.0699000 - 1960 2 -1.6390200 34.8959000 3.3204300 8.9127900 - 1961 2 -1.6219900 34.1347000 1.9099700 5.7635400 - 1962 2 -1.6897200 33.4810000 7.2137700 3.3776700 - 1963 1 2.2768000 31.8465000 10.8196000 8.4102000 - 1964 1 2.6813800 32.9222000 2.9578000 5.1404500 - 1965 1 2.6547300 34.2155000 0.9141610 6.9682800 - 1966 1 2.6865700 33.5051000 3.4450000 10.0730000 - 1967 1 2.5483800 34.1432000 5.7950700 3.7067800 - 1968 1 2.7080000 34.1851000 3.7681600 0.8977110 - 1969 2 -1.6322900 30.9109000 10.2518000 14.5707000 - 1970 2 -1.6437200 30.3387000 10.6098000 7.9784200 - 1971 2 -1.5868500 31.4154000 10.5966000 11.4214000 - 1972 2 -1.6036400 33.4781000 12.6725000 12.7910000 - 1973 2 -1.5062500 32.9660000 9.0908900 13.3040000 - 1974 2 -1.5207600 31.2023000 13.8337000 14.6290000 - 1975 2 -1.6196100 31.3330000 8.3666300 10.2494000 - 1976 1 2.8071900 32.6820000 14.1148000 14.0633000 - 1977 3 -0.0429721 44.2622000 14.8590000 13.1849000 - 1978 1 2.3410500 31.6837000 11.7661000 10.5079000 - 1979 2 -1.4978600 32.2526000 10.1889000 3.5570200 - 1980 2 -1.6571100 31.9872000 8.4788100 5.4587100 - 1981 2 -1.5915200 31.6736000 13.1550000 5.7834700 - 1982 2 -1.6329500 33.8463000 9.5988800 10.6402000 - 1983 2 -1.6420900 31.3378000 13.2480000 10.1992000 - 1984 2 -1.5834400 31.9049000 12.9142000 2.8570400 - 1985 1 2.6727800 31.7280000 14.1748000 6.9681300 - 1986 1 2.6988300 32.9915000 8.9996200 4.1840600 - 1987 2 -1.4812700 32.4777000 13.1039000 0.5060130 - 1988 3 0.2557590 40.8472000 14.6342000 5.4181100 - 1989 2 -1.5802800 34.5217000 9.4141800 4.4896800 - 1990 1 2.7492000 30.9860000 9.9208900 2.7593900 - 1991 1 2.6023300 31.4499000 13.8576000 1.1735600 - 1992 2 -1.6165100 33.1676000 4.5909500 14.5497000 - 1993 2 -1.5741700 30.3434000 14.7165000 7.4653500 - 1994 2 -1.5483100 31.6252000 3.9295900 12.4375000 - 1995 2 -1.5760700 31.1635000 6.3686500 13.0928000 - 1996 2 -1.3891200 35.8251000 14.7569000 8.5018600 - 1997 2 -1.6117100 32.3343000 2.2312400 10.4191000 - 1998 1 2.8066300 31.6801000 4.8324900 13.8038000 - 1999 1 2.7491400 31.8627000 0.8764730 9.2976600 - 2000 2 -1.9382200 33.4187000 4.9258800 2.0586300 - 2001 1 2.4772800 36.1388000 0.8574980 0.2884200 - 2002 2 -1.6249200 32.3044000 3.6036300 6.5693800 - 2003 2 -1.6139200 31.7767000 6.3326100 7.2350000 - 2004 2 -1.6347000 33.1128000 0.9263920 8.1760700 - 2005 2 -1.6422100 30.5877000 5.9690700 4.8775200 - 2006 2 -1.6009200 32.1364000 2.3590200 3.8971000 - 2007 1 2.7262500 30.8742000 7.1476400 5.9948100 - 2008 1 2.7386100 31.6157000 4.6581600 7.7357900 - 2009 1 2.7274400 31.8479000 0.6875680 3.8345900 - 2010 1 2.6591700 34.2275000 0.4923590 2.0893700 - 2011 2 -1.9265700 31.2460000 6.1715400 0.5489690 - 2012 2 -1.5664800 30.5182000 4.4989700 2.7780500 - 2013 2 -1.9276000 33.3882000 2.1301600 0.6932290 - 2014 1 2.7055400 31.5849000 5.1266300 1.9432900 - 2015 2 -1.6527300 37.0429000 0.0661035 5.4231200 - 2016 1 2.7413500 34.2404000 14.4707000 9.0759900 - 2017 1 2.5145300 30.9128000 6.9688100 10.1030000 - 2018 2 -1.6024500 31.0174000 0.2296730 2.1852400 - 2019 2 -1.7187300 31.7446000 9.0216500 1.3988400 - 2020 2 -1.6248500 34.5159000 3.5261500 11.5169000 - 2021 2 -1.6139300 38.7692000 13.2485000 9.4673800 - 2022 2 -1.6137100 39.1081000 11.6508000 14.1355000 - 2023 1 1.9878000 36.6577000 9.4963900 9.4788500 - 2024 1 2.7570000 37.7422000 11.4033000 13.0864000 - 2025 2 -1.6549600 38.5643000 6.3629200 7.3532600 - 2026 1 2.1975500 36.2368000 11.8587000 7.6018000 - 2027 2 -1.6019300 39.3004000 8.7010500 6.2312000 - 2028 2 -1.5993900 38.6731000 11.3826000 5.8257700 - 2029 1 2.7171000 37.7660000 12.7143000 5.1302900 - 2030 1 2.4018000 37.1988000 6.6388000 8.1422900 - 2031 1 2.4143700 38.8480000 7.1659700 5.9393900 - 2032 2 -1.6681000 38.1586000 9.0705100 1.1556400 - 2033 1 2.6019500 38.9820000 10.2213000 1.9743500 - 2034 3 -0.0404006 44.4148000 14.8634000 8.1378900 - 2035 2 -1.6666300 39.6508000 3.8072900 4.9194600 - 2036 2 -1.6342300 38.3146000 0.8179480 7.4191600 - 2037 1 2.5079000 35.3150000 4.7142800 6.2506700 - 2038 1 2.6001800 37.4105000 1.8063200 6.4232500 - 2039 1 1.6198000 39.3963000 3.8276200 1.8299700 - 2040 1 2.3226900 35.2564000 9.8663100 11.2234000 - 2041 2 -1.5628300 37.1627000 9.8244000 13.5978000 - 2042 1 2.7477200 37.3294000 8.1941400 14.1156000 - 2043 2 -1.6560200 36.9226000 11.0829000 8.9395600 - 2044 2 -1.5544500 36.8264000 12.9319000 12.8355000 - 2045 2 -1.5799000 35.2073000 14.4364000 12.1280000 - 2046 2 -1.5883000 35.6937000 11.1768000 11.8722000 - 2047 2 -1.6036400 38.8384000 7.5913900 13.8059000 - 2048 2 -1.6561800 38.4808000 11.1351000 11.2896000 - 2049 1 2.7609600 34.8296000 12.7943000 11.8625000 - 2050 1 2.7746600 36.8889000 14.4952000 13.5423000 - 2051 1 2.4684600 37.9715000 12.1135000 10.0790000 - 2052 2 -1.5816800 39.9794000 11.2678000 1.2641600 - 2053 2 -1.6256100 37.0291000 9.4476200 5.0202900 - 2054 1 2.5059400 34.4786000 7.6964700 7.0003800 - 2055 2 -1.6083700 36.8488000 13.0236000 6.7579100 - 2056 2 -1.6340200 38.7487000 13.7723000 4.2610300 - 2057 1 2.4507700 34.1790000 11.5109000 6.6669700 - 2058 1 2.6717200 38.6234000 9.8523100 5.1979500 - 2059 2 -1.5476600 35.9720000 9.9274200 2.3528600 - 2060 2 -1.5826500 36.4035000 12.0187000 0.3691590 - 2061 2 -1.6305100 36.5223000 11.8475000 4.2527500 - 2062 2 -1.6174300 39.2599000 9.9108100 3.5537500 - 2063 1 2.7568300 35.8706000 10.2829000 3.9861300 - 2064 1 2.7685100 35.9279000 13.7228000 0.8746280 - 2065 2 -1.5870300 35.9235000 1.1996300 7.3752500 - 2066 2 -1.6411000 35.7098000 2.4956900 13.7176000 - 2067 2 -1.6535100 36.3979000 1.2892500 9.9008400 - 2068 3 0.2366140 40.2438000 0.2488650 7.8286100 - 2069 1 2.5396200 35.8981000 2.2424800 8.6193900 - 2070 2 -1.5947200 33.1325000 14.7490000 4.2569100 - 2071 2 -1.9337900 34.4702000 9.3582100 7.3793000 - 2072 2 -1.9126000 35.2385000 1.8596000 3.0219100 - 2073 2 -1.9581000 35.5915000 6.3673400 7.0734100 - 2074 2 -1.9319100 36.8199000 3.6295200 6.7525300 - 2075 2 -1.5893800 37.4922000 2.2791100 4.9401200 - 2076 1 2.6375700 36.6428000 1.1131400 4.1180800 - 2077 2 -1.9195700 39.1901000 5.5965000 1.7203400 - 2078 2 -1.6462200 36.5520000 7.2151700 2.4331000 - 2079 1 1.4733800 37.9259000 2.7165200 1.8020500 - 2080 2 -1.9241500 37.0875000 4.8777900 3.7598900 - 2081 1 2.6364200 36.4673000 8.5724100 1.5703800 - 2082 1 2.7115400 37.8451000 6.6009800 3.0320300 - 2083 3 -0.0426042 44.5750000 7.1865400 0.7782000 - 2084 3 0.0479593 40.6673000 2.2259000 0.4534770 - 2085 3 -0.0542478 42.5047000 1.0364400 1.9331400 - 2086 3 -0.0399543 44.5456000 2.1932500 0.8880890 - 2087 3 0.2020540 41.0553000 4.5496700 0.7667180 - 2088 3 -0.0204118 42.7120000 3.4432600 2.1086200 - 2089 3 -0.0394067 44.5816000 4.7100200 0.7901290 - 2090 3 0.0129862 42.6807000 5.8233700 2.1600800 - 2091 3 -0.0417319 44.6398000 7.2605000 3.2761800 - 2092 3 0.1132880 40.7508000 2.3471100 3.1474700 - 2093 3 -0.0128892 42.6151000 1.2052500 4.4385800 - 2094 3 -0.0432956 44.5887000 2.3206600 3.4087700 - 2095 3 0.2647260 40.4383000 4.8986300 3.4056400 - 2096 3 0.0296947 42.5982000 3.6571100 4.5831700 - 2097 3 -0.0386198 44.5971000 4.7828100 3.2896200 - 2098 3 0.0328510 42.5990000 6.0879600 4.6510800 - 2099 3 -0.0432873 44.5615000 7.3237300 5.7259200 - 2100 3 0.2705700 40.8695000 2.3598900 5.9092800 - 2101 3 0.0159870 42.5575000 1.1773100 6.9651500 - 2102 3 -0.0367191 44.4928000 2.4088800 5.8701000 - 2103 3 0.2850860 40.9134000 4.9806900 6.1281700 - 2104 3 0.0212076 42.5788000 3.6911100 7.1122800 - 2105 3 -0.0356418 44.5340000 4.8729800 5.8072700 - 2106 3 0.0071547 42.6700000 6.2155000 7.1071300 - 2107 1 2.6188200 30.4643000 14.4158000 11.1740000 - 2108 3 -0.0440142 44.4943000 14.7588000 5.6292600 - 2109 2 -1.5366600 31.2663000 0.3271250 14.1897000 - 2110 2 -1.6038100 30.3467000 1.0692100 8.4223200 - 2111 3 -0.0426540 44.3934000 14.6860000 0.7290600 - 2112 3 0.2112940 41.0357000 7.0249000 1.0276300 - 2113 3 0.3002220 41.0753000 7.4051400 3.3787800 - 2114 3 0.2287490 40.9935000 7.4107800 5.8083500 - 2115 3 0.2810460 40.8205000 9.4229400 0.8379400 - 2116 3 0.0180806 42.8097000 8.4089400 2.0180300 - 2117 3 -0.0376103 44.6108000 9.7064300 0.7261220 - 2118 3 0.0849711 42.4926000 11.1575000 1.7333000 - 2119 3 -0.0316880 44.3887000 12.2841000 0.5818890 - 2120 3 0.0741614 42.4496000 13.4303000 1.9753200 - 2121 3 0.2796480 41.1994000 9.7706300 3.0845200 - 2122 3 0.0236759 42.8203000 8.5663000 4.4817300 - 2123 3 -0.0391291 44.5825000 9.8585700 3.1553100 - 2124 3 0.4163290 40.8199000 12.3586000 3.3993700 - 2125 3 0.0444003 42.6984000 11.1715000 4.3272400 - 2126 3 -0.0336740 44.5026000 12.4209000 3.0556900 - 2127 3 0.0399202 42.6184000 13.6129000 4.3418900 - 2128 3 0.2862130 41.0576000 9.8829900 5.5026600 - 2129 3 0.0037348 42.6774000 8.6737500 6.9243000 - 2130 3 -0.0451645 44.5870000 9.8119200 5.6663100 - 2131 3 0.2980070 41.0509000 12.2724000 5.7582000 - 2132 3 0.0108019 42.6956000 11.0856000 6.8906500 - 2133 3 -0.0435120 44.5626000 12.3205000 5.6394500 - 2134 1 2.3287000 34.0009000 7.4110100 0.1436620 - 2135 3 0.0209105 42.6548000 13.5475000 6.9241600 - 2136 3 -0.0457582 44.5801000 7.4558000 8.2777300 - 2137 3 0.1719250 40.7459000 2.5555700 8.2907600 - 2138 3 0.0170183 42.3829000 1.1314800 9.3924600 - 2139 3 -0.0393856 44.4849000 2.4970200 8.3619000 - 2140 3 0.1581960 40.8692000 4.9902000 8.5039000 - 2141 3 -0.0256888 42.4764000 3.7309300 9.6675200 - 2142 3 -0.0420615 44.5745000 4.9931600 8.3546900 - 2143 3 -0.0263213 42.6558000 6.2528000 9.5846600 - 2144 3 -0.0457591 44.5514000 7.2797600 10.6702000 - 2145 3 0.1132380 40.6863000 2.6098800 10.7405000 - 2146 3 0.0106834 42.3051000 0.8594500 11.8928000 - 2147 3 -0.0427872 44.3862000 2.2095300 10.6955000 - 2148 3 0.0744802 40.7493000 5.0046700 10.8372000 - 2149 3 -0.0226310 42.5048000 3.4484900 12.0952000 - 2150 3 -0.0451519 44.5030000 4.7092200 10.7525000 - 2151 3 -0.0184646 42.5602000 6.0406000 11.9466000 - 2152 3 -0.0411867 44.4841000 7.3834200 13.1469000 - 2153 3 0.1703840 40.5536000 2.1985600 13.0720000 - 2154 3 -0.0049063 42.5188000 1.0457000 14.2617000 - 2155 3 -0.0396177 44.4677000 2.3388200 13.2746000 - 2156 3 0.1557850 40.8190000 4.6795400 13.2586000 - 2157 3 0.0040215 42.5883000 3.5411700 14.6334000 - 2158 3 -0.0404906 44.5060000 4.8750100 13.2182000 - 2159 3 0.0178562 42.5611000 6.0454900 14.2962000 - 2160 2 -1.5731900 30.3875000 1.4099900 12.1149000 - 2161 3 0.1622120 40.7031000 7.4249200 8.2551100 - 2162 3 0.1094860 40.7115000 7.3939100 10.6720000 - 2163 3 0.2374270 40.8909000 7.3479700 13.0709000 - 2164 3 0.1927590 40.7924000 9.8736400 7.9316400 - 2165 3 -0.0193979 42.5492000 8.5818600 9.3284600 - 2166 3 -0.0474309 44.5380000 9.8742200 8.1912300 - 2167 3 0.2756210 40.8358000 12.3431000 8.2814200 - 2168 3 0.0101965 42.6176000 11.1081000 9.3631400 - 2169 3 -0.0430963 44.5729000 12.3511000 8.1285300 - 2170 3 0.0363086 42.5905000 13.5917000 9.3601900 - 2171 3 0.1837090 40.7407000 9.8705500 10.3843000 - 2172 3 0.0021895 42.5560000 8.6214600 11.8022000 - 2173 3 -0.0457294 44.4775000 9.7585300 10.5895000 - 2174 3 0.3039440 40.8019000 12.2963000 10.6786000 - 2175 3 0.0259489 42.5782000 11.1331000 11.8447000 - 2176 3 -0.0411358 44.4767000 12.3502000 10.6745000 - 2177 3 0.0293572 42.4698000 13.6231000 11.8726000 - 2178 3 0.3262150 40.7068000 10.0232000 12.9980000 - 2179 3 0.0470490 42.5667000 8.6402200 14.2106000 - 2180 3 -0.0408487 44.4966000 9.8474900 13.0645000 - 2181 3 0.3453960 40.6225000 12.4902000 13.0244000 - 2182 3 0.0630231 42.5237000 11.0691000 14.2706000 - 2183 3 -0.0409447 44.4101000 12.3347000 13.1287000 - 2184 3 0.0373145 42.3479000 13.5537000 14.3253000 - 2185 2 -1.6202800 36.9779000 2.4546800 0.4205190 - 2186 2 -1.5828400 35.4293000 8.2531600 0.2832190 - 2187 3 0.3148700 40.1241000 13.8783000 0.6078910 - 2188 2 -1.6163400 37.2371000 0.0627433 12.0620000 - 2189 3 0.2169030 40.6266000 0.1028350 5.5054200 - 2190 1 2.5650400 35.7354000 0.1894210 10.9612000 - 2191 3 0.2527560 40.1394000 0.0151446 12.6019000 - 2192 1 2.4642300 31.6936000 8.0482100 0.1086160 - 2193 2 -1.6926100 38.2388000 2.5384200 15.1843000 - 2194 1 2.6439700 36.9520000 1.4514600 22.1675000 - 2195 1 2.7850900 30.3550000 10.2146000 18.4894000 - 2196 2 -1.6041200 39.7759000 10.4922000 24.2663000 - 2197 1 2.5191500 37.0600000 11.9170000 21.2870000 - 2198 2 -1.6141400 38.6161000 13.7533000 24.9260000 - 2199 2 -1.6158900 36.1039000 4.8489800 19.9517000 - 2200 2 -1.5643200 32.0411000 12.8256000 25.2946000 - 2201 2 -1.6391800 31.9731000 13.2252000 28.4987000 - 2202 1 2.6215200 33.4714000 13.9670000 29.0287000 - 2203 1 2.1248500 39.1514000 8.9845400 24.8389000 - 2204 2 -1.9301000 32.7590000 2.8186400 27.4939000 - 2205 2 -1.6242300 33.3978000 10.5454000 29.4592000 - 2206 1 2.4728000 31.6327000 2.7989900 25.7065000 - 2207 2 -1.6207500 33.7396000 11.6145000 26.7231000 - 2208 2 -1.6577800 32.4731000 9.1438900 26.1690000 - 2209 2 -1.6863700 30.4868000 4.5958100 14.9617000 - 2210 1 2.5995500 32.3198000 12.2376000 26.6546000 - 2211 2 -1.6021800 31.3870000 6.9151000 28.1498000 - 2212 2 -1.6203800 30.7221000 7.0874500 25.4445000 - 2213 1 2.5319900 31.4744000 7.8108800 26.8266000 - 2214 3 0.1068710 40.4152000 14.7663000 21.1748000 - 2215 1 2.6870300 32.3618000 1.7061100 28.9383000 - 2216 1 2.5942100 33.9404000 10.1029000 26.2478000 - 2217 2 -1.6866000 31.1537000 11.2552000 27.5493000 - 2218 2 -1.6800500 31.8894000 0.6942700 25.2328000 - 2219 1 2.4686100 30.3175000 10.3939000 28.5289000 - 2220 2 -1.5467500 34.9846000 7.7682500 18.0556000 - 2221 2 -1.6077100 35.1166000 10.5148000 18.1461000 - 2222 1 2.8050800 33.9623000 11.0382000 16.9661000 - 2223 1 2.8177500 35.3449000 9.0555900 19.0289000 - 2224 2 -1.5698900 34.7816000 11.7960000 23.6086000 - 2225 1 2.7724500 34.0488000 12.3431000 22.2326000 - 2226 1 2.6946000 35.0956000 13.2507000 19.4656000 - 2227 2 -1.5704700 33.4762000 4.6565000 17.6787000 - 2228 2 -1.5548000 34.7475000 6.2458400 15.8329000 - 2229 1 2.7624600 32.5364000 1.1156300 18.1040000 - 2230 1 2.6749700 34.7706000 1.8660900 19.9962000 - 2231 1 2.7803200 34.1379000 4.7300200 16.0341000 - 2232 2 -1.5992500 33.6415000 6.1822500 20.2258000 - 2233 2 -1.5531000 35.1466000 5.2305700 23.4160000 - 2234 2 -1.5910700 35.9733000 6.9024000 21.5845000 - 2235 1 2.7537400 33.9019000 4.0109300 23.7444000 - 2236 1 2.8233500 34.9786000 5.5212900 21.1055000 - 2237 2 -1.5798700 33.9017000 14.1108000 15.1661000 - 2238 2 -1.5858200 31.6425000 7.7922700 15.4691000 - 2239 2 -1.5893300 32.8908000 12.2697000 17.5899000 - 2240 2 -1.5708400 34.3780000 9.3633700 20.3061000 - 2241 2 -1.5866700 31.0755000 12.8866000 19.8889000 - 2242 2 -1.5671100 33.3220000 9.7941900 16.1566000 - 2243 1 2.7860100 32.1953000 13.3859000 18.7006000 - 2244 2 -1.5156800 33.7485000 13.8597000 19.4641000 - 2245 2 -1.5624600 31.8488000 9.9787300 19.4003000 - 2246 2 -1.5764000 32.2369000 8.6213800 22.0645000 - 2247 1 2.6857100 37.5645000 14.4568000 26.0243000 - 2248 2 -1.5377500 33.0168000 11.3530000 21.5170000 - 2249 1 2.7288100 32.5616000 13.9980000 24.2278000 - 2250 2 -1.5649700 33.4332000 13.7573000 22.8697000 - 2251 1 2.7995900 32.8513000 9.7645700 20.7266000 - 2252 2 -1.5963100 31.4069000 4.8568400 19.3554000 - 2253 2 -1.5410100 32.0661000 7.0984800 18.0578000 - 2254 2 -1.5612700 33.4027000 1.4201800 19.6573000 - 2255 2 -1.4796000 33.3941000 1.4053600 16.7553000 - 2256 1 2.7661100 30.9771000 7.5140800 16.9079000 - 2257 2 -1.6258700 30.3512000 14.6601000 26.4163000 - 2258 2 -1.5863400 31.0478000 2.0313500 19.1412000 - 2259 2 -1.5491600 34.5813000 2.5976600 24.4489000 - 2260 2 -1.5809600 31.0894000 3.1144000 21.5589000 - 2261 2 -1.5862900 31.3334000 6.1347100 23.1122000 - 2262 1 2.5352000 31.0601000 1.9889000 22.5975000 - 2263 2 -1.6214000 33.8266000 4.0251500 21.9931000 - 2264 1 2.6899200 31.3505000 7.2826700 21.9086000 - 2265 1 2.6488500 31.2856000 5.6750100 24.7374000 - 2266 1 2.6405100 31.1573000 3.4581400 19.8899000 - 2267 1 2.7875100 32.7209000 5.8927700 18.8847000 - 2268 2 -1.5056400 31.3951000 0.3878980 22.6417000 - 2269 3 0.2717330 40.5384000 14.4927000 15.5851000 - 2270 2 -1.6031100 35.1908000 9.6490800 25.4663000 - 2271 2 -1.5866000 33.6960000 14.8426000 25.3511000 - 2272 2 -1.6540900 32.3299000 4.0119100 24.5010000 - 2273 2 -1.5755200 38.6417000 13.9011000 19.5642000 - 2274 2 -1.5740200 30.6554000 4.4216300 25.8125000 - 2275 2 -1.7184400 38.7631000 10.1313000 28.0418000 - 2276 1 2.6828000 37.6653000 11.5387000 28.9550000 - 2277 2 -1.6505100 38.9652000 1.9679600 28.3309000 - 2278 2 -1.5379200 34.4050000 12.7260000 29.3747000 - 2279 2 -1.6311900 39.5293000 8.0304800 26.3445000 - 2280 1 2.4731100 37.5296000 3.0712100 28.5908000 - 2281 2 -1.6297800 37.2385000 7.6258200 27.9463000 - 2282 2 -1.6648100 33.9377000 0.1393190 23.4353000 - 2283 2 -1.6139600 36.0387000 13.8326000 26.3453000 - 2284 2 -1.6676200 35.2503000 0.1831250 25.8888000 - 2285 1 1.9842600 38.8521000 8.4145800 27.8615000 - 2286 1 2.6396900 34.6924000 3.9338900 27.2071000 - 2287 2 -1.9306400 36.0861000 2.7623100 27.4896000 - 2288 2 -1.5774400 38.0278000 4.7392000 28.3956000 - 2289 1 2.5586100 35.9197000 8.7049000 28.2886000 - 2290 1 2.0343100 37.0916000 5.9058000 27.7549000 - 2291 1 2.6221900 36.4147000 2.4732300 15.2265000 - 2292 2 -1.9124900 34.3034000 7.7821700 28.1858000 - 2293 2 -1.6225300 36.0976000 10.4564000 28.4753000 - 2294 1 2.4732400 32.2384000 11.6993000 29.5854000 - 2295 3 -0.0494516 44.4024000 14.7222000 20.6555000 - 2296 1 2.6115400 38.3365000 12.2910000 24.0422000 - 2297 1 2.5879400 37.7827000 10.6624000 16.6951000 - 2298 2 -1.4856500 38.4945000 9.4406600 16.7578000 - 2299 2 -1.5193600 36.0783000 0.6547970 23.2118000 - 2300 2 -1.6094200 35.5179000 12.5688000 20.9711000 - 2301 2 -1.6519900 39.5657000 4.6698300 23.0345000 - 2302 2 -1.6411100 38.0834000 12.6525000 22.4074000 - 2303 2 -1.6499400 37.8132000 0.7853140 18.0268000 - 2304 1 2.3455500 37.4208000 2.4215800 18.2090000 - 2305 2 -1.6272700 39.8168000 1.7731200 25.4816000 - 2306 1 2.4228100 38.4910000 3.5398900 24.0780000 - 2307 1 2.4698200 38.9139000 1.1371300 26.7769000 - 2308 1 2.3434000 38.6478000 5.4595500 21.7517000 - 2309 2 -1.5477400 36.4730000 8.0835700 15.5636000 - 2310 2 -1.5822800 35.0405000 11.8202000 15.8834000 - 2311 2 -1.6081200 38.2714000 12.1599000 17.2251000 - 2312 2 -1.5951000 40.1683000 9.3729600 19.6634000 - 2313 2 -1.6092600 36.1782000 13.5368000 18.3696000 - 2314 1 2.6232800 37.8382000 13.5525000 17.9552000 - 2315 1 2.7364500 35.2239000 13.3096000 15.4555000 - 2316 2 -1.5766000 36.6181000 13.8071000 15.1022000 - 2317 2 -1.6056300 37.2188000 11.3300000 24.7011000 - 2318 3 -0.0471097 44.3068000 14.7136000 15.5932000 - 2319 2 -1.5853600 37.0585000 8.7755500 19.2570000 - 2320 2 -1.7113500 38.1549000 8.4485400 21.9491000 - 2321 2 -1.6073100 36.0102000 9.5766600 22.9170000 - 2322 2 -1.5579700 37.7543000 10.9674000 20.3324000 - 2323 1 2.7245000 35.7528000 10.5452000 24.1750000 - 2324 1 2.7176000 38.3196000 9.2705400 20.1727000 - 2325 2 -1.6393900 38.2987000 3.5833500 18.6489000 - 2326 2 -1.5791400 37.2442000 6.1582700 17.6779000 - 2327 2 -1.6258600 35.6243000 1.4380400 18.7124000 - 2328 2 -1.6863100 38.0539000 0.1448910 21.1117000 - 2329 2 -1.5773600 38.7213000 5.7426500 20.2274000 - 2330 2 -1.6101700 36.0075000 0.9636840 15.9285000 - 2331 1 2.8063700 35.7825000 7.1164100 16.7946000 - 2332 1 2.6807200 37.3322000 5.3755800 18.9934000 - 2333 2 -1.6648800 35.5636000 3.6539800 16.0934000 - 2334 2 -1.4930800 33.1306000 0.0573043 27.6374000 - 2335 2 -1.6682900 34.8222000 5.0674000 25.8980000 - 2336 2 -1.6383900 35.9597000 2.0885500 21.0540000 - 2337 2 -1.6692800 37.4928000 4.6177400 24.9603000 - 2338 2 -1.6581800 36.8111000 7.1847700 24.2494000 - 2339 2 -1.6304400 37.9609000 2.5767200 22.8298000 - 2340 1 2.7108600 36.4770000 8.0089100 22.6082000 - 2341 1 2.7352900 34.8879000 1.1399900 24.5286000 - 2342 1 2.5303900 36.1306000 5.7567800 24.7331000 - 2343 3 0.1802460 40.6940000 14.6930000 17.9450000 - 2344 3 -0.0431549 44.4833000 7.3315000 15.5610000 - 2345 3 0.1964950 40.7476000 2.3627000 15.5404000 - 2346 3 -0.0217526 42.5054000 1.1113300 16.7913000 - 2347 3 -0.0402996 44.4758000 2.2691600 15.7048000 - 2348 3 0.2342140 40.6566000 4.7796900 15.7283000 - 2349 3 -0.0011981 42.4586000 3.5658300 16.9962000 - 2350 3 -0.0417371 44.4935000 4.8577500 15.6202000 - 2351 3 0.0189208 42.5605000 5.9785800 16.7775000 - 2352 3 -0.0420658 44.4477000 7.3144500 18.0687000 - 2353 3 0.1615760 40.5110000 2.4601600 18.0591000 - 2354 3 0.0005650 42.3148000 1.1636400 19.1655000 - 2355 3 -0.0458556 44.3975000 2.3768900 18.2522000 - 2356 3 0.2373640 40.4844000 4.8397700 18.1868000 - 2357 3 0.0055532 42.3697000 3.5770400 19.5340000 - 2358 3 -0.0461099 44.4715000 4.8302200 18.0517000 - 2359 3 0.0403172 42.5137000 6.0280600 19.1481000 - 2360 3 -0.0405299 44.4159000 7.3396100 20.5729000 - 2361 3 0.2294740 40.4543000 2.5510500 20.4788000 - 2362 3 0.0330638 42.1489000 1.0867600 21.5801000 - 2363 3 -0.0438272 44.2179000 2.3321300 20.7633000 - 2364 3 0.2848280 40.2903000 4.7771600 21.0937000 - 2365 3 0.0394942 42.2335000 3.5048600 22.0945000 - 2366 3 -0.0439449 44.3394000 4.7095200 20.5181000 - 2367 3 0.0312149 42.5316000 6.0514100 21.6203000 - 2368 3 -0.0459691 44.3498000 14.4640000 25.6078000 - 2369 3 0.2720940 40.2638000 0.2712400 19.9166000 - 2370 3 0.2719040 40.8691000 14.5845000 25.8043000 - 2371 3 0.3198820 40.7674000 7.1442600 15.3935000 - 2372 3 0.3454210 40.5180000 7.3129300 17.7432000 - 2373 3 0.3812460 39.3723000 7.5320800 19.6188000 - 2374 3 0.3544720 40.6595000 9.5749300 15.4183000 - 2375 3 0.0564233 42.6136000 8.4397500 16.7281000 - 2376 3 -0.0427827 44.4545000 9.7795200 15.5744000 - 2377 3 0.3293470 40.7527000 12.0599000 15.5194000 - 2378 3 0.0527431 42.5397000 10.9759000 16.7661000 - 2379 3 -0.0452229 44.4147000 12.2550000 15.5789000 - 2380 3 0.0118097 42.4682000 13.4905000 16.7830000 - 2381 3 0.4205130 40.8893000 9.7873700 17.8344000 - 2382 3 0.0942226 42.5094000 8.3927800 19.3676000 - 2383 3 -0.0380279 44.3590000 9.7769300 18.1353000 - 2384 3 0.3126380 40.7169000 12.2373000 17.9188000 - 2385 3 0.0773201 42.3169000 11.0163000 19.3768000 - 2386 3 -0.0460441 44.3284000 12.2879000 18.0686000 - 2387 3 0.3577340 40.5009000 14.6501000 28.1617000 - 2388 3 -0.0031271 42.1691000 13.3865000 19.4515000 - 2389 3 0.4017470 40.1851000 10.0807000 21.4526000 - 2390 3 0.0459468 42.5130000 8.5155600 21.9706000 - 2391 3 -0.0352222 44.3208000 9.8132200 20.5988000 - 2392 3 0.3318650 40.1840000 12.2301000 21.0036000 - 2393 3 0.0510214 42.3167000 11.2344000 21.8823000 - 2394 3 -0.0429962 44.2412000 12.2915000 20.5568000 - 2395 3 0.3068490 40.7313000 12.0956000 28.7423000 - 2396 3 0.0063834 42.4488000 13.7203000 21.9282000 - 2397 3 -0.0472316 44.4219000 7.2587000 23.0059000 - 2398 3 0.3051780 40.4885000 2.8647800 23.7054000 - 2399 3 0.1067230 41.8754000 1.0280300 24.1595000 - 2400 3 -0.0386060 44.1451000 2.2219800 23.1573000 - 2401 3 0.2671530 40.1534000 4.8089100 25.2523000 - 2402 3 0.0281050 42.6144000 3.4727600 24.5567000 - 2403 3 -0.0427878 44.3914000 4.6978400 22.9936000 - 2404 3 0.0017462 42.5674000 6.0424500 24.0749000 - 2405 3 -0.0472361 44.5770000 7.3168900 25.6291000 - 2406 3 0.3553240 38.3423000 3.2734600 26.0911000 - 2407 3 0.0346055 42.4846000 0.9698010 26.6347000 - 2408 3 -0.0347261 44.4445000 2.2295500 25.7318000 - 2409 3 0.2279620 38.4409000 6.1637400 26.0836000 - 2410 3 0.0184090 42.6352000 3.5361800 26.8924000 - 2411 3 -0.0430042 44.4081000 4.8648500 25.5961000 - 2412 3 -0.0144153 42.3505000 5.9450000 26.6310000 - 2413 3 -0.0459331 44.4840000 7.0074400 28.0412000 - 2414 3 0.1920430 40.8927000 2.2648800 27.8513000 - 2415 3 -0.0212193 42.4647000 0.8726320 29.2200000 - 2416 3 -0.0349456 44.5341000 2.0435700 28.1389000 - 2417 3 0.1655870 40.5525000 4.6318100 28.0231000 - 2418 3 -0.0159694 42.7136000 3.2629500 29.3290000 - 2419 3 -0.0406802 44.5398000 4.5366400 28.0144000 - 2420 3 -0.0108769 42.6829000 5.8362400 29.1716000 - 2421 3 -0.0451977 44.4574000 14.6102000 28.0877000 - 2422 2 -1.9208000 34.7510000 4.5652100 29.2046000 - 2423 3 0.2423540 40.3643000 6.6535700 22.5827000 - 2424 3 0.1987080 40.5307000 7.1164500 24.7886000 - 2425 3 0.1239000 40.4805000 7.0067300 28.4764000 - 2426 3 0.2849660 40.6117000 8.8211900 23.2315000 - 2427 3 -0.0006376 42.6657000 8.4244800 24.5536000 - 2428 3 -0.0428281 44.3814000 9.8745600 23.0556000 - 2429 3 0.2844280 40.5122000 12.7815000 23.1534000 - 2430 3 0.0397099 42.3086000 10.8099000 24.2499000 - 2431 3 -0.0404932 44.2924000 12.4834000 23.0176000 - 2432 2 -1.6436400 30.5907000 8.9084900 28.7584000 - 2433 3 0.0204429 42.4203000 13.1965000 24.4765000 - 2434 3 0.2809220 40.6206000 9.6236100 25.8841000 - 2435 3 -0.0061411 42.5704000 8.2907100 27.0409000 - 2436 3 -0.0458333 44.5320000 9.7525200 25.5782000 - 2437 3 0.3716610 39.9774000 12.2099000 25.3191000 - 2438 3 0.0119327 42.5728000 10.8351000 26.7178000 - 2439 3 -0.0440243 44.3624000 12.0975000 25.4235000 - 2440 3 0.0312757 42.4715000 13.2503000 27.0522000 - 2441 3 0.2175300 40.8372000 9.4480700 28.2356000 - 2442 3 -0.0101010 42.6570000 8.2629600 29.4092000 - 2443 3 -0.0455748 44.5728000 9.5144100 28.0229000 - 2444 3 0.4761050 38.9004000 11.9250000 27.4013000 - 2445 3 0.0287906 42.7073000 10.7729000 29.1407000 - 2446 3 -0.0423287 44.5060000 12.0963000 27.8402000 - 2447 2 -1.5366600 38.2728000 13.1071000 28.7802000 - 2448 3 0.0209748 42.3717000 13.6013000 29.3542000 - 2449 1 2.6476700 34.6239000 14.5502000 26.5939000 - 2450 3 -0.0506293 44.3263000 14.8197000 18.0495000 - 2451 1 2.6726300 34.9658000 11.2243000 29.4853000 - 2452 1 2.5609000 34.4515000 0.4695540 15.7183000 - 2453 3 0.3922040 40.2559000 0.1455890 22.6346000 - 2454 2 -1.4921400 31.7717000 0.2847730 28.8482000 - 2455 2 -1.6103300 30.9617000 14.4661000 23.6531000 - 2456 2 -1.4338900 36.8861000 0.0636379 29.0516000 - 2457 1 2.7131500 30.2229000 14.6443000 28.0121000 - 2458 2 -1.5892900 30.3203000 7.3973900 20.7207000 - 2459 1 2.7618800 32.2380000 9.2978100 14.9185000 - 2460 2 -1.5775700 31.5331000 14.8577000 18.1969000 - 2461 3 -0.0429559 44.4385000 14.9672000 3.1283500 - 2462 3 -0.0492481 44.4784000 17.1358000 0.7063310 - 2463 2 -1.5190300 35.2535000 26.4155000 12.7939000 - 2464 2 -1.6025600 35.0728000 26.4742000 7.0551000 - 2465 2 -1.4270900 32.8796000 28.0895000 12.8729000 - 2466 2 -1.5891900 32.1397000 26.4093000 10.7420000 - 2467 1 2.7213100 37.3501000 18.9329000 1.0605800 - 2468 1 2.6949400 34.4001000 27.9410000 12.9153000 - 2469 1 2.6493200 33.8647000 27.5719000 7.1899000 - 2470 2 -1.6618300 33.3235000 28.6911000 0.8228290 - 2471 1 2.5642000 34.1419000 27.3668000 1.3310800 - 2472 3 0.0005629 42.5183000 15.8896000 6.7783300 - 2473 2 -1.6218600 30.8672000 27.6410000 8.3016000 - 2474 2 -1.5498400 31.8899000 29.2381000 10.0640000 - 2475 1 2.7654400 32.1968000 27.6798000 9.7131800 - 2476 2 -1.6185800 33.6130000 27.5723000 8.8645700 - 2477 1 2.5968900 33.8738000 28.3067000 4.1563300 - 2478 2 -1.6285300 34.0037000 27.2654000 2.9462200 - 2479 2 -1.6710800 32.9820000 27.5807000 5.6265200 - 2480 1 2.6525500 31.3871000 26.5839000 5.0490100 - 2481 2 -1.5469900 31.1475000 17.5822000 0.7213000 - 2482 2 -1.6439700 30.7113000 24.9131000 0.3539740 - 2483 2 -1.5503100 30.5158000 22.9531000 3.8324600 - 2484 2 -1.5810100 33.7689000 19.8905000 12.5794000 - 2485 1 2.7077600 33.0793000 21.0056000 11.5605000 - 2486 1 2.8030800 34.6177000 18.4821000 13.3784000 - 2487 2 -1.6123400 33.0253000 17.3360000 2.9086400 - 2488 2 -1.6141300 34.3695000 17.2685000 5.6733000 - 2489 1 2.6119500 34.4741000 17.4946000 7.3580400 - 2490 1 2.7243400 31.1578000 20.0047000 6.3054200 - 2491 3 0.0023273 42.4457000 15.9369000 1.8484300 - 2492 2 -1.4587300 33.6325000 20.2557000 1.3467700 - 2493 2 -1.5209600 34.1959000 17.3105000 12.2305000 - 2494 2 -1.5921200 33.8092000 15.5231000 10.1905000 - 2495 1 2.7926500 34.0216000 15.1988000 3.0181300 - 2496 1 2.6667500 30.5634000 21.4486000 12.4865000 - 2497 2 -1.5706500 31.7471000 25.5092000 3.9385600 - 2498 3 0.2527430 40.6138000 14.9871000 3.0147900 - 2499 3 -0.0235595 44.3755000 29.5762000 10.7473000 - 2500 2 -1.5969500 30.6180000 19.5707000 7.8851800 - 2501 2 -1.5914100 31.2084000 17.6276000 9.5786400 - 2502 2 -1.5486600 30.6272000 17.3246000 12.2937000 - 2503 2 -1.5950600 32.1021000 21.9024000 12.5716000 - 2504 2 -1.4749300 32.2328000 24.6362000 12.9358000 - 2505 3 0.2468010 40.3697000 26.5440000 14.4965000 - 2506 2 -1.5904000 31.4012000 23.8698000 10.3466000 - 2507 2 -1.6436900 31.6817000 20.4110000 9.8118300 - 2508 1 2.8058400 31.3806000 25.1656000 11.7191000 - 2509 2 -1.6097400 31.2212000 29.7488000 5.0378500 - 2510 1 2.7656000 32.6198000 23.3218000 13.7596000 - 2511 1 2.7332800 31.8854000 24.0560000 8.8179100 - 2512 1 2.6810500 31.9473000 18.9590000 9.0104000 - 2513 2 -1.5510700 33.0182000 23.0764000 2.8829500 - 2514 2 -1.5128300 34.3400000 21.9744000 5.9566800 - 2515 1 2.7800100 31.6361000 16.8979000 2.0344500 - 2516 2 -1.5504500 34.0678000 21.6982000 10.5119000 - 2517 3 -0.0473862 44.4238000 19.8316000 3.1821400 - 2518 2 -1.6315900 33.7085000 18.6048000 8.2859600 - 2519 2 -1.5798600 33.1210000 23.4944000 8.0875700 - 2520 2 -1.5300500 32.3690000 23.7428000 5.5476200 - 2521 2 -1.5853300 32.1426000 18.7281000 5.5058800 - 2522 1 2.8466800 31.9038000 23.7753000 3.9881300 - 2523 1 2.7939100 32.9982000 22.6522000 6.5841100 - 2524 2 -1.5731900 31.8708000 21.4030000 6.6317400 - 2525 1 2.7172400 33.1834000 18.1538000 4.5259300 - 2526 2 -1.4982400 34.3117000 23.1812000 0.5439700 - 2527 2 -1.5736400 34.2158000 19.1317000 3.7113100 - 2528 2 -1.5706700 31.4215000 22.0999000 1.2236200 - 2529 1 2.8044300 33.0945000 22.3097000 1.3306600 - 2530 2 -1.5734000 32.8039000 15.2738000 12.9171000 - 2531 1 2.8202000 34.0273000 15.7568000 11.9417000 - 2532 2 -1.5977500 32.3109000 15.3362000 1.8178700 - 2533 2 -1.5603500 30.8039000 24.7453000 7.9016500 - 2534 3 -0.0348612 44.4554000 29.5673000 13.1122000 - 2535 2 -1.6731500 40.0628000 28.2735000 8.9849100 - 2536 2 -1.5915500 38.6358000 27.7764000 13.1206000 - 2537 1 2.4912100 37.6875000 28.3389000 12.0274000 - 2538 1 2.3676800 38.1191000 26.8677000 14.3903000 - 2539 2 -1.6210100 37.6979000 28.3454000 7.8644500 - 2540 1 2.5350000 36.5234000 29.2129000 6.9433500 - 2541 1 2.3285000 38.3698000 26.7882000 7.7193900 - 2542 2 -1.6790100 39.2818000 25.7007000 6.4071500 - 2543 1 1.7847500 38.5854000 29.4433000 2.0440400 - 2544 2 -1.5774400 34.9792000 28.7981000 14.0752000 - 2545 2 -1.6504700 36.6294000 28.1088000 10.6381000 - 2546 1 2.5013000 36.3619000 26.4676000 9.8922600 - 2547 2 -1.6582000 38.7504000 17.8094000 1.3581800 - 2548 2 -1.5916900 34.8466000 29.0445000 6.9499500 - 2549 1 2.4849600 36.9270000 27.8264000 3.9285300 - 2550 2 -1.5968800 38.4096000 28.0878000 2.9612700 - 2551 2 -1.6504500 36.8126000 26.7722000 5.0959800 - 2552 2 -1.6615400 35.7222000 29.0145000 3.6788700 - 2553 2 -1.5681100 34.1119000 26.0542000 0.5111180 - 2554 2 -1.6314500 39.2366000 16.2552000 10.1321000 - 2555 2 -1.6177000 39.3018000 22.1303000 10.9211000 - 2556 2 -1.6435200 40.4300000 24.6850000 11.4451000 - 2557 1 2.6918100 38.1095000 21.2869000 11.8587000 - 2558 1 2.4879400 39.3182000 23.7167000 10.4755000 - 2559 2 -1.6390100 39.4968000 22.7507000 5.6711200 - 2560 2 -1.6315400 38.5743000 19.7596000 4.6105200 - 2561 1 2.5098000 38.2798000 21.1053000 5.8568800 - 2562 1 2.3217600 38.5301000 16.2385000 8.7125400 - 2563 1 2.4279500 39.0873000 24.3189000 5.3058800 - 2564 1 2.6547200 32.7514000 29.5411000 13.5961000 - 2565 2 -1.6145500 38.4008000 15.0663000 12.5634000 - 2566 1 2.7728000 35.6043000 22.7501000 5.5522400 - 2567 1 2.5187700 38.2343000 16.2281000 11.5069000 - 2568 2 -1.6732600 36.9126000 25.5427000 11.0802000 - 2569 2 -1.5974700 37.3445000 25.8623000 8.6736800 - 2570 3 -0.0386621 44.3825000 29.4843000 0.6434740 - 2571 2 -1.6591200 38.8367000 23.7443000 8.8423300 - 2572 2 -1.7312200 38.1305000 19.2065000 8.2052000 - 2573 2 -1.5487100 37.0958000 17.2581000 11.2005000 - 2574 2 -1.6114500 36.6477000 21.9387000 11.6505000 - 2575 2 -1.5850900 36.2828000 18.8584000 13.2313000 - 2576 2 -1.6208900 38.5755000 21.8493000 14.0685000 - 2577 2 -1.6305100 37.0838000 24.2044000 13.3057000 - 2578 2 -1.6095400 39.3791000 18.4653000 14.5099000 - 2579 2 -1.4891000 34.8585000 23.8787000 11.5678000 - 2580 2 -1.5933600 38.1721000 19.7063000 11.6008000 - 2581 1 2.7810700 35.7962000 24.8187000 12.3466000 - 2582 1 2.7526300 37.4440000 22.9897000 14.5852000 - 2583 1 2.7823400 35.4357000 22.6363000 10.6552000 - 2584 1 2.7345900 36.7839000 18.8490000 11.6514000 - 2585 2 -1.5704200 38.5190000 24.5628000 3.8750700 - 2586 2 -1.5578800 36.6871000 19.3102000 2.4264900 - 2587 2 -1.6132400 37.0604000 17.6439000 4.7972700 - 2588 2 -1.5702600 36.9527000 21.5155000 6.4064300 - 2589 2 -1.5950200 36.4619000 24.1402000 6.8179100 - 2590 2 -1.5634600 35.9745000 19.3484000 10.1837000 - 2591 2 -1.6092600 36.1358000 23.2195000 9.2735400 - 2592 2 -1.5045700 35.0483000 24.5182000 5.2075700 - 2593 1 2.7460000 35.7731000 25.5650000 5.9761900 - 2594 1 2.6618900 36.2186000 18.9409000 8.7200400 - 2595 1 2.6674300 37.1591000 24.0141000 8.3151000 - 2596 1 2.6050800 37.3858000 18.8550000 3.8676800 - 2597 2 -1.5849400 37.8786000 22.9397000 1.6770000 - 2598 2 -1.5822200 36.1098000 24.8008000 2.2033700 - 2599 2 -1.5638100 35.9013000 17.6097000 7.9333900 - 2600 2 -1.5780700 36.4096000 22.6145000 3.9850200 - 2601 1 2.6460100 35.0632000 24.7684000 0.6317380 - 2602 1 2.7457900 34.4946000 18.9764000 1.9594600 - 2603 1 2.7364800 37.1174000 23.8069000 2.9691800 - 2604 2 -1.6028700 36.3582000 16.0176000 13.9005000 - 2605 2 -1.6221800 38.2971000 15.4802000 7.4347800 - 2606 2 -1.6009300 36.0002000 15.2424000 5.8910900 - 2607 3 0.1933820 40.5831000 17.0443000 5.5439000 - 2608 2 -1.5135200 35.3196000 16.1164000 3.4411500 - 2609 1 2.7231900 35.7440000 16.4558000 4.8657100 - 2610 2 -1.6360200 34.4070000 29.3836000 11.6454000 - 2611 1 2.7102500 37.2640000 22.3076000 0.2836400 - 2612 2 -1.6478200 36.0517000 15.2888000 0.3057750 - 2613 1 2.2360700 38.6623000 29.3993000 8.6879200 - 2614 3 -0.0028571 42.2246000 18.5932000 4.2612800 - 2615 2 -1.5806300 34.0254000 23.2674000 14.5154000 - 2616 3 0.0609931 40.5454000 29.5153000 0.8420810 - 2617 3 0.2996010 40.6221000 17.0864000 0.5326760 - 2618 2 -1.4630400 30.5104000 27.1008000 14.6869000 - 2619 3 0.2542120 40.5147000 19.8782000 2.9574400 - 2620 2 -1.5741600 31.4600000 18.6993000 14.6078000 - 2621 3 -0.0518354 44.3639000 17.4381000 3.1042300 - 2622 3 0.0021564 42.5760000 28.1199000 13.9487000 - 2623 3 -0.0380849 44.5199000 27.0862000 12.8336000 - 2624 3 0.0131642 42.5419000 25.5914000 14.1372000 - 2625 3 0.2438840 39.1843000 25.9632000 12.5911000 - 2626 3 -0.0443065 44.4931000 24.6928000 12.9867000 - 2627 3 0.0157797 42.3388000 23.2536000 14.0147000 - 2628 3 0.4194290 39.9584000 24.1343000 13.8790000 - 2629 3 0.0872926 40.9919000 27.6473000 12.1722000 - 2630 3 -0.0241479 44.5813000 27.2088000 10.3273000 - 2631 3 0.0463882 42.6496000 25.9507000 11.5198000 - 2632 3 0.4137760 39.1676000 27.3559000 10.6707000 - 2633 3 -0.0407892 44.5586000 24.7869000 10.5332000 - 2634 3 0.0323307 42.6693000 23.4944000 11.6354000 - 2635 3 0.3400250 40.5378000 25.6297000 9.6729000 - 2636 3 0.1311600 42.5780000 28.4713000 9.4682500 - 2637 2 -1.5914900 35.6336000 18.1913000 0.8203390 - 2638 3 -0.0218398 44.5045000 27.3898000 7.8948200 - 2639 3 0.0864110 42.6924000 26.0896000 8.9821200 - 2640 3 0.3991030 40.4376000 26.7696000 7.7370100 - 2641 3 -0.0391167 44.7210000 24.9438000 8.0323700 - 2642 3 0.0349021 42.8733000 23.7207000 9.1615700 - 2643 3 0.3220270 41.1564000 24.5400000 7.6665500 - 2644 3 0.0762304 40.5742000 29.4278000 3.1708300 - 2645 3 0.3756260 40.0039000 21.9620000 12.7410000 - 2646 3 0.3168550 41.1438000 22.4675000 10.1543000 - 2647 3 0.3600220 40.0315000 21.5642000 8.2586300 - 2648 3 -0.0045901 42.4342000 16.1146000 4.2455400 - 2649 3 0.1789290 40.4818000 17.3255000 3.1627600 - 2650 3 -0.0400464 44.3701000 14.9095000 10.6337000 - 2651 3 -0.0461056 44.3737000 22.2670000 13.0272000 - 2652 3 0.0447803 42.2967000 20.8160000 14.1561000 - 2653 3 -0.0433018 44.3044000 19.8190000 13.0236000 - 2654 3 0.1996390 41.5232000 18.5714000 14.1316000 - 2655 3 0.4877690 38.6943000 20.0074000 13.4700000 - 2656 3 -0.0430665 44.3113000 17.3408000 13.0370000 - 2657 3 0.0239974 42.2867000 16.2527000 14.2228000 - 2658 3 0.4290300 39.9490000 17.2111000 12.3870000 - 2659 3 -0.0482942 44.4136000 22.2221000 10.4237000 - 2660 3 0.0574230 42.4517000 21.0475000 11.6386000 - 2661 3 -0.0502039 44.3850000 19.7713000 10.5525000 - 2662 3 0.0498660 42.3188000 18.5064000 11.8729000 - 2663 3 0.2821180 40.7010000 20.1487000 10.1351000 - 2664 3 -0.0464781 44.2794000 17.3672000 10.5684000 - 2665 3 0.0292513 42.3261000 16.1251000 11.7093000 - 2666 3 0.2702090 40.5977000 17.8457000 10.2515000 - 2667 3 -0.0485804 44.5300000 22.2538000 7.9666400 - 2668 3 0.0045988 42.5579000 20.9879000 9.0544600 - 2669 3 -0.0506401 44.5326000 19.8040000 8.0309000 - 2670 3 -0.0067686 42.5677000 18.5446000 9.2308900 - 2671 3 0.2135500 40.6551000 19.3345000 7.8548800 - 2672 3 -0.0481219 44.4605000 17.3657000 8.0241900 - 2673 3 0.0220533 42.4464000 16.1384000 9.2226600 - 2674 3 0.2221020 40.3939000 17.0468000 7.9994900 - 2675 3 -0.0272134 44.5895000 22.1462000 0.5977070 - 2676 3 0.0400311 42.5665000 20.7333000 1.8607500 - 2677 3 0.0742281 42.1012000 28.6414000 7.0663900 - 2678 3 -0.0371514 44.5679000 27.3308000 5.4408500 - 2679 3 0.0818805 42.6521000 26.2386000 6.5634600 - 2680 3 0.3060750 40.7373000 27.4385000 5.2622000 - 2681 3 -0.0416296 44.6550000 24.9198000 5.2733600 - 2682 3 0.0066060 43.1653000 23.8103000 6.6842700 - 2683 3 0.2123190 41.1343000 24.8059000 5.2525700 - 2684 3 -0.0120275 42.5608000 28.5360000 4.2686200 - 2685 3 -0.0471460 44.4516000 27.2282000 2.9869300 - 2686 3 0.0107727 42.6547000 26.0182000 4.0199900 - 2687 3 0.1426600 40.7169000 26.9977000 2.8911400 - 2688 3 -0.0459691 44.5665000 24.7631000 2.8546300 - 2689 3 0.0032265 42.7153000 23.4475000 4.1952100 - 2690 3 0.3724250 40.8130000 24.5941000 2.7749900 - 2691 3 -0.0522802 42.4066000 28.2072000 1.6296100 - 2692 3 0.0224190 42.4116000 21.1568000 4.2815800 - 2693 3 -0.0447167 44.3654000 27.0905000 0.5909140 - 2694 3 -0.0116905 42.5055000 25.7159000 1.6114800 - 2695 3 0.0974685 40.5957000 26.8679000 0.4572890 - 2696 3 -0.0382167 44.5427000 24.6543000 0.4253340 - 2697 3 0.0396548 42.3344000 23.1179000 1.6794000 - 2698 3 0.2696300 40.6379000 24.4056000 0.4637400 - 2699 3 0.2600010 41.3263000 22.4341000 6.4077800 - 2700 3 0.3123750 40.3548000 22.3238000 3.2960800 - 2701 3 0.3987560 39.6345000 22.2183000 0.8904380 - 2702 3 -0.0422801 44.5247000 22.2567000 3.0020300 - 2703 3 -0.0379870 44.5232000 19.6277000 0.7228990 - 2704 3 0.0050831 42.2864000 18.2914000 1.8484900 - 2705 3 0.3489470 40.7315000 19.6943000 0.5606640 - 2706 3 -0.0448119 44.6001000 22.1923000 5.4109200 - 2707 3 0.0057273 42.6316000 20.6165000 6.7433200 - 2708 3 -0.0497615 44.4171000 19.6329000 5.5021600 - 2709 3 -0.0101879 42.5087000 18.2221000 6.7172200 - 2710 3 0.2710130 40.0962000 20.9016000 5.0713200 - 2711 3 -0.0510117 44.4150000 17.1868000 5.4994800 - 2712 3 0.3023680 40.6035000 14.9439000 13.0015000 - 2713 1 2.8049800 30.3615000 18.4302000 13.4391000 - 2714 1 2.6742900 30.9697000 28.6186000 0.6330030 - 2715 3 0.2780300 38.8587000 29.6721000 10.7447000 - 2716 3 -0.0452392 44.4712000 29.6389000 3.1604300 - 2717 2 -1.5692500 33.2930000 29.6765000 3.4259600 - 2718 3 -0.0225151 44.4027000 29.7492000 8.3187600 - 2719 2 -1.6453200 30.2558000 19.9098000 12.5114000 - 2720 2 -1.1439000 35.3089000 29.6795000 1.2238800 - 2721 3 -0.0380493 44.4825000 29.7552000 5.6751400 - 2722 1 2.7120200 37.1293000 16.2671000 16.8332000 - 2723 2 -1.6529700 38.2534000 15.1245000 17.3545000 - 2724 1 2.6421700 37.1435000 17.8390000 21.6433000 - 2725 1 2.8270100 35.3306000 23.0528000 21.4461000 - 2726 1 2.6701000 37.7202000 22.8768000 24.5843000 - 2727 1 2.7573400 35.3256000 24.7266000 23.9898000 - 2728 2 -1.5660100 36.8282000 19.3992000 22.3043000 - 2729 2 -1.4981000 35.3080000 24.4003000 22.4450000 - 2730 2 -1.6275600 36.9884000 24.3913000 24.2859000 - 2731 2 -1.5880500 38.2348000 21.9057000 23.3663000 - 2732 2 -1.5605400 35.0811000 21.7999000 22.4700000 - 2733 2 -1.6534700 37.2216000 16.5307000 22.8808000 - 2734 1 2.7694800 37.2684000 24.2576000 17.3082000 - 2735 2 -1.6610300 36.2838000 17.2365000 20.3240000 - 2736 2 -1.6235300 38.2423000 18.9475000 18.4107000 - 2737 3 0.3389330 39.9518000 29.0945000 24.7421000 - 2738 2 -1.6445700 38.8326000 20.8665000 20.1350000 - 2739 1 2.6535400 36.4086000 16.6341000 24.2526000 - 2740 3 -0.0493183 44.3897000 17.0528000 25.5647000 - 2741 1 2.6813200 34.1145000 26.5210000 26.0536000 - 2742 1 2.6955900 31.4331000 26.7429000 27.7599000 - 2743 2 -1.6190100 30.8102000 28.2501000 26.6436000 - 2744 2 -1.6033800 31.8639000 27.6881000 29.0909000 - 2745 1 2.6010000 33.5099000 28.6430000 28.7489000 - 2746 2 -1.6120200 32.8077000 28.3926000 25.2337000 - 2747 2 -1.5524300 34.3766000 28.2785000 19.1629000 - 2748 1 2.8109500 32.7542000 27.9799000 18.8256000 - 2749 1 2.5605100 33.2001000 28.7573000 23.8454000 - 2750 2 -1.6071500 33.5705000 26.5465000 21.4724000 - 2751 2 -1.5600700 35.1913000 17.5390000 24.2177000 - 2752 2 -1.6548600 31.9727000 28.1336000 20.3990000 - 2753 2 -1.5563800 32.1827000 29.3822000 18.0123000 - 2754 2 -1.5183400 32.1785000 28.0790000 23.0790000 - 2755 1 2.7575700 32.1091000 27.1510000 21.7153000 - 2756 3 0.1849060 40.6579000 29.6204000 17.6190000 - 2757 2 -1.6044300 34.4700000 23.4655000 24.9212000 - 2758 2 -1.5517200 33.5992000 18.3012000 26.1915000 - 2759 2 -1.5256500 34.2618000 21.6238000 27.0048000 - 2760 1 2.7897100 30.5498000 21.9804000 28.9748000 - 2761 1 2.8337300 32.5952000 21.6182000 27.2867000 - 2762 1 2.6074500 31.9607000 29.5206000 26.5997000 - 2763 2 -1.6083100 30.7654000 18.0560000 25.2317000 - 2764 2 -1.5163200 31.4072000 18.2900000 27.8778000 - 2765 2 -1.5100400 32.1399000 21.8693000 28.7383000 - 2766 2 -1.5098300 31.9251000 20.3969000 26.5108000 - 2767 1 2.8332400 31.6667000 18.8392000 26.4116000 - 2768 2 -1.4992000 32.1644000 22.9270000 26.1173000 - 2769 2 -1.6215600 31.1093000 15.8344000 28.6516000 - 2770 1 2.7475500 31.8736000 17.3500000 29.0472000 - 2771 2 -1.5994700 32.8793000 26.1850000 27.0156000 - 2772 2 -1.3089300 35.1950000 25.8129000 19.6259000 - 2773 2 -1.5740200 34.0137000 21.1641000 16.5442000 - 2774 1 2.7899200 33.6551000 20.0595000 17.7359000 - 2775 1 2.8606000 34.8065000 22.6063000 15.8912000 - 2776 2 -1.5456000 34.1292000 20.1878000 24.4320000 - 2777 1 2.8045000 33.7587000 20.9598000 23.0570000 - 2778 2 -1.5783600 30.8760000 25.4766000 19.4363000 - 2779 2 -1.5638300 32.0836000 16.5402000 16.0403000 - 2780 2 -1.5246200 33.5459000 29.0139000 15.0802000 - 2781 2 -1.5627200 33.1619000 20.8821000 19.0810000 - 2782 2 -1.5466700 32.6546000 23.5430000 18.2757000 - 2783 2 -1.5646300 33.6743000 17.1754000 19.4258000 - 2784 2 -1.6222200 31.3794000 22.3794000 20.6501000 - 2785 1 2.5091800 35.6352000 29.6179000 18.1740000 - 2786 2 -1.5699300 32.6807000 18.9083000 17.1670000 - 2787 1 2.8202500 31.8870000 25.0568000 18.0566000 - 2788 2 -1.4946700 36.5764000 20.9980000 29.5613000 - 2789 1 2.8010400 32.9320000 22.4319000 19.4922000 - 2790 3 0.0265302 42.6254000 15.7160000 26.9062000 - 2791 1 2.8014000 32.6255000 18.0965000 15.6725000 - 2792 2 -1.4690200 34.0883000 22.9763000 20.4666000 - 2793 2 -1.5632400 31.4598000 18.4081000 20.2659000 - 2794 2 -1.6282500 31.9551000 16.3704000 21.6752000 - 2795 1 2.6498800 31.6143000 19.4839000 21.5093000 - 2796 2 -1.5510400 32.6415000 22.1592000 23.4622000 - 2797 2 -1.5635800 32.0212000 24.8199000 24.2652000 - 2798 2 -1.6341600 34.2218000 14.9661000 28.1024000 - 2799 2 -1.5925200 33.0885000 19.8758000 21.9381000 - 2800 1 2.7751200 30.6299000 25.1907000 23.4991000 - 2801 1 2.0024600 37.5529000 29.3044000 16.4564000 - 2802 1 2.6423500 32.4330000 17.1061000 20.3133000 - 2803 1 2.7466200 30.4051000 23.2489000 16.1976000 - 2804 1 2.7495600 30.9086000 15.9081000 17.0147000 - 2805 1 2.6663300 30.5830000 15.5904000 22.1905000 - 2806 1 2.8080500 34.7025000 18.8825000 25.2568000 - 2807 2 -1.6168500 30.7373000 26.2937000 22.2255000 - 2808 2 -1.5322600 32.6776000 26.5560000 18.0080000 - 2809 1 2.7921400 32.7631000 23.2745000 24.6664000 - 2810 3 0.2002500 40.3970000 14.9053000 23.5221000 - 2811 2 -1.6009600 35.0818000 28.4157000 28.2319000 - 2812 2 -1.5801300 35.6319000 26.4212000 26.8146000 - 2813 2 -1.6182300 39.7151000 27.4259000 22.7997000 - 2814 1 2.5392000 38.0629000 27.4914000 22.0161000 - 2815 2 -1.6012800 37.7835000 25.8947000 17.7072000 - 2816 2 -1.4812300 35.0628000 26.1297000 15.9280000 - 2817 2 -1.7072700 40.0126000 27.3902000 18.7060000 - 2818 2 -1.5541200 35.5491000 28.9612000 16.6578000 - 2819 1 2.8073500 34.8369000 26.9901000 20.4252000 - 2820 2 -1.5710100 36.8172000 27.9459000 23.0526000 - 2821 2 -1.5839500 37.4242000 26.9239000 20.5144000 - 2822 1 2.5875000 35.5529000 28.9921000 22.6799000 - 2823 2 -1.6046600 37.3445000 28.6842000 18.4836000 - 2824 1 2.6885500 37.8833000 27.0190000 18.8090000 - 2825 2 -1.6008800 35.1684000 28.5431000 21.2984000 - 2826 1 2.3033700 38.7775000 23.4842000 27.4931000 - 2827 2 -1.5730000 36.2887000 19.2875000 25.7587000 - 2828 2 -1.5920900 38.3757000 20.3892000 26.8980000 - 2829 2 -1.6137200 30.5015000 19.2225000 22.6732000 - 2830 1 2.7084200 37.5130000 20.8286000 28.2253000 - 2831 2 -1.6275300 38.2267000 25.0433000 27.6333000 - 2832 2 -1.5856000 36.3020000 17.2147000 28.0385000 - 2833 2 -1.5430500 36.1901000 22.1530000 15.1375000 - 2834 2 -1.7022100 41.3495000 22.1622000 29.2843000 - 2835 2 -1.6000200 36.3440000 25.1616000 29.4104000 - 2836 2 -1.5887600 35.6637000 23.7926000 27.3375000 - 2837 1 2.5738500 31.4621000 29.3212000 23.8532000 - 2838 1 2.7174000 36.3785000 25.3136000 27.7073000 - 2839 1 2.7921900 35.2426000 22.6096000 26.2092000 - 2840 1 2.7805200 37.5535000 16.8784000 27.1283000 - 2841 2 -1.6076600 36.6519000 22.0576000 25.5572000 - 2842 1 2.4721400 38.6923000 19.2897000 19.8781000 - 2843 1 2.7304800 36.1795000 29.4529000 27.2303000 - 2844 3 0.3432970 40.7941000 19.6290000 25.4479000 - 2845 2 -1.5842500 38.3057000 15.4090000 27.1429000 - 2846 2 -1.6022000 38.5068000 17.9659000 26.5076000 - 2847 2 -1.5237700 37.9856000 22.3849000 28.5654000 - 2848 2 -1.5596100 37.0581000 16.0510000 25.5420000 - 2849 2 -1.6209800 39.0833000 23.1930000 25.8310000 - 2850 1 2.7472100 37.4654000 22.1558000 19.4645000 - 2851 2 -1.6466700 38.5672000 19.4677000 24.2215000 - 2852 1 2.6291800 37.6816000 20.4130000 23.2100000 - 2853 1 2.7107500 37.9558000 19.2871000 25.7380000 - 2854 1 1.9034900 38.3830000 15.0399000 20.5066000 - 2855 2 -1.6128000 34.5839000 26.1244000 24.5608000 - 2856 1 2.7328200 36.7816000 19.7586000 17.5879000 - 2857 1 2.7262600 35.1640000 17.5780000 18.8558000 - 2858 2 -1.6389300 38.1820000 24.0855000 15.7098000 - 2859 2 -1.5627500 37.6586000 17.3162000 15.8954000 - 2860 2 -1.5739000 35.8879000 16.5459000 17.7674000 - 2861 2 -1.5893600 36.6068000 21.1090000 18.5250000 - 2862 2 -1.5981400 38.2210000 23.2500000 18.5022000 - 2863 2 -1.5873100 35.2601000 19.0705000 18.1482000 - 2864 2 -1.6608700 38.7849000 18.0413000 20.9681000 - 2865 2 -1.5628100 36.9026000 22.8359000 20.7555000 - 2866 2 -1.5456800 35.7897000 23.6238000 17.0902000 - 2867 2 -1.5785600 37.1475000 19.7573000 16.0179000 - 2868 3 0.0255884 42.4453000 25.9391000 16.8761000 - 2869 3 -0.0364132 44.2633000 29.3855000 27.6667000 - 2870 2 -1.4474200 30.7595000 24.7009000 16.8901000 - 2871 2 -1.6798100 37.4303000 27.7809000 15.4896000 - 2872 2 -1.5982200 33.6529000 17.0377000 29.5175000 - 2873 3 -0.0481341 44.3856000 24.4062000 15.4737000 - 2874 3 0.0180174 42.4023000 23.4430000 16.5846000 - 2875 3 0.3493200 40.6288000 24.7472000 15.9077000 - 2876 3 0.2707840 40.6979000 22.1860000 20.2542000 - 2877 3 0.2717810 40.4646000 22.2191000 17.8843000 - 2878 3 -0.0338902 44.2569000 29.5890000 20.2714000 - 2879 1 2.6397000 35.1156000 27.6031000 15.6287000 - 2880 3 0.2874960 40.6350000 22.3606000 15.4756000 - 2881 3 -0.0457121 44.3237000 26.8826000 15.3868000 - 2882 3 0.0030441 42.2427000 28.3820000 16.3939000 - 2883 3 0.3354070 40.3240000 24.6527000 18.1741000 - 2884 3 0.0289950 42.2274000 23.4994000 18.9883000 - 2885 3 -0.0416675 44.3030000 24.8124000 17.8909000 - 2886 3 0.3417180 40.1200000 26.8324000 16.7754000 - 2887 3 0.0833443 42.0888000 25.9805000 19.2156000 - 2888 3 -0.0320324 44.2173000 27.3036000 17.9459000 - 2889 3 0.0542019 42.5581000 18.2932000 26.8463000 - 2890 3 0.0730544 42.3166000 28.4369000 18.9569000 - 2891 3 0.2022890 40.6033000 17.2483000 23.2602000 - 2892 3 0.3025980 40.1710000 24.5412000 20.6058000 - 2893 3 0.0141414 42.2605000 23.4564000 21.5274000 - 2894 3 -0.0416443 44.1899000 24.6617000 20.2671000 - 2895 3 -0.0016576 42.4239000 15.8255000 24.4347000 - 2896 3 0.3246610 40.1379000 26.9202000 20.8417000 - 2897 3 0.0265289 42.1978000 25.8653000 21.6532000 - 2898 3 -0.0257901 44.1344000 27.1084000 20.5191000 - 2899 3 -0.0323371 44.0645000 29.5061000 22.7094000 - 2900 3 -0.0491702 44.4839000 17.3482000 23.1023000 - 2901 3 0.3312010 40.7863000 17.0068000 25.9500000 - 2902 3 0.2661420 40.7155000 19.6291000 22.8720000 - 2903 3 -0.0326828 42.2977000 28.2967000 28.7830000 - 2904 3 -0.0450992 44.3798000 26.9494000 27.8923000 - 2905 3 -0.0043532 42.4352000 25.9174000 29.0622000 - 2906 3 0.1706390 40.4846000 27.0530000 27.7206000 - 2907 3 -0.0401922 44.4673000 24.5420000 27.5540000 - 2908 3 0.0842779 42.8716000 23.5776000 28.9723000 - 2909 3 0.2308690 40.6978000 24.5232000 27.9556000 - 2910 3 -0.0488358 44.2569000 22.2714000 20.4900000 - 2911 3 0.0138362 42.5147000 20.9923000 21.6515000 - 2912 3 -0.0479037 44.3990000 19.7345000 20.4418000 - 2913 3 -0.0024247 41.9281000 28.3289000 26.2744000 - 2914 2 -1.6939600 31.4839000 22.8892000 15.0001000 - 2915 3 -0.0063480 42.5526000 18.5630000 21.8324000 - 2916 3 0.2278880 40.8089000 19.6742000 20.4484000 - 2917 3 -0.0494035 44.4008000 17.2634000 20.4934000 - 2918 3 -0.0271394 42.4333000 16.0581000 21.9719000 - 2919 3 -0.0469907 44.1937000 27.0039000 25.4648000 - 2920 3 0.1532870 40.9066000 17.2920000 20.6226000 - 2921 3 -0.0499659 44.3138000 22.3449000 17.9861000 - 2922 3 0.0119278 42.4904000 21.0154000 19.2132000 - 2923 3 -0.0487054 44.3114000 19.7924000 17.8815000 - 2924 3 -0.0043692 42.3346000 25.8264000 26.6715000 - 2925 3 -0.0111520 42.5327000 18.4212000 19.1756000 - 2926 3 0.2466130 40.5923000 19.8438000 18.0047000 - 2927 3 0.2678650 39.9695000 26.8537000 25.2707000 - 2928 3 -0.0489537 44.3723000 17.3078000 17.8728000 - 2929 3 -0.0490314 44.2519000 24.6353000 25.1731000 - 2930 3 0.0417768 42.4226000 23.1423000 26.4847000 - 2931 3 -0.0313674 42.4509000 15.8279000 19.4454000 - 2932 3 0.2153240 40.7072000 17.3732000 18.1503000 - 2933 3 -0.0487070 44.3318000 21.9568000 15.4632000 - 2934 3 0.0091696 42.3240000 20.9740000 16.6988000 - 2935 3 0.2654650 40.5587000 24.6553000 25.5331000 - 2936 3 -0.0445503 44.3346000 19.5010000 15.4735000 - 2937 3 0.0241151 42.1712000 18.5685000 16.5050000 - 2938 3 0.3956360 40.2022000 20.1320000 15.4213000 - 2939 3 -0.0432846 44.3562000 17.1135000 15.5041000 - 2940 3 0.0005774 42.3581000 16.0914000 16.9116000 - 2941 3 0.0986448 41.8970000 28.3924000 23.6812000 - 2942 3 0.3658540 40.4709000 16.9016000 15.5835000 - 2943 3 0.0579359 42.0025000 28.4719000 21.2853000 - 2944 1 2.7945100 35.3031000 16.7951000 29.4055000 - 2945 3 -0.0401456 44.1125000 27.0619000 23.0129000 - 2946 3 0.0180053 42.1047000 25.8556000 24.1824000 - 2947 3 0.4650280 38.1303000 25.4781000 22.9596000 - 2948 3 -0.0495797 44.1797000 24.6782000 22.6768000 - 2949 3 0.0281130 42.3249000 23.3773000 24.0416000 - 2950 3 -0.0451602 44.3335000 29.4728000 15.5342000 - 2951 3 0.0155854 42.5062000 18.3242000 24.2951000 - 2952 3 0.2641020 40.3220000 24.5264000 23.0377000 - 2953 3 0.4039760 40.4517000 22.0877000 27.5086000 - 2954 3 0.4122520 40.6534000 22.0114000 25.0735000 - 2955 3 -0.0486567 44.5117000 19.8447000 23.0474000 - 2956 3 0.3138420 40.6682000 22.0751000 22.6498000 - 2957 3 0.0249862 42.5212000 20.8502000 24.1594000 - 2958 3 -0.0497879 44.3291000 22.2913000 22.9453000 - 2959 3 0.1553630 39.6146000 29.0767000 15.7260000 - 2960 3 -0.0357076 44.0771000 29.4686000 25.1871000 - 2961 3 -0.0280926 44.4780000 22.1204000 27.8586000 - 2962 3 0.1184350 42.9088000 20.8210000 29.2237000 - 2963 3 -0.0369186 44.5373000 19.6205000 27.9532000 - 2964 3 0.0610255 42.5870000 18.5500000 29.2232000 - 2965 3 0.5138380 40.4380000 19.4468000 28.0611000 - 2966 3 -0.0479056 44.4320000 17.1326000 28.0345000 - 2967 3 0.0324198 42.5793000 16.0832000 29.2275000 - 2968 3 0.4055470 39.0277000 17.5314000 28.6191000 - 2969 3 -0.0442181 44.3628000 22.0736000 25.4246000 - 2970 3 0.0480730 42.5180000 20.6947000 26.7862000 - 2971 3 -0.0461617 44.3360000 19.5564000 25.4800000 - 2972 3 -0.0396749 44.3669000 29.6901000 17.9327000 - 2973 1 2.7314000 37.5611000 18.6204000 14.9112000 - 2974 2 -1.5903500 37.8565000 29.6941000 26.6614000 - 2975 2 -1.5983500 30.2093000 22.0592000 17.3530000 - 2976 2 -1.6091700 38.3015000 20.0040000 29.6175000 - 2977 3 0.1703220 40.4468000 29.6896000 27.9093000 - 2978 1 2.6879800 30.2216000 23.0611000 21.4011000 - 2979 3 -0.0437198 44.4207000 14.9128000 23.1289000 - 2980 1 2.3845600 38.1920000 29.7333000 19.4423000 - 2981 2 -1.5806000 34.1463000 18.2015000 14.9341000 - 2982 3 -0.0218192 48.3026000 14.7980000 3.1457300 - 2983 3 0.2046960 59.8738000 0.1175990 10.3178000 - 2984 3 -0.0220607 52.0629000 14.8046000 12.8539000 - 2985 3 -0.0221586 52.1828000 14.8258000 5.5363600 - 2986 3 -0.0489148 55.8149000 7.3257900 0.7131380 - 2987 3 0.2118530 59.8569000 8.0371500 0.3980890 - 2988 3 -0.0463530 55.9056000 2.4086300 0.8266680 - 2989 3 0.0419270 57.8711000 1.3283400 2.0735800 - 2990 3 0.3099710 59.8500000 2.5109200 0.4845220 - 2991 3 -0.0454912 55.6694000 4.7870100 0.3598880 - 2992 3 0.0345491 57.9736000 3.7893300 1.8563700 - 2993 3 0.3259850 59.6373000 4.7934400 0.3387910 - 2994 3 0.0321630 58.0087000 6.3774000 1.9340100 - 2995 3 -0.0504198 55.9477000 7.3346400 3.2270800 - 2996 3 0.2157080 59.8886000 7.9375800 2.9938100 - 2997 3 -0.0377293 56.1088000 2.4651700 3.2435800 - 2998 3 0.0923988 58.1429000 1.2789900 4.3805400 - 2999 3 -0.0217444 52.1029000 14.7617000 3.0847000 - 3000 3 -0.0449738 56.0214000 4.9190400 2.8807700 - 3001 3 0.0531770 57.9882000 3.6587600 4.3398700 - 3002 3 0.2622510 59.4541000 0.3415870 0.8177710 - 3003 3 0.0303682 57.9901000 5.9963500 4.2427300 - 3004 3 -0.0490910 55.9786000 7.3074100 5.6875400 - 3005 3 0.2825990 59.7546000 7.2324700 5.2721200 - 3006 3 -0.0422316 56.0978000 2.4807100 5.7141000 - 3007 3 0.0629313 58.5171000 0.9126650 6.7075700 - 3008 3 -0.0187768 52.0601000 14.7718000 0.6417620 - 3009 3 -0.0463073 56.0135000 4.8959000 5.3731400 - 3010 3 0.0212918 58.0983000 3.4096700 6.8186500 - 3011 3 0.3567770 60.0081000 4.4209600 5.5178300 - 3012 3 0.0064323 58.0611000 5.8997500 6.6878900 - 3013 3 -0.0218425 46.4396000 0.9832320 2.0297000 - 3014 3 -0.0259382 46.4313000 3.5545900 1.9929900 - 3015 3 -0.0288308 46.4364000 6.0634200 1.9991400 - 3016 3 -0.0262280 46.4653000 1.1132100 4.5479200 - 3017 3 -0.0288157 46.4782000 3.6580700 4.5580100 - 3018 3 -0.0311374 46.4930000 6.1084700 4.5288600 - 3019 3 -0.0284623 46.3899000 1.1856000 7.0243800 - 3020 3 -0.0297932 46.4324000 3.6903400 6.9938100 - 3021 3 -0.0305605 46.4248000 6.1407000 7.0055500 - 3022 3 -0.0218268 48.2750000 7.3077800 0.7250000 - 3023 3 -0.0226187 52.0471000 7.3271100 0.6660380 - 3024 3 -0.0180567 48.2872000 2.3477400 0.7022000 - 3025 3 -0.0191549 50.2272000 1.1300200 1.9313500 - 3026 3 -0.0213259 52.0768000 2.3569500 0.6298180 - 3027 3 -0.0347140 54.0453000 1.1501200 1.8631600 - 3028 3 -0.0207058 48.2489000 4.8368800 0.7205420 - 3029 3 -0.0231480 50.1769000 3.5888100 1.9289100 - 3030 3 -0.0227658 52.0184000 4.8357900 0.7016360 - 3031 3 -0.0361021 53.9774000 3.6404200 1.8608100 - 3032 3 -0.0240418 50.1600000 6.0874200 1.9563100 - 3033 3 -0.0347129 54.0100000 6.1128100 1.9309000 - 3034 3 -0.0262170 48.3248000 7.3509400 3.2262700 - 3035 3 -0.0261283 52.1161000 7.3201300 3.1359300 - 3036 3 -0.0229518 48.3069000 2.3716900 3.2299800 - 3037 3 -0.0220766 50.2862000 1.1804000 4.4013000 - 3038 3 -0.0257755 52.1514000 2.3801800 3.1401300 - 3039 3 -0.0362821 54.1477000 1.2001200 4.2949300 - 3040 3 -0.0253923 48.2840000 4.8757100 3.2461900 - 3041 3 -0.0255691 50.2365000 3.6512800 4.3949600 - 3042 3 -0.0267330 52.1106000 4.8479900 3.1699900 - 3043 3 -0.0378608 54.1073000 3.6475100 4.3245300 - 3044 3 -0.0260384 50.2169000 6.1374400 4.4260100 - 3045 3 -0.0364395 54.0139000 6.1344700 4.3806400 - 3046 3 -0.0265944 48.3681000 7.3787100 5.7195400 - 3047 3 -0.0268520 52.1841000 7.4041500 5.6516300 - 3048 3 -0.0249922 48.3310000 2.4078300 5.7231000 - 3049 3 -0.0226965 50.2947000 1.2226000 6.8751300 - 3050 3 -0.0265734 52.1821000 2.4425800 5.6079800 - 3051 3 -0.0358411 54.0864000 1.2422100 6.7438800 - 3052 3 -0.0264545 48.3505000 4.9121900 5.7417800 - 3053 3 -0.0257464 50.2556000 3.6602300 6.8772200 - 3054 3 -0.0274278 52.1507000 4.9132400 5.6218300 - 3055 3 -0.0366953 54.0833000 3.6785900 6.8184700 - 3056 3 -0.0260996 50.2759000 6.1767700 6.8812100 - 3057 3 -0.0357214 54.0615000 6.1269600 6.8765300 - 3058 3 -0.0483465 55.8288000 9.8554100 0.6130030 - 3059 3 -0.0082582 57.8730000 8.7638500 1.6065800 - 3060 3 0.2417720 59.3826000 10.2700000 0.4164320 - 3061 3 -0.0459894 55.7890000 12.4000000 0.6457680 - 3062 3 0.0125497 57.8464000 11.2143000 2.0153300 - 3063 3 0.2902350 59.5501000 12.5751000 0.4473010 - 3064 3 0.0249372 57.7894000 13.5838000 1.6081500 - 3065 3 -0.0524560 55.9003000 9.8263100 3.1002700 - 3066 3 0.0013511 57.8622000 8.7379200 4.1422900 - 3067 3 0.2746270 59.8129000 10.1790000 2.8627900 - 3068 3 -0.0486281 55.9249000 12.3281000 3.1551400 - 3069 3 0.0120664 57.9701000 11.1322000 4.4152400 - 3070 3 0.3290470 59.7468000 12.4959000 3.0138000 - 3071 3 0.0445975 58.0129000 13.6110000 4.2080000 - 3072 3 -0.0496218 55.9742000 9.9016500 5.6024600 - 3073 3 0.0120203 57.8485000 8.5067000 6.6640000 - 3074 3 0.2637930 59.6521000 9.8122200 5.6115700 - 3075 3 -0.0509271 56.0352000 12.3588000 5.6145400 - 3076 3 0.0215506 57.9180000 10.9917000 6.9759100 - 3077 3 0.3164520 59.8634000 12.2045000 5.4265700 - 3078 3 0.0208929 58.1941000 13.4683000 6.6281400 - 3079 3 -0.0188580 48.3071000 14.7185000 0.6891800 - 3080 3 -0.0313341 46.4425000 8.5920400 2.0238000 - 3081 3 -0.0423059 55.9072000 0.0223576 12.7270000 - 3082 3 -0.0318614 46.4132000 11.0673000 1.8931600 - 3083 2 -1.6329500 59.8844000 2.5594600 3.0267000 - 3084 3 -0.0356574 54.1047000 13.5270000 14.1789000 - 3085 3 -0.0310716 46.3387000 13.5329000 1.9016400 - 3086 3 -0.0331673 46.4361000 8.6096100 4.4416300 - 3087 3 -0.0342195 46.4624000 11.1213000 4.3802400 - 3088 3 -0.0217746 48.2647000 14.8005000 10.5653000 - 3089 3 -0.0325662 46.3992000 13.6232000 4.3866400 - 3090 3 -0.0319945 46.4203000 8.5876000 6.9185500 - 3091 3 -0.0331499 46.4378000 11.0739000 6.9087000 - 3092 3 -0.0191654 50.1461000 13.5057000 14.1791000 - 3093 3 -0.0313392 46.4135000 13.5514000 6.9017900 - 3094 3 -0.0224295 48.2874000 9.8189100 0.6831610 - 3095 3 -0.0239806 50.2254000 8.5766300 1.9227300 - 3096 3 -0.0227652 52.0494000 9.8242800 0.6520840 - 3097 3 -0.0345648 53.9946000 8.5874700 1.9332100 - 3098 3 -0.0210825 48.2958000 12.2711000 0.7083960 - 3099 3 -0.0235650 50.2010000 11.0833000 1.9652300 - 3100 3 -0.0218611 52.0599000 12.2895000 0.6745460 - 3101 3 -0.0347295 54.0083000 11.0722000 1.9213900 - 3102 3 -0.0511671 56.1428000 14.8198000 5.2461300 - 3103 3 -0.0205164 50.2061000 13.5538000 1.9444200 - 3104 3 -0.0399016 56.1137000 14.7993000 12.7356000 - 3105 3 -0.0340852 53.9731000 13.5897000 1.9673100 - 3106 3 -0.0265216 48.3354000 9.8137600 3.1915500 - 3107 3 -0.0259451 50.2786000 8.6321700 4.4178300 - 3108 3 -0.0259001 52.1229000 9.8358400 3.1607200 - 3109 3 -0.0363077 54.0258000 8.5825800 4.4059500 - 3110 3 -0.0250637 48.3287000 12.3256000 3.1886700 - 3111 3 -0.0255397 50.2746000 11.1085000 4.4293900 - 3112 3 -0.0254597 52.1493000 12.3047000 3.1644500 - 3113 3 -0.0368899 54.0132000 11.0716000 4.3861400 - 3114 3 -0.0367659 56.0455000 0.0653863 5.4080700 - 3115 3 -0.0225202 50.2376000 13.5959000 4.3925400 - 3116 3 -0.0216721 50.1741000 11.0436000 14.3145000 - 3117 3 -0.0357667 54.1044000 13.5950000 4.4171200 - 3118 3 -0.0265394 48.3806000 9.8744200 5.6571300 - 3119 3 -0.0259798 50.2953000 8.6905400 6.9127000 - 3120 3 -0.0270031 52.1558000 9.8939000 5.6964900 - 3121 3 -0.0356886 54.0764000 8.6114600 6.8707200 - 3122 3 -0.0252488 48.3560000 12.3778000 5.6438200 - 3123 3 -0.0255576 50.3120000 11.1773000 6.9116600 - 3124 3 -0.0262325 52.1599000 12.3750000 5.6324600 - 3125 3 -0.0364734 54.0934000 11.1383000 6.8601400 - 3126 3 -0.0244003 48.2386000 12.3151000 13.0908000 - 3127 3 -0.0228719 50.2525000 13.6383000 6.8176200 - 3128 3 -0.0323354 54.0858000 8.6109800 14.4164000 - 3129 3 -0.0375310 54.1555000 13.6065000 6.8763700 - 3130 3 -0.0482196 56.0147000 7.3147200 8.1178100 - 3131 3 0.1799730 59.7238000 7.3547500 7.7276400 - 3132 3 -0.0488744 56.0056000 2.4565400 8.1130400 - 3133 3 -0.0001668 58.1654000 1.1830400 9.1715800 - 3134 3 0.2745530 60.3124000 2.1893900 7.8483200 - 3135 3 -0.0496417 56.0764000 4.9072600 7.8790300 - 3136 3 -0.0180697 57.8280000 3.6381000 9.3609900 - 3137 3 0.1773610 59.5615000 4.7551500 8.1867400 - 3138 3 -0.0260792 57.9784000 6.1764100 9.2785400 - 3139 3 -0.0458110 56.0610000 7.5583300 10.5314000 - 3140 3 -0.0215294 48.2976000 14.8154000 8.0958600 - 3141 3 -0.0499069 55.9648000 2.4586900 10.5802000 - 3142 3 -0.0273413 57.9719000 1.3437300 11.6438000 - 3143 3 0.1451440 59.8164000 2.5136000 10.5229000 - 3144 3 -0.0484601 55.9391000 5.0486000 10.4638000 - 3145 3 -0.0305493 57.9453000 3.7233400 11.9277000 - 3146 3 0.1119580 59.5789000 4.9401400 10.7251000 - 3147 3 -0.0115371 57.7156000 6.1454900 11.7781000 - 3148 3 -0.0349589 56.0712000 7.4531000 13.1139000 - 3149 3 0.2257290 59.7345000 7.1939600 12.1537000 - 3150 3 -0.0460264 55.9309000 2.5608300 13.0127000 - 3151 3 -0.0096220 57.8034000 1.3084500 14.0888000 - 3152 3 0.1299440 59.7782000 2.6476700 13.0357000 - 3153 3 -0.0429006 55.9325000 4.9969700 13.0317000 - 3154 3 -0.0087153 57.9831000 3.7254700 14.3372000 - 3155 3 0.1547560 60.0375000 4.9589000 13.1115000 - 3156 3 0.0514107 58.1068000 6.0928900 14.1680000 - 3157 3 -0.0286866 46.4179000 1.2862100 9.5379400 - 3158 3 -0.0298628 46.4432000 3.6961600 9.4710700 - 3159 3 -0.0297331 46.4858000 6.1768800 9.4557100 - 3160 3 -0.0279879 46.3069000 1.0250600 12.0309000 - 3161 3 -0.0295175 46.3885000 3.5286500 11.8873000 - 3162 3 -0.0304178 46.3725000 6.1141700 11.9230000 - 3163 3 -0.0260462 46.3081000 1.0207700 14.4873000 - 3164 3 -0.0273458 46.3889000 3.5653200 14.3693000 - 3165 3 -0.0303472 46.3961000 6.1157600 14.3627000 - 3166 3 -0.0257814 48.3609000 7.4458800 8.1955800 - 3167 3 -0.0265715 52.2118000 7.4144900 8.1445000 - 3168 3 -0.0251054 48.3176000 2.4790400 8.1725400 - 3169 3 -0.0226442 50.2584000 1.2367700 9.3492500 - 3170 3 -0.0252044 52.1896000 2.4716400 8.0839300 - 3171 3 -0.0329727 54.0515000 1.2177600 9.2424400 - 3172 3 -0.0262836 48.3469000 4.9383100 8.1938500 - 3173 3 -0.0256623 50.2649000 3.7244500 9.3644500 - 3174 3 -0.0264505 52.1656000 4.9518700 8.1127800 - 3175 3 -0.0341849 54.0527000 3.7384200 9.3148500 - 3176 3 -0.0260580 50.3135000 6.2039600 9.3547300 - 3177 3 -0.0345248 54.0758000 6.2174700 9.3455500 - 3178 3 -0.0257683 48.4031000 7.4216200 10.6669000 - 3179 3 -0.0271682 52.2352000 7.4652600 10.6263000 - 3180 3 -0.0246314 48.3614000 2.4543500 10.6052000 - 3181 3 -0.0214712 50.1964000 1.1869200 11.8115000 - 3182 3 -0.0244957 52.1721000 2.4724300 10.5591000 - 3183 3 -0.0300785 54.0143000 1.2487700 11.7226000 - 3184 3 -0.0256044 48.3577000 4.9475800 10.6429000 - 3185 3 -0.0249113 50.2555000 3.6633800 11.8363000 - 3186 3 -0.0264613 52.1725000 4.9469200 10.6015000 - 3187 3 -0.0324413 54.0455000 3.7266700 11.7667000 - 3188 3 -0.0254492 50.3379000 6.1536400 11.8326000 - 3189 3 -0.0330095 54.1287000 6.2579500 11.7931000 - 3190 3 -0.0248013 48.3737000 7.3529600 13.0860000 - 3191 3 -0.0252778 52.2205000 7.3977500 13.1275000 - 3192 3 -0.0228785 48.2538000 2.3874500 13.0756000 - 3193 3 -0.0179971 50.1340000 1.1448900 14.2475000 - 3194 3 -0.0224833 52.1123000 2.3886500 12.9914000 - 3195 3 -0.0254860 53.9825000 1.1646800 14.2052000 - 3196 3 -0.0243548 48.3107000 4.8820500 13.0856000 - 3197 3 -0.0210947 50.1527000 3.5931700 14.3143000 - 3198 3 -0.0248357 52.1813000 4.9052300 13.0431000 - 3199 3 -0.0288819 53.9688000 3.6606400 14.2487000 - 3200 3 -0.0220214 50.2353000 6.0815700 14.3116000 - 3201 3 -0.0301427 54.1171000 6.1523300 14.2465000 - 3202 3 -0.0430501 55.9797000 9.8035800 8.1712900 - 3203 3 0.0196798 57.8877000 8.6053000 9.1752000 - 3204 3 0.1660880 59.7834000 0.1240340 12.7241000 - 3205 3 -0.0496899 56.0848000 12.3705000 8.0806200 - 3206 3 0.0618343 57.8865000 11.0467000 9.4502600 - 3207 3 0.3146330 60.0914000 12.0323000 7.9236100 - 3208 3 -0.0259356 52.1445000 9.8881500 13.1080000 - 3209 3 0.0302649 58.1200000 13.4797000 9.0562500 - 3210 3 -0.0408146 56.0385000 9.9904600 10.7052000 - 3211 3 0.0207072 57.9030000 8.7706800 11.7805000 - 3212 3 0.3118410 60.1613000 9.4229300 11.3385000 - 3213 3 -0.0464836 56.1063000 12.4205000 10.5518000 - 3214 3 0.0609586 58.1460000 11.1439000 11.8887000 - 3215 3 0.3743110 59.8280000 12.5092000 10.3445000 - 3216 3 -0.0221745 50.2355000 8.5784200 14.3281000 - 3217 3 0.0655297 58.1177000 13.5813000 11.7316000 - 3218 3 0.2996510 60.0719000 14.4703000 7.7555900 - 3219 3 -0.0387129 56.1356000 9.9069200 13.1796000 - 3220 3 0.0782301 58.0710000 8.5362700 14.5978000 - 3221 3 0.3558480 59.7643000 9.6002600 13.4446000 - 3222 3 -0.0446465 56.0866000 12.3120000 12.9999000 - 3223 3 0.0572704 58.1355000 11.0529000 14.3455000 - 3224 3 0.4679830 59.9873000 12.3339000 12.7446000 - 3225 3 -0.0252348 48.3191000 9.8614700 13.0747000 - 3226 3 0.0822605 58.0130000 13.4689000 14.1863000 - 3227 3 -0.0394721 54.1252000 13.6221000 11.7558000 - 3228 3 -0.0222026 48.2748000 14.8438000 5.6195000 - 3229 3 -0.0311357 46.4862000 8.6125100 9.4116000 - 3230 3 -0.0327683 46.4259000 11.0804000 9.4162000 - 3231 3 0.2947240 59.7502000 14.8608000 10.2603000 - 3232 3 -0.0316444 46.3810000 13.6249000 9.4177400 - 3233 3 -0.0322891 46.4169000 8.6089500 11.8931000 - 3234 3 -0.0341835 46.3606000 11.1236000 11.8588000 - 3235 3 -0.0351942 54.0677000 11.0395000 14.3074000 - 3236 3 -0.0335865 46.3228000 13.5905000 11.8578000 - 3237 3 -0.0329591 46.4043000 8.6236100 14.3793000 - 3238 3 -0.0340175 46.3290000 11.0862000 14.3459000 - 3239 3 -0.0570753 56.1866000 14.8397000 7.6908500 - 3240 3 -0.0252452 52.1378000 12.3723000 13.0873000 - 3241 3 -0.0327453 46.2579000 13.5260000 14.3254000 - 3242 3 -0.0256924 48.3469000 9.9164300 8.1343700 - 3243 3 -0.0259002 50.3351000 8.6852200 9.3821100 - 3244 3 -0.0267631 52.2271000 9.9236500 8.1354400 - 3245 3 -0.0357104 54.0991000 8.6884300 9.3843200 - 3246 3 -0.0247447 48.3337000 12.3598000 8.1168400 - 3247 3 -0.0257837 50.3022000 11.1485000 9.3919500 - 3248 3 -0.0264078 52.1878000 12.4536000 8.1540000 - 3249 3 -0.0370015 54.1435000 11.1927000 9.3514100 - 3250 3 -0.0204686 48.2780000 14.7745000 13.0518000 - 3251 3 -0.0231853 50.2373000 13.6190000 9.2856400 - 3252 3 -0.0229202 50.1401000 13.5907000 11.7214000 - 3253 3 -0.0384102 54.1413000 13.6365000 9.3216300 - 3254 3 -0.0260506 48.3729000 9.8635900 10.6239000 - 3255 3 -0.0255156 50.3285000 8.6467200 11.8518000 - 3256 3 -0.0274798 52.2394000 9.9388800 10.6304000 - 3257 3 -0.0355273 54.1265000 8.7806900 11.9609000 - 3258 3 -0.0255098 48.2921000 12.3065000 10.5819000 - 3259 3 -0.0252676 50.2495000 11.1542000 11.8759000 - 3260 3 -0.0271371 52.1752000 12.4401000 10.6310000 - 3261 3 -0.0378273 54.1040000 11.2176000 11.8948000 - 3262 3 0.2538560 60.1965000 7.4219200 9.9347900 - 3263 3 -0.0239477 52.1571000 14.8479000 10.3477000 - 3264 3 -0.0453629 55.9852000 14.7590000 2.8628000 - 3265 3 -0.0481904 56.1980000 14.8345000 10.1904000 - 3266 3 -0.0474478 56.0178000 0.0237003 10.2532000 - 3267 3 -0.0453692 56.0629000 0.0043727 7.8107800 - 3268 3 -0.0447860 55.8675000 14.8722000 0.4134070 - 3269 3 -0.0223759 52.1704000 14.6350000 17.7947000 - 3270 3 -0.0500973 55.9591000 14.5620000 17.7126000 - 3271 3 -0.0209629 48.1512000 14.7667000 22.9497000 - 3272 3 0.2052890 59.5601000 0.2137460 20.4977000 - 3273 3 -0.0226364 50.1042000 13.5296000 24.1541000 - 3274 3 -0.0259843 48.2436000 9.8311500 25.5075000 - 3275 3 -0.0215995 52.0257000 14.7722000 25.3982000 - 3276 3 0.4811840 59.6664000 14.6944000 15.5044000 - 3277 3 -0.0260450 51.9539000 9.8348200 25.4325000 - 3278 3 -0.0248083 48.2142000 12.3192000 25.4569000 - 3279 3 0.2366430 59.4288000 14.6448000 22.8918000 - 3280 3 -0.0425887 55.7791000 14.7434000 27.7418000 - 3281 3 -0.0254160 50.1392000 8.5551500 26.7285000 - 3282 3 -0.0350274 56.0595000 7.3235700 15.5831000 - 3283 3 0.3206850 59.8920000 7.2479400 16.0605000 - 3284 3 -0.0455428 55.7955000 2.3302000 15.4280000 - 3285 3 -0.0264849 57.4015000 0.9260830 16.6233000 - 3286 3 0.1820110 59.6561000 2.3642800 15.4957000 - 3287 3 -0.0402349 55.9734000 4.7695200 15.4053000 - 3288 3 0.0253968 57.6724000 3.5001100 16.7206000 - 3289 3 0.2648880 59.9421000 4.7919100 15.5625000 - 3290 3 0.0379163 57.8855000 6.0199600 16.6280000 - 3291 3 -0.0445011 55.9979000 7.2449400 18.0754000 - 3292 3 0.2989880 59.7350000 7.3202300 18.3342000 - 3293 3 -0.0460752 55.7536000 2.2854900 17.9306000 - 3294 3 -0.0086840 57.6175000 1.0281800 19.1871000 - 3295 3 0.2103390 59.2911000 1.6589400 17.6038000 - 3296 3 -0.0382434 55.9656000 4.7940300 17.8926000 - 3297 3 0.0609962 57.8431000 3.4588500 19.1759000 - 3298 3 -0.0204798 52.1381000 14.7144000 15.3365000 - 3299 3 0.0543859 57.8634000 5.9965700 19.2158000 - 3300 3 -0.0455432 55.8684000 7.2988800 20.5542000 - 3301 3 0.2595730 59.5917000 7.3540000 20.7701000 - 3302 3 -0.0435079 55.7622000 2.3839600 20.3932000 - 3303 3 0.0070034 57.7391000 1.3317300 21.6470000 - 3304 3 0.2768960 59.6173000 2.6191400 20.6782000 - 3305 3 -0.0394425 55.8646000 4.8866200 20.3044000 - 3306 3 0.0290510 57.6926000 3.7745800 21.5715000 - 3307 3 0.3257380 60.1897000 4.7853100 19.7375000 - 3308 3 0.0237852 57.7398000 6.1347900 21.8309000 - 3309 3 -0.0272426 46.3356000 1.0960500 16.9227000 - 3310 3 -0.0302087 46.3768000 3.6046000 16.8360000 - 3311 3 -0.0330335 46.3530000 6.1172400 16.8551000 - 3312 3 -0.0301021 46.2230000 1.2260700 19.4068000 - 3313 3 -0.0327538 46.2958000 3.6383600 19.3353000 - 3314 3 -0.0338970 46.2697000 6.0883700 19.2928000 - 3315 3 -0.0308683 46.1354000 1.0012100 21.8394000 - 3316 3 -0.0326131 46.2433000 3.4792300 21.7574000 - 3317 3 -0.0329810 46.2590000 6.0182900 21.7436000 - 3318 3 -0.0229832 48.3110000 7.3552000 15.5360000 - 3319 3 -0.0234937 52.1306000 7.2975700 15.5760000 - 3320 3 -0.0194727 48.2441000 2.2896600 15.5612000 - 3321 3 -0.0190874 50.1678000 1.1059500 16.7137000 - 3322 3 -0.0200286 52.0569000 2.3706000 15.4397000 - 3323 3 -0.0290395 53.8629000 1.0738800 16.7021000 - 3324 3 -0.0219806 48.2693000 4.8024400 15.5574000 - 3325 3 -0.0234752 50.1343000 3.5803400 16.7698000 - 3326 3 -0.0231981 52.1086000 4.8535200 15.4856000 - 3327 3 -0.0331781 53.9383000 3.5567200 16.7035000 - 3328 3 -0.0242940 50.1564000 6.0469000 16.7860000 - 3329 3 -0.0345714 54.0383000 6.0482800 16.7086000 - 3330 3 -0.0265202 48.2328000 7.3089800 18.0189000 - 3331 3 -0.0269077 52.0349000 7.2913700 18.0379000 - 3332 3 -0.0236421 48.2487000 2.3243900 18.0772000 - 3333 3 -0.0216826 50.1271000 1.1163700 19.2252000 - 3334 3 -0.0242552 52.0068000 2.3602100 17.9731000 - 3335 3 -0.0318880 53.8865000 1.1307100 19.1590000 - 3336 3 -0.0259474 48.2437000 4.8332300 18.0426000 - 3337 3 -0.0255242 50.0985000 3.5942200 19.2575000 - 3338 3 -0.0266168 52.0711000 4.8348700 17.9662000 - 3339 3 -0.0351804 53.9391000 3.5984900 19.1437000 - 3340 3 -0.0260269 50.1053000 6.0418600 19.2955000 - 3341 3 -0.0362462 54.0013000 6.1544800 19.1525000 - 3342 3 -0.0269242 48.1802000 7.3160400 20.5257000 - 3343 3 -0.0269094 52.0308000 7.3041600 20.4844000 - 3344 3 -0.0250902 48.1886000 2.3355200 20.5647000 - 3345 3 -0.0220274 50.0462000 1.1137000 21.7174000 - 3346 3 -0.0250226 52.0260000 2.3732000 20.4575000 - 3347 3 -0.0333400 53.8931000 1.1171400 21.6311000 - 3348 3 -0.0265871 48.1789000 4.8375600 20.5874000 - 3349 3 -0.0255555 50.0780000 3.5995200 21.7108000 - 3350 3 -0.0272727 52.0529000 4.8634700 20.4476000 - 3351 3 -0.0357994 53.8682000 3.6180700 21.6434000 - 3352 3 -0.0259083 50.0965000 6.0641700 21.7952000 - 3353 3 -0.0359452 53.9298000 6.1134000 21.6673000 - 3354 3 -0.0441325 56.0723000 9.8101800 15.6179000 - 3355 3 0.0276460 58.0537000 8.5665800 16.9977000 - 3356 3 0.3720590 60.1333000 9.6759600 15.7871000 - 3357 3 -0.0508917 56.1105000 12.2412000 15.4418000 - 3358 3 0.0152125 58.1967000 10.9262000 16.7910000 - 3359 3 0.3661400 60.0198000 12.3564000 15.1624000 - 3360 3 -0.0335895 53.8653000 13.5639000 24.0862000 - 3361 3 0.0329945 58.0284000 13.3298000 16.6586000 - 3362 3 -0.0463687 55.9467000 14.7032000 15.1863000 - 3363 3 -0.0537410 56.0048000 9.7173200 18.0544000 - 3364 3 0.0024898 57.8913000 8.4578300 19.3490000 - 3365 3 0.2433630 59.9011000 9.7689600 18.1041000 - 3366 3 -0.0570501 56.0690000 12.1168000 17.9031000 - 3367 3 -0.0214932 57.9798000 10.9731000 19.1931000 - 3368 3 0.1968750 59.8164000 12.4952000 18.4052000 - 3369 3 -0.0203465 57.8420000 13.4902000 19.1734000 - 3370 3 -0.0344498 53.8695000 11.0607000 26.6471000 - 3371 3 -0.0524934 55.9087000 9.8535400 20.4703000 - 3372 3 -0.0056229 57.7140000 8.6483900 21.7406000 - 3373 3 0.2112040 59.5807000 9.7575900 20.5126000 - 3374 3 -0.0545850 55.9128000 12.2852000 20.4296000 - 3375 3 -0.0168180 57.7303000 11.0977000 21.8414000 - 3376 3 0.2200670 59.5598000 12.2080000 20.7353000 - 3377 3 -0.0027566 57.7343000 13.5133000 21.6434000 - 3378 3 -0.0350286 46.3624000 8.6129800 16.8452000 - 3379 3 -0.0355805 46.2985000 11.0542000 16.8424000 - 3380 3 0.2423540 59.9157000 14.4045000 20.0596000 - 3381 3 -0.0300974 53.9111000 13.5608000 29.1704000 - 3382 3 -0.0328554 46.2786000 13.5264000 16.7710000 - 3383 3 -0.0342520 46.2825000 8.5716500 19.3006000 - 3384 3 -0.0340655 46.2401000 11.0273000 19.2911000 - 3385 3 -0.0316573 46.2338000 13.4939000 19.2356000 - 3386 3 -0.0334578 46.2957000 8.5712700 21.7561000 - 3387 3 -0.0325682 46.2386000 11.0464000 21.7653000 - 3388 3 -0.0177344 50.1637000 13.5156000 29.2149000 - 3389 3 -0.0299327 46.2872000 13.5033000 21.7775000 - 3390 3 -0.0236286 48.2495000 9.8458700 15.5621000 - 3391 3 -0.0243170 50.1736000 8.5230300 16.7914000 - 3392 3 -0.0232914 52.0982000 9.8013700 15.5722000 - 3393 3 -0.0355987 54.0114000 8.4528400 16.8652000 - 3394 3 -0.0221988 48.2107000 12.3045000 15.5423000 - 3395 3 -0.0240079 50.1468000 11.0312000 16.7566000 - 3396 3 -0.0227256 52.0852000 12.3066000 15.5320000 - 3397 3 -0.0363767 54.0533000 10.8997000 16.7124000 - 3398 3 -0.0210113 50.1728000 11.0302000 29.2446000 - 3399 3 -0.0217970 50.1013000 13.5247000 16.6999000 - 3400 3 -0.0233191 48.2488000 12.2813000 27.9680000 - 3401 3 -0.0363076 54.0912000 13.3344000 16.6061000 - 3402 3 -0.0268361 48.2139000 9.7822500 18.0306000 - 3403 3 -0.0259965 50.1682000 8.5156300 19.2836000 - 3404 3 -0.0261202 52.1059000 9.7814600 18.0228000 - 3405 3 -0.0363008 53.9949000 8.6212400 19.2669000 - 3406 3 -0.0254233 48.1579000 12.2534000 17.9986000 - 3407 3 -0.0257422 50.1443000 11.0121000 19.2286000 - 3408 3 -0.0250230 52.1229000 12.2465000 17.9704000 - 3409 3 -0.0361557 54.0519000 11.0563000 19.1857000 - 3410 3 -0.0308231 53.8884000 8.5901900 29.1851000 - 3411 3 -0.0234895 50.1077000 13.5030000 19.1632000 - 3412 3 -0.0248545 50.1522000 11.0550000 26.7515000 - 3413 3 -0.0352717 54.0824000 13.4881000 19.1065000 - 3414 3 -0.0269861 48.2320000 9.7951500 20.5042000 - 3415 3 -0.0257527 50.1093000 8.5560400 21.7829000 - 3416 3 -0.0262747 52.0421000 9.7994700 20.4802000 - 3417 3 -0.0351043 53.8918000 8.6183000 21.7182000 - 3418 3 -0.0256954 48.1552000 12.2447000 20.4607000 - 3419 3 -0.0254873 50.0945000 11.0301000 21.6936000 - 3420 3 -0.0254883 52.0819000 12.2783000 20.4053000 - 3421 3 -0.0343572 53.9075000 11.0678000 21.6685000 - 3422 2 -1.6726100 60.2973000 2.9468800 18.7159000 - 3423 3 -0.0230683 50.1031000 13.4946000 21.6221000 - 3424 3 -0.0252190 51.9830000 12.3191000 25.4211000 - 3425 3 -0.0335302 53.9376000 13.5720000 21.6110000 - 3426 3 -0.0477694 55.7674000 7.3742700 23.1209000 - 3427 3 0.2171870 59.4718000 7.3568100 23.1936000 - 3428 3 -0.0461269 55.7724000 2.4287400 22.8451000 - 3429 3 0.0173050 57.7543000 1.1483400 24.1690000 - 3430 3 0.2754200 59.4593000 2.3027800 22.9868000 - 3431 3 -0.0450745 55.7802000 4.9401800 22.7979000 - 3432 3 0.0323996 57.6521000 3.6535900 23.9586000 - 3433 3 0.2844900 59.6284000 4.9359300 22.9413000 - 3434 3 0.0108902 57.7127000 6.1471700 24.3753000 - 3435 3 -0.0487694 55.6957000 7.3029000 25.5991000 - 3436 3 0.2229370 59.5512000 7.4266600 25.5685000 - 3437 3 -0.0467758 55.7639000 2.3612100 25.3261000 - 3438 3 0.0081181 57.6266000 1.1905700 26.6006000 - 3439 3 0.2948610 59.4422000 2.7117600 25.3152000 - 3440 3 -0.0461427 55.7561000 4.8903000 25.2218000 - 3441 3 0.0249775 57.5746000 3.7125900 26.5214000 - 3442 3 0.3089180 59.4346000 5.0350600 25.4602000 - 3443 3 0.0087490 57.7558000 6.1529700 26.8493000 - 3444 3 -0.0484335 55.6618000 7.2406400 27.9989000 - 3445 3 0.2539820 59.7491000 7.1594200 27.8908000 - 3446 3 -0.0479176 55.7716000 2.3639500 27.9537000 - 3447 3 0.0088430 57.6706000 1.2828900 29.3096000 - 3448 3 0.2564850 59.1302000 2.4312800 27.9220000 - 3449 3 -0.0469930 55.6927000 4.8272300 27.7013000 - 3450 3 0.0199831 57.6595000 3.8142800 29.0944000 - 3451 3 0.2831760 59.5627000 4.7148200 27.8662000 - 3452 3 0.0134571 57.9193000 6.2827700 29.2696000 - 3453 3 -0.0303155 46.1544000 1.0548000 24.3874000 - 3454 3 -0.0315625 46.1700000 3.5386100 24.2423000 - 3455 3 -0.0321265 46.3263000 6.0489700 24.2542000 - 3456 3 -0.0257736 46.2999000 0.9364840 26.8033000 - 3457 3 -0.0282760 46.3780000 3.3987300 26.6471000 - 3458 3 -0.0296868 46.3818000 5.9590000 26.7536000 - 3459 3 -0.0203337 46.3981000 0.9553290 29.3294000 - 3460 3 -0.0237399 46.4110000 3.4694300 29.2390000 - 3461 3 -0.0269622 46.4209000 5.9694100 29.2457000 - 3462 3 -0.0266238 48.1788000 7.3235300 23.0328000 - 3463 3 -0.0267214 51.9790000 7.3137700 22.9575000 - 3464 3 -0.0250055 48.0758000 2.3280900 23.0476000 - 3465 3 -0.0220199 50.0254000 1.1431600 24.2009000 - 3466 3 -0.0255579 51.9519000 2.3555600 22.9221000 - 3467 3 -0.0335855 53.8699000 1.1179100 24.0959000 - 3468 3 -0.0260977 48.1330000 4.8235500 23.0419000 - 3469 3 -0.0256053 50.0585000 3.5883000 24.1880000 - 3470 3 -0.0272587 51.9695000 4.8378100 22.9360000 - 3471 3 -0.0356654 53.8708000 3.5611700 24.1282000 - 3472 3 -0.0259036 50.0914000 6.0698700 24.2623000 - 3473 3 -0.0357321 53.8338000 6.0468100 24.1414000 - 3474 3 -0.0256089 48.2634000 7.3073600 25.4867000 - 3475 3 -0.0263450 51.9756000 7.2882800 25.4342000 - 3476 3 -0.0245687 48.1270000 2.2447100 25.4885000 - 3477 3 -0.0208599 50.0542000 1.0823400 26.6569000 - 3478 3 -0.0250388 51.9541000 2.3342700 25.3814000 - 3479 3 -0.0336618 53.9578000 1.1187100 26.6360000 - 3480 3 -0.0260757 48.2168000 4.7773900 25.4841000 - 3481 3 -0.0246587 50.0618000 3.5366500 26.6828000 - 3482 3 -0.0265746 51.9519000 4.7952900 25.4267000 - 3483 3 -0.0351594 53.8828000 3.5606100 26.5848000 - 3484 3 -0.0254430 50.0832000 6.0036400 26.7354000 - 3485 3 -0.0347978 53.7853000 6.0150000 26.6349000 - 3486 3 -0.0236560 48.2752000 7.2574600 27.9877000 - 3487 3 -0.0245594 51.9775000 7.3140900 27.9386000 - 3488 3 -0.0213464 48.2341000 2.2425800 27.9813000 - 3489 3 -0.0171888 50.1628000 1.0772000 29.1574000 - 3490 3 -0.0229304 52.0215000 2.3209600 27.8680000 - 3491 3 -0.0320145 54.0030000 1.1386800 29.1808000 - 3492 3 -0.0235409 48.2436000 4.7400400 27.9749000 - 3493 3 -0.0206775 50.1168000 3.5298700 29.2011000 - 3494 3 -0.0247432 51.9664000 4.8005100 27.9071000 - 3495 3 -0.0321102 53.8304000 3.6023900 29.1126000 - 3496 3 -0.0216679 50.1234000 6.0010900 29.2234000 - 3497 3 -0.0308352 53.7976000 6.1052300 29.1858000 - 3498 3 -0.0499709 55.7856000 9.8297100 22.9095000 - 3499 3 -0.0115555 57.6584000 8.5947400 24.1631000 - 3500 3 0.2018280 59.4931000 9.7914700 22.8999000 - 3501 3 -0.0496257 55.7944000 12.3761000 22.9463000 - 3502 3 -0.0155908 57.6530000 11.1213000 24.2514000 - 3503 3 0.2139240 59.5920000 12.1791000 23.1887000 - 3504 3 -0.0242798 52.0069000 9.8235200 27.9232000 - 3505 3 0.0074441 57.6748000 13.4898000 24.0606000 - 3506 3 -0.0216755 50.1416000 8.5263700 29.2087000 - 3507 3 -0.0501104 55.6776000 9.7323400 25.3506000 - 3508 3 -0.0094436 57.7036000 8.4988000 26.5928000 - 3509 3 0.1833720 59.4342000 9.7727200 25.3617000 - 3510 3 -0.0484171 55.7219000 12.2454000 25.3677000 - 3511 3 -0.0128492 57.5614000 10.9215000 26.7540000 - 3512 3 0.2080570 59.3697000 12.1015000 25.6143000 - 3513 3 -0.0244631 48.2650000 9.8196300 28.0087000 - 3514 3 0.0132748 57.6053000 13.4352000 26.5496000 - 3515 3 -0.0333463 53.8591000 13.5402000 26.6359000 - 3516 3 -0.0475071 55.6908000 9.7072200 27.7993000 - 3517 3 -0.0027751 57.6429000 8.5970200 28.9911000 - 3518 3 0.2134990 59.3991000 9.7338100 27.8311000 - 3519 3 -0.0459362 55.7565000 12.2903000 27.8865000 - 3520 3 0.0027290 57.5163000 11.1652000 29.2007000 - 3521 3 0.1992530 59.2690000 12.1165000 27.9561000 - 3522 3 0.0260438 57.6621000 13.6654000 28.9600000 - 3523 3 -0.0326334 46.3628000 8.5717100 24.2088000 - 3524 3 -0.0323857 46.2968000 11.0789000 24.2437000 - 3525 3 -0.0308036 46.2762000 13.5858000 24.2531000 - 3526 3 -0.0318822 46.4786000 8.4768900 26.7412000 - 3527 3 -0.0328339 46.4192000 10.9736000 26.7104000 - 3528 3 -0.0340287 53.7722000 8.5316900 26.6577000 - 3529 3 -0.0310611 53.9659000 11.1065000 29.1867000 - 3530 3 -0.0317339 46.3316000 13.4662000 26.7348000 - 3531 3 -0.0293958 46.4200000 8.5080300 29.2308000 - 3532 3 -0.0311332 46.3584000 11.0559000 29.1747000 - 3533 3 -0.0234273 52.0394000 12.2828000 27.9324000 - 3534 3 -0.0308400 46.3316000 13.4880000 29.2087000 - 3535 3 -0.0266216 48.2335000 9.8405500 23.0370000 - 3536 3 -0.0258527 50.1014000 8.6017200 24.2691000 - 3537 3 -0.0261279 51.9489000 9.7855400 22.9531000 - 3538 3 -0.0346087 53.8119000 8.5537000 24.1780000 - 3539 3 -0.0249644 48.1937000 12.2902000 22.9618000 - 3540 3 -0.0254325 50.1175000 11.0924000 24.2106000 - 3541 3 -0.0253976 52.0024000 12.2901000 22.9006000 - 3542 3 -0.0344868 53.8205000 11.0765000 24.1524000 - 3543 3 -0.0214943 50.1311000 13.4984000 26.7053000 - 3544 3 -0.0186088 48.2597000 14.7472000 15.4861000 - 3545 3 0.2862760 59.6146000 0.3026760 25.5354000 - 3546 3 -0.0226491 52.0793000 14.7274000 20.2932000 - 3547 3 -0.0224395 51.9999000 14.7800000 22.8223000 - 3548 3 -0.0431074 55.7837000 14.6986000 25.2113000 - 3549 3 -0.0215550 48.1908000 14.7610000 25.4304000 - 3550 3 -0.0206882 48.1437000 14.7150000 20.4568000 - 3551 3 -0.0199842 52.0478000 14.7500000 27.9327000 - 3552 3 -0.0205615 48.2047000 14.7236000 27.9325000 - 3553 3 -0.0206403 48.1805000 14.7274000 17.9635000 - 3554 3 -0.0477310 55.9018000 14.7181000 20.2644000 - 3555 3 0.3118800 59.7919000 14.5733000 27.6629000 - 3556 3 0.2317270 59.5422000 0.1107380 28.0830000 - 3557 3 -0.0234028 52.1895000 29.6953000 3.0664900 - 3558 3 -0.0458243 55.9490000 29.6352000 0.6069450 - 3559 3 -0.0323437 46.3309000 16.0507000 6.9012400 - 3560 3 0.3774690 60.1333000 29.5402000 8.0461500 - 3561 3 -0.0202061 48.2568000 29.6395000 13.0141000 - 3562 3 0.3487900 59.7907000 14.9047000 0.2222720 - 3563 3 -0.0158069 48.3400000 29.5844000 0.7388870 - 3564 3 -0.0213066 50.1786000 25.9500000 14.2037000 - 3565 3 -0.0243634 48.2706000 27.1413000 12.9106000 - 3566 3 -0.0364874 53.7986000 23.4280000 14.2176000 - 3567 3 -0.0261579 51.9960000 24.7399000 12.9583000 - 3568 3 -0.0218760 50.1252000 23.4197000 14.2143000 - 3569 3 -0.0250037 48.3071000 24.6686000 12.9260000 - 3570 3 -0.0315770 53.9602000 28.5352000 11.7218000 - 3571 3 -0.0226365 50.1968000 28.4724000 11.6906000 - 3572 3 -0.0337197 46.3505000 16.1277000 4.3071700 - 3573 3 -0.0372504 53.9198000 26.0318000 11.7103000 - 3574 3 -0.0265585 52.1042000 27.2750000 10.4494000 - 3575 3 -0.0251606 50.2027000 25.9887000 11.6859000 - 3576 3 -0.0268091 48.3326000 27.2668000 10.4514000 - 3577 3 -0.0390244 53.9132000 23.5891000 11.7895000 - 3578 3 -0.0282932 52.0662000 24.7859000 10.4777000 - 3579 3 -0.0254875 50.1414000 23.4244000 11.7325000 - 3580 3 -0.0274099 48.3037000 24.7478000 10.4658000 - 3581 3 -0.0360866 54.0214000 28.5114000 9.2440400 - 3582 3 -0.0242068 50.2433000 28.5096000 9.2177600 - 3583 3 -0.0395675 54.0331000 26.0562000 9.2128300 - 3584 3 -0.0274860 52.1671000 27.2541000 7.9806300 - 3585 3 -0.0258184 50.1456000 26.0281000 9.2102300 - 3586 3 -0.0265007 48.3701000 27.2485000 7.9788300 - 3587 3 -0.0408988 53.9903000 23.5674000 9.3244900 - 3588 3 -0.0283415 52.0812000 24.8046000 8.0686900 - 3589 3 -0.0258034 50.1412000 23.5253000 9.2317800 - 3590 3 -0.0272737 48.3052000 24.7494000 7.9657400 - 3591 3 -0.0285079 46.3585000 28.3659000 14.2196000 - 3592 3 -0.0236068 52.0411000 27.2354000 12.9405000 - 3593 3 -0.0182027 52.0881000 29.7062000 12.9564000 - 3594 3 -0.0315724 46.3729000 25.8963000 14.2045000 - 3595 3 -0.0324964 46.3782000 23.3821000 14.1951000 - 3596 3 -0.0303106 46.3771000 28.3976000 11.6995000 - 3597 3 -0.0322542 53.8705000 25.9442000 14.2269000 - 3598 3 -0.0333797 46.3903000 18.6367000 4.4198200 - 3599 3 -0.0334883 46.4147000 25.9180000 11.7119000 - 3600 3 -0.0344131 46.4018000 23.4723000 11.6964000 - 3601 3 -0.0293351 46.4483000 28.6006000 9.2569900 - 3602 3 -0.0327290 46.5168000 26.0647000 9.2203300 - 3603 3 -0.0338567 46.3879000 23.5449000 9.2144500 - 3604 3 -0.0209674 52.0846000 17.2555000 0.6129200 - 3605 3 -0.0030578 57.5139000 28.5439000 14.1820000 - 3606 3 -0.0226043 48.3179000 29.7429000 10.5562000 - 3607 3 0.2228170 59.3971000 27.5607000 13.0077000 - 3608 3 0.0098751 57.6093000 25.9467000 14.2344000 - 3609 3 -0.0448996 55.7094000 27.2789000 13.0623000 - 3610 3 0.3290250 59.5071000 24.9452000 13.1138000 - 3611 3 0.0606164 57.6886000 23.4662000 14.2817000 - 3612 3 -0.0421516 55.7896000 24.7173000 12.9996000 - 3613 3 -0.0070377 57.8297000 28.6595000 11.4635000 - 3614 3 0.5458020 60.3062000 20.3773000 10.3766000 - 3615 3 0.2749250 59.7265000 27.4800000 10.5358000 - 3616 3 0.0253537 57.7322000 26.2640000 11.8822000 - 3617 3 -0.0488725 55.8189000 27.3244000 10.6044000 - 3618 3 0.3296510 59.6165000 25.1283000 10.8600000 - 3619 3 0.0978978 57.8751000 23.7651000 11.8622000 - 3620 3 -0.0418130 55.8506000 24.8793000 10.5647000 - 3621 3 0.0261757 58.0598000 28.4336000 8.9968300 - 3622 3 0.3633740 59.6899000 26.8719000 8.1286100 - 3623 3 0.0437554 57.7912000 26.0429000 9.5221000 - 3624 3 -0.0463358 55.9576000 27.3251000 8.1304100 - 3625 3 0.3309110 58.9489000 24.6736000 7.9619200 - 3626 3 0.0877145 57.8143000 23.4890000 9.5239900 - 3627 3 -0.0441304 56.0275000 24.8278000 8.0722700 - 3628 3 -0.0367835 53.9326000 21.0059000 14.4081000 - 3629 3 -0.0276873 51.9732000 22.2188000 13.0012000 - 3630 3 -0.0219274 50.0828000 20.8914000 14.2542000 - 3631 3 -0.0244655 48.2662000 22.0923000 12.9218000 - 3632 3 -0.0365138 53.9822000 18.5148000 14.3068000 - 3633 3 -0.0278787 52.0923000 19.7407000 13.1037000 - 3634 3 -0.0210691 50.2001000 18.4322000 14.3073000 - 3635 3 -0.0244842 48.2189000 19.6322000 13.0438000 - 3636 3 -0.0373472 54.0360000 16.0143000 13.9487000 - 3637 3 -0.0262573 52.0798000 17.2916000 13.0194000 - 3638 3 -0.0174222 50.2564000 15.9859000 14.2510000 - 3639 3 -0.0225444 48.2964000 17.2016000 13.0397000 - 3640 3 -0.0354970 53.9066000 21.0844000 11.9228000 - 3641 3 -0.0294668 52.0242000 22.2926000 10.5430000 - 3642 3 -0.0256615 50.1467000 20.9583000 11.7859000 - 3643 3 -0.0260330 48.2491000 22.2736000 10.4907000 - 3644 3 -0.0338558 53.9786000 18.4975000 11.8496000 - 3645 3 -0.0292598 52.0501000 19.7766000 10.6203000 - 3646 3 -0.0250410 50.1706000 18.5014000 11.8192000 - 3647 3 -0.0257686 48.2430000 19.8187000 10.6017000 - 3648 3 -0.0385351 54.0730000 16.0389000 11.5008000 - 3649 3 -0.0275468 52.0388000 17.3224000 10.5461000 - 3650 3 -0.0212060 50.1905000 16.0527000 11.7350000 - 3651 3 -0.0245271 48.2908000 17.2963000 10.5611000 - 3652 3 -0.0387535 53.9172000 21.0397000 9.4491500 - 3653 3 -0.0286783 52.0742000 22.3102000 8.1164700 - 3654 3 -0.0260363 50.1416000 21.0340000 9.3405500 - 3655 3 -0.0259422 48.2788000 22.2493000 7.9938400 - 3656 3 -0.0375932 53.9832000 18.5169000 9.4196800 - 3657 3 -0.0284603 52.1041000 19.8071000 8.1347700 - 3658 3 -0.0256351 50.1405000 18.5636000 9.3122400 - 3659 3 -0.0256446 48.2710000 19.7891000 8.0400000 - 3660 3 -0.0389649 54.1738000 16.0299000 9.0361600 - 3661 3 -0.0264978 52.0953000 17.3560000 8.0712200 - 3662 3 -0.0224076 50.1957000 16.0553000 9.2616800 - 3663 3 -0.0247327 48.2362000 17.3063000 8.0844500 - 3664 3 -0.0315419 46.3172000 20.9115000 14.2289000 - 3665 3 -0.0309815 46.3032000 18.3626000 14.1970000 - 3666 3 -0.0303578 46.2766000 15.9570000 14.2956000 - 3667 3 -0.0340081 46.2800000 20.9964000 11.7405000 - 3668 3 -0.0338577 46.3075000 18.4975000 11.7092000 - 3669 3 -0.0321733 46.3065000 16.0630000 11.9109000 - 3670 3 -0.0335232 46.4061000 21.0377000 9.2460500 - 3671 3 -0.0335557 46.3404000 18.5970000 9.2691500 - 3672 3 -0.0319635 46.3009000 16.1648000 9.4485100 - 3673 3 0.4346480 59.6058000 22.0993000 13.0365000 - 3674 3 0.0826321 57.5631000 20.9530000 14.3052000 - 3675 3 -0.0286703 55.7420000 22.2623000 13.1703000 - 3676 3 -0.0317577 46.4081000 20.9852000 6.6844300 - 3677 3 0.1263390 57.9031000 18.4617000 14.0171000 - 3678 3 -0.0018989 55.8659000 19.6961000 13.1391000 - 3679 3 0.4913150 60.0612000 17.9913000 12.5708000 - 3680 3 0.0792308 57.8789000 15.9546000 13.9150000 - 3681 3 -0.0179609 55.9053000 17.2615000 12.8325000 - 3682 3 0.4836270 60.0091000 22.7816000 10.7299000 - 3683 3 0.1848000 57.5974000 21.2985000 11.8837000 - 3684 3 -0.0255370 55.8246000 22.4254000 10.7010000 - 3685 3 -0.0323882 46.3505000 18.5481000 6.7916000 - 3686 3 0.2564020 57.4813000 18.5806000 11.6320000 - 3687 3 0.0004294 55.8101000 19.9611000 10.5909000 - 3688 3 0.4242840 60.0717000 17.1381000 10.3488000 - 3689 3 0.0864075 58.1075000 16.2939000 11.6043000 - 3690 3 -0.0211281 55.9416000 17.2667000 10.3833000 - 3691 3 0.3824280 59.6295000 22.4177000 8.4515500 - 3692 3 0.0998906 58.0396000 21.0095000 9.5623200 - 3693 3 -0.0430977 55.7713000 22.2193000 8.2514100 - 3694 3 0.4143590 59.7392000 20.1385000 8.0062900 - 3695 3 0.1243730 58.4209000 18.5559000 9.3106700 - 3696 3 -0.0420701 55.8204000 19.6667000 8.2208500 - 3697 3 0.2881740 59.8185000 16.8997000 7.7299600 - 3698 3 0.0393450 58.0482000 15.9951000 9.0652900 - 3699 3 -0.0511215 56.1465000 17.2298000 7.9856200 - 3700 3 -0.0376298 54.0532000 28.5117000 6.7523600 - 3701 3 -0.0330267 46.4127000 20.9515000 1.8576500 - 3702 3 -0.0240132 50.3057000 28.4946000 6.7907100 - 3703 3 -0.0221404 48.3198000 17.2110000 0.6613800 - 3704 3 -0.0400957 54.0378000 26.0643000 6.7996800 - 3705 3 -0.0276551 52.1807000 27.2500000 5.5410800 - 3706 3 -0.0257932 50.1961000 26.0034000 6.7429400 - 3707 3 -0.0251729 48.3850000 27.2526000 5.4075000 - 3708 3 -0.0402435 53.9380000 23.5484000 6.8399000 - 3709 3 -0.0278698 52.1077000 24.7636000 5.5915900 - 3710 3 -0.0257855 50.1955000 23.5380000 6.7866900 - 3711 3 -0.0264611 48.3324000 24.7475000 5.4551600 - 3712 3 -0.0377519 54.0520000 28.4390000 4.2850800 - 3713 3 -0.0235142 50.2909000 28.4932000 4.3322400 - 3714 2 -1.6145300 59.0669000 19.7660000 12.1507000 - 3715 3 -0.0395213 54.0156000 25.9640000 4.2999800 - 3716 3 -0.0268199 52.1903000 27.2083000 3.0402000 - 3717 3 -0.0257182 50.2812000 26.0097000 4.2455500 - 3718 3 -0.0239787 48.3698000 27.1926000 2.9283400 - 3719 3 -0.0391061 53.9069000 23.4523000 4.3048800 - 3720 3 -0.0265995 52.0901000 24.7775000 3.0371700 - 3721 3 -0.0257630 50.2694000 23.5146000 4.3286200 - 3722 3 -0.0258283 48.3808000 24.7306000 2.9853500 - 3723 3 -0.0359027 54.0211000 28.4330000 1.7911700 - 3724 3 0.3849520 60.2447000 15.5657000 12.4378000 - 3725 3 -0.0215727 50.2895000 28.4278000 1.8171600 - 3726 3 -0.0364490 54.0088000 25.9199000 1.8011800 - 3727 3 -0.0227806 52.1288000 27.2033000 0.5335730 - 3728 3 -0.0238277 50.2148000 25.9518000 1.7271200 - 3729 3 -0.0199475 48.2278000 27.1464000 0.4569770 - 3730 3 -0.0357845 53.9600000 23.3995000 1.7611500 - 3731 3 -0.0230321 52.1098000 24.7252000 0.5368630 - 3732 3 -0.0239657 50.1909000 23.4708000 1.7707800 - 3733 3 -0.0224722 48.2861000 24.6719000 0.5145120 - 3734 3 -0.0294080 46.5086000 28.5347000 6.7811600 - 3735 3 -0.0183042 50.1149000 28.4318000 14.1522000 - 3736 3 -0.0190819 48.3445000 29.6310000 3.2024600 - 3737 3 -0.0321248 46.5009000 26.1363000 6.6175000 - 3738 3 -0.0327155 46.4060000 23.5428000 6.7564100 - 3739 3 -0.0275216 46.4364000 28.4299000 4.3067200 - 3740 3 -0.0324816 46.4557000 21.0538000 4.2738200 - 3741 3 -0.0316289 46.4728000 26.0577000 4.1073700 - 3742 3 -0.0323817 46.4844000 23.5373000 4.2441600 - 3743 3 -0.0243707 46.3664000 28.3287000 1.7767900 - 3744 3 -0.0256910 53.8282000 28.4364000 14.2462000 - 3745 3 -0.0294192 46.4048000 25.9136000 1.7138400 - 3746 3 -0.0320568 46.4709000 23.4424000 1.7994800 - 3747 3 0.1007220 58.6610000 28.2924000 6.5290600 - 3748 3 -0.0352147 46.4079000 18.4199000 1.9917400 - 3749 3 -0.0343095 46.3908000 16.0105000 1.9061300 - 3750 3 0.0503580 57.6821000 26.0821000 6.7794500 - 3751 3 -0.0417354 55.9547000 27.3878000 5.6004800 - 3752 3 0.3290490 60.0303000 26.5466000 5.5225800 - 3753 3 0.0451359 57.4394000 23.3437000 6.8485200 - 3754 3 -0.0458314 55.9862000 17.3015000 0.5604120 - 3755 3 0.0245836 57.9753000 16.0789000 1.5900800 - 3756 3 0.2712350 59.9339000 17.3566000 0.4487810 - 3757 3 -0.0441735 55.9140000 19.7200000 0.8954660 - 3758 3 0.0474552 58.0773000 18.6156000 1.9198500 - 3759 3 0.2887720 59.9313000 19.8155000 0.6192220 - 3760 3 -0.0450304 55.7797000 22.0722000 0.5765430 - 3761 3 0.0139472 57.7662000 20.9505000 2.1312000 - 3762 3 0.2333300 60.0761000 21.8195000 1.5565100 - 3763 3 -0.0461652 56.0691000 17.2027000 3.0077300 - 3764 3 0.0457993 58.0970000 16.1745000 4.0549600 - 3765 3 0.3641620 59.8963000 17.2961000 2.9467600 - 3766 3 -0.0462414 55.8916000 19.6252000 3.3009300 - 3767 3 0.0358922 58.0737000 18.5296000 4.4168900 - 3768 3 0.3303930 59.6798000 19.9831000 3.2226800 - 3769 3 -0.0531555 55.7543000 22.0944000 3.1164700 - 3770 3 0.0113054 57.6102000 21.0245000 4.5797200 - 3771 3 0.2118930 59.2967000 22.3843000 3.6612700 - 3772 3 -0.0535371 56.1110000 17.2581000 5.5206000 - 3773 3 0.0216434 58.1555000 15.9739000 6.4442000 - 3774 3 0.3915390 60.1354000 17.5534000 5.3290900 - 3775 3 -0.0535095 55.8590000 19.6067000 5.7793300 - 3776 3 0.0190552 58.1023000 18.3764000 6.8800300 - 3777 3 0.2805330 59.6035000 19.8428000 5.6653900 - 3778 3 -0.0539936 55.6962000 22.2560000 5.6312300 - 3779 3 0.0350803 57.6057000 20.7893000 7.0959200 - 3780 3 0.2620160 59.1971000 22.0318000 6.0330000 - 3781 3 -0.0511156 55.7952000 24.8407000 5.5622900 - 3782 3 0.1148230 57.9133000 28.6544000 4.2450300 - 3783 3 -0.0208884 48.4252000 29.7048000 5.6296100 - 3784 3 0.0385919 57.7327000 26.1647000 4.2685400 - 3785 3 -0.0442816 55.8167000 27.1645000 3.1159700 - 3786 3 0.2688120 59.5249000 24.7210000 2.8954500 - 3787 3 -0.0027707 57.6389000 23.7360000 4.5270300 - 3788 3 -0.0533101 55.8076000 24.6420000 3.1107900 - 3789 3 0.0400112 57.6066000 28.3662000 1.8919200 - 3790 3 -0.0206130 50.2288000 16.0059000 1.8637000 - 3791 3 0.2564240 58.8444000 26.9285000 0.6405520 - 3792 3 0.0094224 57.5021000 25.6652000 1.9218900 - 3793 3 -0.0462286 55.9114000 27.1020000 0.6110910 - 3794 2 -1.7026100 59.9123000 24.0330000 0.0289520 - 3795 3 -0.0073000 57.6755000 23.3023000 1.9501700 - 3796 3 -0.0434117 55.8114000 24.5710000 0.5465840 - 3797 3 -0.0410068 53.9631000 20.9844000 6.9011200 - 3798 3 -0.0276150 52.1131000 22.2400000 5.6316900 - 3799 3 -0.0255618 50.2181000 21.0164000 6.8699800 - 3800 3 -0.0257113 48.3475000 22.2542000 5.5148600 - 3801 3 -0.0405711 53.9992000 18.4949000 6.9562900 - 3802 3 -0.0274803 52.1278000 19.7270000 5.6188000 - 3803 3 -0.0252374 50.2115000 18.5359000 6.8308100 - 3804 3 -0.0254181 48.3303000 19.7859000 5.5796200 - 3805 3 -0.0377816 54.2095000 16.0274000 6.5658900 - 3806 3 -0.0257926 52.1303000 17.2393000 5.6167100 - 3807 3 -0.0227705 50.1688000 16.0911000 6.8075200 - 3808 3 -0.0249896 48.2706000 17.3048000 5.6050600 - 3809 3 -0.0386389 53.9491000 20.9718000 4.3763900 - 3810 3 -0.0266558 52.0636000 22.2436000 3.0792200 - 3811 3 -0.0255370 50.2702000 20.9616000 4.3558600 - 3812 3 -0.0261779 48.3874000 22.2645000 3.0501300 - 3813 3 -0.0376723 54.0236000 18.4588000 4.4353700 - 3814 3 -0.0263065 52.1271000 19.6943000 3.0749200 - 3815 3 -0.0251433 50.2652000 18.4724000 4.3604700 - 3816 3 -0.0259827 48.3140000 19.7674000 3.1332700 - 3817 3 -0.0345319 54.0740000 16.0243000 4.1289900 - 3818 3 -0.0247554 52.1138000 17.2323000 3.1240300 - 3819 3 -0.0226227 50.2201000 16.0127000 4.3440900 - 3820 3 -0.0248862 48.3295000 17.2587000 3.1431400 - 3821 3 -0.0348057 53.9297000 20.9420000 1.8583400 - 3822 3 -0.0227381 51.9910000 22.2286000 0.5653540 - 3823 3 -0.0237541 50.1949000 20.9765000 1.8393200 - 3824 3 -0.0233782 48.2990000 22.1892000 0.5546070 - 3825 3 -0.0347406 53.9978000 18.4207000 1.9435800 - 3826 3 -0.0227000 52.0507000 19.7124000 0.6220320 - 3827 3 -0.0234759 50.1970000 18.4871000 1.8785900 - 3828 3 -0.0236788 48.2588000 19.6886000 0.6805470 - 3829 3 -0.0323290 53.9693000 16.0379000 1.7059600 - 3830 3 -0.0224785 48.4199000 29.7407000 8.0838900 - 3831 3 -0.0195645 52.1160000 29.6594000 0.5753920 - 3832 3 -0.0213620 52.1306000 29.7472000 10.4912000 - 3833 3 -0.0235664 52.2092000 29.7552000 8.0321100 - 3834 3 0.3024530 59.7383000 14.9581000 2.6571700 - 3835 3 -0.0240924 52.1821000 29.7211000 5.5306900 - 3836 3 -0.0229319 52.1714000 14.9376000 7.9158300 - 3837 3 0.3748700 59.8798000 14.9219000 5.1573300 - 3838 3 -0.0370823 55.9878000 29.7461000 2.9965000 - 3839 3 -0.0171391 52.0287000 29.6646000 15.4534000 - 3840 3 -0.0469170 55.7113000 29.6722000 20.3655000 - 3841 3 -0.0214437 48.2482000 29.6492000 17.9706000 - 3842 3 0.2408430 59.6423000 29.6061000 22.8403000 - 3843 3 -0.0211195 51.9972000 29.6143000 17.9711000 - 3844 3 -0.0332844 46.3141000 23.3874000 16.7025000 - 3845 3 -0.0315056 46.2854000 25.8497000 16.6770000 - 3846 3 -0.0204971 52.0376000 29.6555000 27.8408000 - 3847 3 -0.0180098 48.2463000 29.5716000 15.5002000 - 3848 3 -0.0212006 50.1217000 25.9052000 29.0213000 - 3849 3 -0.0331222 53.9770000 28.3986000 29.0795000 - 3850 3 -0.0290528 46.2812000 28.3525000 16.6570000 - 3851 3 -0.0226747 48.1521000 27.1434000 27.7416000 - 3852 3 -0.0325365 53.9365000 23.3956000 29.0266000 - 3853 3 -0.0335954 46.2432000 23.4838000 19.1585000 - 3854 3 -0.0255831 52.0373000 24.6916000 27.8277000 - 3855 3 -0.0218406 50.1486000 23.4297000 29.0403000 - 3856 3 -0.0249214 48.2093000 24.6177000 27.8048000 - 3857 3 -0.0314649 46.2538000 26.0369000 19.2223000 - 3858 3 -0.0355584 53.9442000 28.4345000 26.6198000 - 3859 3 -0.0293628 46.2120000 28.4819000 19.1120000 - 3860 3 -0.0231419 50.1345000 28.3584000 26.5984000 - 3861 3 -0.0339462 46.2779000 23.4252000 21.5963000 - 3862 3 -0.0369436 53.9080000 25.9365000 26.5848000 - 3863 3 -0.0268484 51.9947000 27.1748000 25.3153000 - 3864 3 -0.0331876 46.1654000 25.8395000 21.7145000 - 3865 3 0.0148941 57.7821000 28.4635000 21.7072000 - 3866 3 -0.0462800 55.8056000 29.6144000 25.2509000 - 3867 3 -0.0251564 50.1149000 25.8726000 26.5546000 - 3868 3 -0.0257143 48.1833000 27.1316000 25.3100000 - 3869 3 -0.0371387 53.8968000 23.3804000 26.5202000 - 3870 3 0.2442820 59.5303000 27.6787000 20.2016000 - 3871 3 0.0703033 57.8811000 26.0056000 21.4244000 - 3872 3 -0.0458676 55.7539000 27.1294000 20.5389000 - 3873 3 0.3694260 59.7085000 24.4332000 20.0928000 - 3874 3 0.0482902 57.7752000 23.4391000 21.5653000 - 3875 3 -0.0462432 55.8755000 24.6988000 20.3287000 - 3876 3 -0.0275629 51.9983000 24.6610000 25.3453000 - 3877 3 -0.0251992 50.1394000 23.4249000 26.5526000 - 3878 3 -0.0426741 55.6667000 29.5695000 15.3573000 - 3879 3 -0.0216588 57.5010000 28.3578000 19.2135000 - 3880 3 -0.0318473 46.3232000 16.1245000 24.3218000 - 3881 3 0.1757470 59.7394000 26.6431000 17.7165000 - 3882 3 -0.0264269 48.2107000 24.6637000 25.3202000 - 3883 3 -0.0355532 53.9026000 28.4026000 24.0839000 - 3884 3 0.0277764 57.7704000 25.8642000 19.0386000 - 3885 3 -0.0521244 55.7180000 26.9795000 18.0243000 - 3886 3 0.2540530 59.9107000 24.3080000 17.7962000 - 3887 3 0.0246518 57.7383000 23.3902000 19.0702000 - 3888 3 -0.0243225 50.0400000 28.4406000 24.1242000 - 3889 3 -0.0523154 55.8311000 24.5114000 17.8589000 - 3890 3 -0.0234389 48.0699000 29.6401000 22.9721000 - 3891 3 -0.0358663 57.5282000 28.1838000 16.8055000 - 3892 3 -0.0225378 51.9773000 29.6277000 20.4569000 - 3893 3 0.1478470 59.1129000 27.2044000 15.4147000 - 3894 3 -0.0171828 57.7534000 25.6175000 16.7076000 - 3895 3 -0.0379503 53.8918000 25.9701000 24.1024000 - 3896 3 -0.0268744 51.9845000 27.1883000 22.8844000 - 3897 3 -0.0260642 50.0376000 25.9302000 24.0632000 - 3898 3 -0.0464096 55.6978000 27.1597000 15.5853000 - 3899 3 0.2489000 59.5639000 24.2794000 15.4727000 - 3900 3 0.0100249 57.8109000 23.1203000 16.6879000 - 3901 3 -0.0472573 55.7236000 24.5953000 15.4295000 - 3902 3 -0.0384843 53.9704000 20.9488000 21.8872000 - 3903 3 -0.0269516 52.0476000 22.1999000 20.4699000 - 3904 3 -0.0268509 48.0792000 27.1867000 22.8257000 - 3905 3 -0.0257188 50.1492000 20.9804000 21.6319000 - 3906 3 -0.0259893 48.1780000 22.1995000 20.3391000 - 3907 3 -0.0380944 53.9268000 23.5232000 23.9634000 - 3908 3 -0.0274676 51.9943000 24.6680000 22.8602000 - 3909 3 -0.0259233 50.1061000 23.4217000 24.0568000 - 3910 3 -0.0370358 53.8703000 18.4788000 21.5694000 - 3911 3 -0.0269565 51.9769000 19.7190000 20.4325000 - 3912 3 -0.0251249 50.0806000 18.4997000 21.6916000 - 3913 3 -0.0255699 48.1957000 19.7618000 20.3881000 - 3914 3 -0.0329284 53.9034000 15.9890000 21.5152000 - 3915 3 -0.0252670 52.0001000 17.2157000 20.3962000 - 3916 3 -0.0264171 48.1626000 24.6838000 22.8259000 - 3917 3 -0.0220106 50.0774000 15.9774000 21.6843000 - 3918 3 -0.0236335 48.1809000 17.2289000 20.4208000 - 3919 3 -0.0223826 46.2301000 28.3224000 29.0016000 - 3920 3 -0.0385181 53.9338000 20.9075000 19.4225000 - 3921 3 -0.0265658 52.0351000 22.0864000 18.0010000 - 3922 3 -0.0247776 52.0832000 27.1446000 27.8212000 - 3923 2 -1.5975800 60.3167000 26.9170000 24.2787000 - 3924 3 -0.0257496 50.1117000 20.9357000 19.1665000 - 3925 3 -0.0254118 48.1558000 22.1213000 17.8640000 - 3926 3 -0.0380542 53.9363000 18.4465000 19.1436000 - 3927 3 -0.0266151 51.9919000 19.6316000 17.9929000 - 3928 3 -0.0248764 50.0794000 18.4871000 19.2340000 - 3929 3 -0.0249761 48.2009000 19.6536000 17.9227000 - 3930 3 -0.0279241 46.3700000 25.8703000 29.0017000 - 3931 3 -0.0342364 53.9919000 15.9494000 18.9716000 - 3932 3 -0.0248102 52.0498000 17.1297000 17.9733000 - 3933 3 -0.0210394 50.0679000 16.0078000 19.2335000 - 3934 3 -0.0230334 48.2028000 17.1910000 17.9317000 - 3935 3 -0.0390258 54.0210000 20.8349000 16.9530000 - 3936 3 -0.0238572 51.9693000 22.1477000 15.4881000 - 3937 3 -0.0239016 50.1472000 20.9111000 16.7142000 - 3938 3 -0.0225286 48.2348000 22.1511000 15.4038000 - 3939 3 -0.0300931 46.3816000 23.4081000 29.0859000 - 3940 3 -0.0385712 53.9997000 18.3350000 16.7659000 - 3941 3 -0.0242068 52.0438000 19.6665000 15.5493000 - 3942 3 -0.0228323 50.1493000 18.4309000 16.7784000 - 3943 3 -0.0216818 48.2096000 19.6474000 15.5016000 - 3944 3 -0.0351837 54.0709000 15.8563000 16.4398000 - 3945 3 -0.0278381 46.1429000 28.2650000 26.4334000 - 3946 3 -0.0223849 52.1143000 17.2276000 15.5258000 - 3947 3 -0.0186461 50.1619000 15.9951000 16.7512000 - 3948 3 -0.0196493 48.2377000 17.1693000 15.4780000 - 3949 3 -0.0326167 46.2607000 21.0039000 21.6345000 - 3950 3 -0.0331814 54.0472000 25.8910000 29.0810000 - 3951 3 -0.0319799 46.3551000 18.4877000 21.6413000 - 3952 3 -0.0319910 46.3192000 25.7706000 26.5229000 - 3953 3 -0.0299750 46.3219000 16.0487000 21.6901000 - 3954 3 -0.0332920 46.2317000 20.9432000 19.1788000 - 3955 3 -0.0317942 46.2701000 18.4805000 19.1180000 - 3956 3 -0.0298937 46.3329000 15.9873000 19.1754000 - 3957 3 -0.0332819 46.3240000 23.2984000 26.5859000 - 3958 3 -0.0331621 46.2531000 20.9017000 16.7014000 - 3959 3 -0.0321510 46.2605000 18.4117000 16.6644000 - 3960 3 -0.0316966 46.1033000 28.3771000 24.0300000 - 3961 3 -0.0302020 46.2923000 15.9537000 16.7398000 - 3962 3 -0.0187301 50.2141000 28.3641000 29.0858000 - 3963 3 -0.0310982 46.1551000 28.3118000 21.5422000 - 3964 3 -0.0339300 46.2208000 25.9550000 24.1512000 - 3965 3 -0.0226876 48.2663000 24.6585000 15.4139000 - 3966 3 -0.0240988 50.1111000 23.3771000 16.7046000 - 3967 3 -0.0225342 51.9698000 24.6766000 15.4793000 - 3968 3 -0.0365833 53.8361000 23.3254000 16.6698000 - 3969 3 -0.0218064 48.1907000 27.0894000 15.4092000 - 3970 3 -0.0239032 50.0903000 25.9042000 16.7358000 - 3971 3 -0.0209404 51.9866000 27.1683000 15.4338000 - 3972 3 -0.0333639 53.8329000 25.8486000 16.7334000 - 3973 3 -0.0340933 46.2387000 23.6076000 24.0881000 - 3974 3 -0.0217771 50.1126000 28.3907000 16.6584000 - 3975 3 -0.0299150 53.8064000 28.3380000 16.7605000 - 3976 3 -0.0261626 48.1873000 24.6403000 17.9042000 - 3977 3 -0.0257394 50.0955000 23.3863000 19.1687000 - 3978 3 -0.0260858 52.0190000 24.6049000 17.9805000 - 3979 3 -0.0381062 53.9770000 23.2967000 19.0808000 - 3980 3 -0.0254630 48.1918000 27.1586000 17.8818000 - 3981 3 -0.0258696 50.0917000 25.8896000 19.2061000 - 3982 3 -0.0248699 51.9626000 27.1055000 17.9709000 - 3983 3 -0.0360550 53.8677000 25.7921000 19.2831000 - 3984 3 -0.0241348 50.1020000 28.3708000 19.2123000 - 3985 3 -0.0338841 46.3934000 18.6219000 24.2575000 - 3986 3 -0.0330986 53.8744000 28.3559000 19.2013000 - 3987 3 0.3580560 59.5191000 22.2151000 20.2591000 - 3988 3 0.0293805 57.8800000 20.8918000 21.7693000 - 3989 3 -0.0509579 55.9215000 22.0617000 20.5549000 - 3990 3 0.3195770 59.7849000 19.2434000 20.5688000 - 3991 3 0.0199569 57.7344000 18.4280000 21.6461000 - 3992 3 -0.0514239 55.8967000 19.6252000 20.4749000 - 3993 3 -0.0230472 51.9963000 29.6650000 22.8896000 - 3994 3 0.0304181 57.9069000 16.0059000 21.4439000 - 3995 3 -0.0469478 55.8730000 17.1392000 20.3338000 - 3996 3 0.2978610 59.6609000 22.0638000 17.8381000 - 3997 3 0.0194643 57.9308000 20.7149000 19.3497000 - 3998 3 -0.0546279 55.9363000 21.9124000 18.1592000 - 3999 3 0.3260800 59.9009000 19.6445000 18.3187000 - 4000 3 0.0308091 57.8648000 18.3379000 19.1748000 - 4001 3 -0.0535062 55.9507000 19.4861000 18.0754000 - 4002 3 0.3431900 59.8448000 17.2277000 18.4413000 - 4003 3 0.0238314 57.7200000 15.9634000 19.0152000 - 4004 3 -0.0486321 55.8889000 17.1180000 17.7443000 - 4005 3 0.3542640 59.5521000 21.8182000 15.3731000 - 4006 3 0.0292323 57.9679000 20.6393000 16.8761000 - 4007 3 -0.0473496 55.7822000 22.1184000 15.6695000 - 4008 3 0.3269410 60.2392000 19.3908000 16.0754000 - 4009 3 0.0440832 57.9204000 18.2439000 16.7205000 - 4010 3 -0.0417709 55.9699000 19.5813000 15.6223000 - 4011 3 0.4143400 59.3112000 16.9839000 15.2379000 - 4012 3 0.0548697 57.6904000 15.7837000 16.4622000 - 4013 3 -0.0409691 55.9071000 17.2156000 15.2562000 - 4014 3 -0.0269305 48.1455000 24.6821000 20.3683000 - 4015 3 -0.0257371 50.0875000 23.4473000 21.6128000 - 4016 3 -0.0273658 52.0023000 24.6429000 20.4553000 - 4017 3 -0.0385283 54.0089000 23.4160000 21.5022000 - 4018 3 -0.0264990 48.1359000 27.1569000 20.3649000 - 4019 3 -0.0260054 50.0635000 25.9173000 21.6291000 - 4020 3 -0.0263781 51.9974000 27.1604000 20.4589000 - 4021 3 -0.0368282 53.9106000 25.8862000 21.7275000 - 4022 3 -0.0243368 50.0556000 28.4151000 21.6833000 - 4023 3 -0.0349361 53.9161000 28.3988000 21.6463000 - 4024 3 -0.0470080 55.7412000 17.3575000 22.7947000 - 4025 3 0.0300810 57.6187000 16.0642000 23.8827000 - 4026 3 0.3027070 59.5347000 17.3106000 23.1474000 - 4027 3 -0.0517202 55.9022000 19.8407000 22.9201000 - 4028 3 0.0118903 57.6859000 18.6534000 24.1177000 - 4029 3 0.2615240 59.6179000 19.7737000 23.0127000 - 4030 3 -0.0482584 55.9804000 22.3168000 23.0112000 - 4031 3 0.0183001 57.9236000 21.1096000 24.3550000 - 4032 3 0.3314690 59.4384000 22.2908000 22.9654000 - 4033 3 -0.0469513 55.8127000 17.2586000 25.3227000 - 4034 3 0.0307594 57.6735000 15.8425000 26.5572000 - 4035 3 0.2912650 59.2131000 17.2683000 25.3590000 - 4036 3 -0.0506005 55.8523000 19.7653000 25.4562000 - 4037 3 0.0197204 57.7118000 18.4508000 26.7162000 - 4038 3 0.2570010 59.4459000 19.5852000 25.4723000 - 4039 3 -0.0340504 46.3437000 21.1250000 24.1815000 - 4040 3 0.0000895 57.6659000 28.5633000 29.1218000 - 4041 3 -0.0184412 48.2542000 29.5195000 27.9972000 - 4042 3 0.2382200 59.4593000 27.2316000 28.1814000 - 4043 3 0.0328832 57.5339000 25.7443000 28.9031000 - 4044 3 -0.0471391 55.8298000 27.1852000 27.8793000 - 4045 3 0.3048900 59.7130000 24.5673000 27.7712000 - 4046 3 0.0494212 57.5842000 23.3178000 29.0619000 - 4047 3 -0.0410571 55.7257000 24.6667000 27.5986000 - 4048 3 0.1394180 59.4517000 29.1069000 18.0240000 - 4049 3 0.0126334 57.6429000 28.5643000 26.7260000 - 4050 3 0.3282380 59.4266000 27.4672000 25.9018000 - 4051 3 0.0459713 57.7445000 25.9264000 26.5065000 - 4052 3 -0.0422294 55.8121000 27.1602000 25.5045000 - 4053 3 -0.0218907 48.1149000 29.5064000 25.5014000 - 4054 3 0.0475012 57.7170000 23.4161000 26.6412000 - 4055 3 -0.0413539 55.8754000 24.7573000 25.1495000 - 4056 3 0.0393502 57.9379000 28.4682000 24.1980000 - 4057 3 -0.0498623 55.6496000 29.4713000 17.8844000 - 4058 3 0.3579620 59.7486000 27.3034000 22.3654000 - 4059 3 0.0846827 57.9475000 26.0634000 23.8904000 - 4060 3 -0.0416706 55.9226000 27.2330000 23.0418000 - 4061 3 0.4144010 59.6836000 24.6686000 22.3777000 - 4062 3 0.0579729 57.9520000 23.6836000 24.1591000 - 4063 3 -0.0415801 55.9401000 24.7743000 22.7631000 - 4064 3 -0.0470102 55.8747000 22.2870000 25.3977000 - 4065 3 -0.0325900 53.9197000 20.9544000 29.1451000 - 4066 3 -0.0252713 51.9705000 22.2014000 27.8254000 - 4067 3 -0.0215203 50.1203000 20.9263000 29.1051000 - 4068 3 0.0163123 57.8054000 20.8790000 26.9811000 - 4069 3 0.3239580 59.4675000 22.0466000 25.7171000 - 4070 3 -0.0466434 55.8531000 17.1907000 27.8798000 - 4071 3 0.0223390 57.8382000 16.1062000 28.9463000 - 4072 3 0.2951960 59.6289000 17.3052000 27.7912000 - 4073 3 -0.0254288 48.2436000 22.1657000 27.8208000 - 4074 3 -0.0487161 55.9024000 19.6916000 28.1326000 - 4075 3 0.0165942 58.0060000 18.5228000 29.2175000 - 4076 3 -0.0233931 48.1496000 29.6441000 20.4484000 - 4077 3 0.2655730 59.6866000 19.7214000 27.9764000 - 4078 3 -0.0451442 55.7994000 22.1965000 27.8437000 - 4079 3 0.0176820 57.9561000 20.9815000 29.4661000 - 4080 3 0.3136480 59.5519000 22.1595000 28.1209000 - 4081 3 -0.0327849 53.9837000 18.3965000 29.2188000 - 4082 3 -0.0251809 52.0368000 19.6849000 27.8981000 - 4083 3 -0.0211174 50.1153000 18.4564000 29.1392000 - 4084 3 -0.0257119 48.2146000 19.6724000 27.8909000 - 4085 3 -0.0308822 53.9732000 15.9902000 29.0494000 - 4086 3 -0.0237809 52.0726000 17.2004000 27.8785000 - 4087 3 -0.0190504 50.1298000 16.0088000 29.1338000 - 4088 3 -0.0242244 48.2102000 17.1858000 27.9164000 - 4089 3 -0.0371550 53.9334000 20.9066000 26.7264000 - 4090 3 -0.0273229 51.9874000 22.1764000 25.3235000 - 4091 3 -0.0251048 50.1075000 20.9130000 26.6023000 - 4092 3 -0.0264908 48.2493000 22.1835000 25.3443000 - 4093 3 -0.0365670 53.9622000 18.4583000 26.6562000 - 4094 3 -0.0265758 52.0318000 19.6912000 25.4020000 - 4095 3 -0.0250548 50.1330000 18.4146000 26.6852000 - 4096 3 -0.0264955 48.2823000 19.7044000 25.4177000 - 4097 3 -0.0333169 53.9648000 16.0243000 26.5592000 - 4098 3 -0.0255533 52.0420000 17.2120000 25.3658000 - 4099 3 -0.0227575 50.1345000 15.9474000 26.6791000 - 4100 3 -0.0252664 48.2595000 17.2185000 25.4445000 - 4101 3 -0.0378319 53.9663000 21.0358000 24.2893000 - 4102 3 -0.0270289 52.0423000 22.1827000 22.8960000 - 4103 3 -0.0258445 50.1433000 20.9454000 24.1097000 - 4104 3 -0.0259965 48.2046000 22.2165000 22.8162000 - 4105 3 -0.0367784 53.9188000 18.5513000 24.0845000 - 4106 3 -0.0268339 52.0156000 19.7186000 22.8964000 - 4107 3 -0.0256674 50.1773000 18.4594000 24.1749000 - 4108 3 -0.0260103 48.2370000 19.7738000 22.9121000 - 4109 3 -0.0330134 53.9629000 16.0622000 24.0630000 - 4110 3 -0.0254280 52.0206000 17.2449000 22.8686000 - 4111 3 -0.0229569 50.1378000 15.9634000 24.1728000 - 4112 3 -0.0243826 48.2441000 17.2526000 22.9462000 - 4113 3 -0.0321034 46.3987000 20.9352000 29.1261000 - 4114 3 -0.0343693 46.4246000 18.4726000 29.1960000 - 4115 3 -0.0329440 46.4132000 15.9662000 29.2095000 - 4116 3 -0.0345617 46.3267000 20.8373000 26.6296000 - 4117 3 -0.0358395 46.3295000 18.3585000 26.6877000 - 4118 3 -0.0344154 46.2979000 15.9382000 26.7743000 - 4119 3 0.1673490 59.1921000 29.6743000 15.4677000 - 4120 3 0.2994070 59.4640000 14.9872000 17.7896000 - 4121 3 -0.0499140 55.8210000 29.6900000 27.9096000 - 4122 3 -0.0227879 52.0164000 29.6528000 25.3616000 - 4123 3 -0.0439987 55.8257000 14.8862000 22.6898000 - 4124 3 -0.0449410 55.8639000 29.7222000 22.7932000 - 4125 3 0.4011260 60.2539000 24.9842000 24.5885000 - 4126 3 0.2888120 59.4294000 14.9051000 25.3258000 diff --git a/examples/granular/in.tableting.200 b/examples/granular/in.tableting.200 new file mode 100644 index 0000000000..41723c72af --- /dev/null +++ b/examples/granular/in.tableting.200 @@ -0,0 +1,149 @@ +##################################### SIMULATION SETTINGS ################################################### + +atom_style sphere 1 +atom_modify map array +comm_modify vel yes +units si +newton off +neighbor 1.0e-3 bin +neigh_modify every 10 delay 60 check no +timestep 4e-6 +#processors 2 2 1 + +############################## SIMULATION BOUNDING BOX AND INSERT PARTICLES ################################# + +boundary f f f +read_data spheres200.data + +#################################### ADD DIE AND ATOM PARAMETERIZATION ###################################### + +variable atomRadius equal 0.44e-3*1.25 +variable atomDiameter equal 2*${atomRadius} +variable atomDensity equal 1560 +variable atomMassAvg equal ${atomDensity}*4.0/3.0*PI*${atomRadius}^3.0 +variable dieRadius equal 4e-3 +variable dieHeight equal 1e-2 + +############################## PARTICLE MATERIAL PROPERTIES AND FORCE MODEL ################################## + +pair_style granular + +# mdr = E, nu, Y, gamma, psi_b, CoR +variable YoungsModulus equal 5e6 +variable YieldStress equal 1.9e5 +variable PoissonsRatio equal 0.4 +variable SurfaceEnergy equal 2 +variable SurfaceEnergyWall equal 0.0 +variable CoR equal 0.5 +variable psi_b equal 0.5 + +# linear_history = k_t, x_gammat, mu_s +variable kt equal 2/7*${YoungsModulus}*${atomRadius} +variable kt_wall equal 2/7*${YoungsModulus}*${atomRadius} +variable xgammat equal 0.0 +variable mu_s equal 0.7 +variable mu_s_wall equal 0.1 + +# sds = mu_roll, k_roll, gamma_roll +variable mu_roll equal 0.6 +variable k_roll equal 2.25*${mu_roll}*${mu_roll}*${YoungsModulus}*${atomRadius} +variable gamma_roll equal 0.0 + +pair_coeff * * mdr ${YoungsModulus} ${PoissonsRatio} ${YieldStress} ${SurfaceEnergy} ${psi_b} ${CoR} tangential linear_history ${kt} ${xgammat} ${mu_s} rolling sds ${k_roll} ${gamma_roll} ${mu_roll} damping none + +######################################### ADD DIE AND PUNCH WALLS ############################################ + +variable disp_upper equal 0.0 +variable disp_lower equal 0.0 + +variable wall_contact_string string "granular mdr ${YoungsModulus} ${PoissonsRatio} ${YieldStress} ${SurfaceEnergyWall} ${psi_b} ${CoR} tangential linear_history ${kt_wall} ${xgammat} ${mu_s_wall} rolling sds ${k_roll} ${gamma_roll} ${mu_roll} damping none" + +variable dieHeight2 equal 2*${dieHeight} + +region lowerPunch plane 0 0 0 0 0 1 side in units box move NULL NULL v_disp_lower units box +region upperPunch plane 0 0 ${dieHeight} 0 0 -1 side in move NULL NULL v_disp_upper units box +region die cylinder z 0 0 ${dieRadius} 0 ${dieHeight2} side in units box + +fix lowerPunch all wall/gran/region ${wall_contact_string} region lowerPunch contacts +fix upperPunch all wall/gran/region ${wall_contact_string} region upperPunch contacts +fix die all wall/gran/region ${wall_contact_string} region die contacts + +compute avgUpperPunchForce all reduce sum f_upperPunch[4] +variable avgUpperPunchForce equal c_avgUpperPunchForce +compute avgLowerPunchForce all reduce sum f_lowerPunch[4] +variable avgLowerPunchForce equal c_avgLowerPunchForce + +fix printFD all print 1 "${disp_upper} ${avgUpperPunchForce} ${avgLowerPunchForce}" file punch_force_disp_tableting200.csv screen no + +##################################### INTEGRATION AND GRAVITY ################################################# + +fix 1 all nve/sphere +fix grav all gravity 9.81 vector 0 0 -1 + +########################################### SCREEN OUTPUT #################################################### + +compute 1 all erotate/sphere +thermo_style custom dt step atoms ke vol v_disp_upper +thermo 100 +thermo_modify lost ignore norm no + +##################################### SET UP DUMP OUTPUTS #################################################### + +compute ke all ke/atom +variable output_rate equal round(1e-3/dt) + +run 0 + +compute sigmaxx all property/atom d_sigmaxx +compute sigmayy all property/atom d_sigmayy +compute sigmazz all property/atom d_sigmazz +compute Velas all property/atom d_Velas + +compute sigmaxx_ave all reduce ave c_sigmaxx +compute sigmayy_ave all reduce ave c_sigmayy +compute sigmazz_ave all reduce ave c_sigmazz +compute Velas_sum all reduce sum c_Velas + +variable sxx_ave equal c_sigmaxx_ave +variable syy_ave equal c_sigmayy_ave +variable szz_ave equal c_sigmazz_ave +variable Vparticles equal c_Velas_sum + +fix log all print 1 "${sxx_ave} ${syy_ave} ${szz_ave} ${Vparticles}" file average_normal_stresses_tableting200.csv screen no +dump dumpParticles all custom ${output_rate} tableting200.dump id type mass diameter x y z vx vy vz fx fy fz c_ke c_sigmaxx c_sigmayy c_sigmazz +#dump dumpParticlesVTK all vtk ${output_rate} post/particles_*.vtk id x y z fx fy fz vx vy vz c_ke radius c_sigmaxx c_sigmayy c_sigmazz + +############################################## RUN SIMULATION ################################################# + +variable upper_punch_stroke equal 0.6733*${dieHeight} +variable vel_upper equal 0.25 + +variable settling_steps equal round(0.02/dt) +variable compression_steps equal 2*round(${upper_punch_stroke}/${vel_upper}/dt) +variable ejection_steps equal ${compression_steps} +variable free_float_steps equal round(0.02/dt) + +##### SETTLING ##### + +run ${settling_steps} + +##### Compression & Release ##### + +variable punch_frequency equal PI/2/(dt*${compression_steps}/2) +variable disp_upper equal -${upper_punch_stroke}*sin(${punch_frequency}*elapsed*dt) +variable short_release equal round(${compression_steps}*1.0) +run ${short_release} + +##### EJECTION ##### + +variable punch_frequency equal PI/2/(dt*${ejection_steps}) +variable disp_lower equal ${dieHeight}*sin(${punch_frequency}*elapsed*dt) +variable disp_upper equal 0.9*v_disp_lower +run ${ejection_steps} + +##### FREE FLOAT ##### + +variable disp_lower equal ${dieHeight} +variable disp_upper equal ${dieHeight}*0.9 +variable max_disp equal ${dieRadius}*0.75 +run ${free_float_steps} \ No newline at end of file diff --git a/examples/granular/in.triaxial.compaction.12 b/examples/granular/in.triaxial.compaction.12 new file mode 100644 index 0000000000..eee9a2bfd8 --- /dev/null +++ b/examples/granular/in.triaxial.compaction.12 @@ -0,0 +1,109 @@ +############################### SIMULATION SETTINGS ################################################### + +atom_style sphere 1 +atom_modify map array +comm_modify vel yes +units si +newton off +neighbor 2 bin +neigh_modify delay 0 +timestep 1e-6 + +##################### SIMULATION BOUNDING BOX, INSERT PARTICLES, AND INTEGRATION ####################### + +boundary f f f +read_data spheres12.data +fix integr all nve/sphere + +# create pair group for contact area outputs +group particles_1_12 id 1 12 + +########################### PARTICLE MATERIAL PROPERTIES AND FORCE MODEL ############################### + +variable atomRadius equal 0.5 + +pair_style granular + +# mdr = E, nu, Y, gamma, psi_b, CoR +variable YoungsModulus equal 1e9 +variable PoissonsRatio equal 0.3 +variable YieldStress equal 50e6 +variable SurfaceEnergy equal 0.0 +variable psi_b equal 0.5 +variable CoR equal 0.5 + +# linear_history = k_t, x_gamma,t, mu_s +variable kt equal 2/7*${YoungsModulus}*${atomRadius} +variable xgammat equal 0.0 +variable mu_s equal 0.5 + +pair_coeff * * mdr ${YoungsModulus} ${PoissonsRatio} ${YieldStress} ${SurfaceEnergy} ${psi_b} ${CoR} tangential linear_history ${kt} ${xgammat} ${mu_s} damping none + +######################################### ADD IN PLANES ################################################ + +variable boxWidth equal 3 +variable halfBoxWidth equal ${boxWidth}/2 + +variable plane_disp equal 0.0 +variable plane_disp_neg equal 0.0 + +region plane_yz_pos plane ${halfBoxWidth} 0 0 -1 0 0 side in move v_plane_disp_neg NULL NULL units box +region plane_yz_neg plane -${halfBoxWidth} 0 0 1 0 0 side in move v_plane_disp NULL NULL units box +region plane_xz_pos plane 0 ${halfBoxWidth} 0 0 -1 0 side in move NULL v_plane_disp_neg NULL units box +region plane_xz_neg plane 0 -${halfBoxWidth} 0 0 1 0 side in move NULL v_plane_disp NULL units box +region plane_xy_pos plane 0 0 ${halfBoxWidth} 0 0 -1 side in move NULL NULL v_plane_disp_neg units box +region plane_xy_neg plane 0 0 -${halfBoxWidth} 0 0 1 side in move NULL NULL v_plane_disp units box + +variable wall_contact_string string "granular mdr ${YoungsModulus} ${PoissonsRatio} ${YieldStress} ${SurfaceEnergy} ${psi_b} ${CoR} tangential linear_history ${kt} ${xgammat} ${mu_s} damping none" + +fix plane_yz_pos all wall/gran/region ${wall_contact_string} region plane_yz_pos contacts +fix plane_yz_neg all wall/gran/region ${wall_contact_string} region plane_yz_neg contacts +fix plane_xz_pos all wall/gran/region ${wall_contact_string} region plane_xz_pos contacts +fix plane_xz_neg all wall/gran/region ${wall_contact_string} region plane_xz_neg contacts +fix plane_xy_pos all wall/gran/region ${wall_contact_string} region plane_xy_pos contacts +fix plane_xy_neg all wall/gran/region ${wall_contact_string} region plane_xy_neg contacts + +compute plane_xy_neg_force all reduce sum f_plane_xy_neg[4] +variable plane_xy_neg_force equal c_plane_xy_neg_force + +compute plane_xz_neg_force all reduce sum f_plane_xz_neg[3] +variable plane_xz_neg_force equal c_plane_xz_neg_force + +compute plane_yz_neg_force all reduce sum f_plane_yz_neg[2] +variable plane_yz_neg_force equal c_plane_yz_neg_force + +fix print1 all print 1 "${plane_disp} ${plane_xy_neg_force} ${plane_xz_neg_force} ${plane_yz_neg_force}" file force_disp_triaxial12.csv screen no + +######################################## SCREEN OUTPUT #################################################### + +compute 1 all erotate/sphere +thermo_style custom dt step atoms ke c_1 vol +thermo 100 +thermo_modify lost ignore norm no + +##################################### DEFINE WALL MOVEMENT ################################################# + +variable disp_max equal 0.499 +variable ddisp equal 0.00001 +variable compression_steps equal round(${disp_max}/${ddisp}) +variable output_rate equal round(${compression_steps}/100) + +##################################### SET UP DUMP OUTPUTS #################################################### + +dump dumpParticles all custom ${output_rate} triaxial_compaction_12.dump id type mass x y z vx vy vz fx fy fz radius +#dump dmp all vtk ${output_rate} post/triaxial12particles_*.vtk id type mass x y z vx vy vz fx fy fz radius + +#################################### COMPRESS THE PARTICLES ################################################## + +run 0 + +# print out contact area evolution for particles 1 and 12 +compute Ac_1_12 particles_1_12 pair/local p13 cutoff radius +compute Ac_1_12_sum particles_1_12 reduce sum c_Ac_1_12 inputs local +variable Ac_1_12 equal c_Ac_1_12_sum +fix logArea all print 100 "${plane_disp} ${Ac_1_12}" file pair_1_12_contact_area_triaxial12.csv screen no + +variable plane_disp equal ${ddisp}*elapsed +variable plane_disp_neg equal -${ddisp}*elapsed + +run ${compression_steps} \ No newline at end of file diff --git a/examples/granular/spheres12.data b/examples/granular/spheres12.data new file mode 100644 index 0000000000..b4f07511ac --- /dev/null +++ b/examples/granular/spheres12.data @@ -0,0 +1,23 @@ +#LAMMPS data file created by matlab. +12 atoms + +1 atom types + +-10.0000000000 10.0000000000 xlo xhi +-10.0000000000 10.0000000000 ylo yhi +-10.0000000000 10.0000000000 zlo zhi + +Atoms + +1 1 0.8000000000 1000.0000000000 0.0717535226 -0.2092222842 0.3662146798 +2 1 1.2000000000 1000.0000000000 -0.8233763986 -0.7426114800 -0.8263932264 +3 1 0.8000000000 1000.0000000000 -1.0685863278 -0.4494609702 0.2196698078 +4 1 0.8000000000 1000.0000000000 0.5829432471 -1.0098803839 -0.7607543861 +5 1 0.8000000000 1000.0000000000 -0.8658471132 0.6951192569 0.0107556658 +6 1 1.2000000000 1000.0000000000 0.3966456126 0.7215053869 -0.7540113087 +7 1 1.2000000000 1000.0000000000 0.7316242921 0.8996483982 0.6751483031 +8 1 1.0000000000 1000.0000000000 0.6267527768 -0.8419367233 0.6964197101 +9 1 0.8000000000 1000.0000000000 -0.0409043189 -0.1452314035 -1.0102948313 +10 1 0.8000000000 1000.0000000000 -0.9495107709 0.6760151650 -0.9220534482 +11 1 1.0000000000 1000.0000000000 -0.7488486472 0.2188003421 0.7892021020 +12 1 1.2000000000 1000.0000000000 0.8968590780 -0.2350366437 -0.2006719701 diff --git a/examples/granular/spheres200.data b/examples/granular/spheres200.data new file mode 100644 index 0000000000..f39724477d --- /dev/null +++ b/examples/granular/spheres200.data @@ -0,0 +1,211 @@ +#LAMMPS data file created by matlab. +200 atoms + +1 atom types + +-0.005000 0.005000 xlo xhi +-0.005000 0.005000 ylo yhi +-0.001000 0.020000 zlo zhi + +Atoms + +1 1 0.001206 1560.000000 -0.000938 0.000556 0.000883 +2 1 0.000953 1560.000000 -0.002626 -0.000145 0.002778 +3 1 0.001035 1560.000000 -0.000434 0.000172 0.008458 +4 1 0.001225 1560.000000 -0.003126 -0.000604 0.004986 +5 1 0.001119 1560.000000 0.000772 0.002972 0.002568 +6 1 0.001243 1560.000000 -0.000363 0.001184 0.004927 +7 1 0.001173 1560.000000 0.000218 0.000243 0.005475 +8 1 0.000937 1560.000000 0.000033 0.000029 0.003141 +9 1 0.001055 1560.000000 -0.001660 0.001975 0.008611 +10 1 0.000938 1560.000000 -0.001818 0.002352 0.002534 +11 1 0.000990 1560.000000 0.001592 0.000435 0.004416 +12 1 0.000927 1560.000000 -0.001659 -0.000004 0.005901 +13 1 0.001272 1560.000000 0.002972 0.000553 0.007291 +14 1 0.001226 1560.000000 0.002090 0.000983 0.001406 +15 1 0.000957 1560.000000 0.002241 -0.001608 0.001304 +16 1 0.001020 1560.000000 -0.001944 0.001290 0.002030 +17 1 0.001289 1560.000000 -0.002256 -0.001173 0.003474 +18 1 0.000998 1560.000000 0.000771 0.002127 0.000906 +19 1 0.000927 1560.000000 0.000186 0.000567 0.001207 +20 1 0.001095 1560.000000 -0.000937 -0.003179 0.008173 +21 1 0.001006 1560.000000 -0.001736 0.000751 0.004618 +22 1 0.001037 1560.000000 0.000784 0.001844 0.002380 +23 1 0.001297 1560.000000 0.000234 -0.001597 0.008560 +24 1 0.001017 1560.000000 0.002454 -0.000505 0.001171 +25 1 0.001110 1560.000000 -0.000803 -0.000415 0.003714 +26 1 0.001192 1560.000000 0.002283 0.000648 0.003048 +27 1 0.000992 1560.000000 -0.000065 -0.000545 0.007062 +28 1 0.001116 1560.000000 0.002174 -0.001463 0.005830 +29 1 0.001258 1560.000000 0.001602 0.001853 0.007246 +30 1 0.001055 1560.000000 -0.001535 -0.002770 0.007196 +31 1 0.000958 1560.000000 -0.000438 -0.000260 0.004709 +32 1 0.001188 1560.000000 0.000339 -0.000355 0.009171 +33 1 0.001166 1560.000000 0.002513 -0.001215 0.004434 +34 1 0.000907 1560.000000 0.001905 -0.000373 0.004921 +35 1 0.001245 1560.000000 -0.000091 -0.002620 0.004150 +36 1 0.001302 1560.000000 0.003292 0.000184 0.005377 +37 1 0.001305 1560.000000 0.002099 0.001261 0.008939 +38 1 0.000988 1560.000000 0.003274 0.000136 0.003667 +39 1 0.000892 1560.000000 0.001798 -0.002104 0.008610 +40 1 0.001247 1560.000000 -0.003058 -0.000575 0.000948 +41 1 0.000900 1560.000000 -0.000258 -0.000469 0.001478 +42 1 0.000945 1560.000000 -0.001434 -0.001711 0.004610 +43 1 0.000977 1560.000000 -0.001410 0.002808 0.004963 +44 1 0.000930 1560.000000 -0.002110 -0.001362 0.006749 +45 1 0.000931 1560.000000 0.001256 -0.000876 0.000844 +46 1 0.000901 1560.000000 0.000899 -0.001189 0.005316 +47 1 0.000940 1560.000000 -0.002189 -0.000047 0.007240 +48 1 0.001217 1560.000000 -0.000108 -0.001333 0.002257 +49 1 0.001088 1560.000000 0.001364 -0.000594 0.002789 +50 1 0.001143 1560.000000 -0.000311 -0.001425 0.006092 +51 1 0.001054 1560.000000 0.002262 0.002312 0.004315 +52 1 0.001016 1560.000000 -0.000724 0.000741 0.003295 +53 1 0.001051 1560.000000 0.000527 -0.001987 0.003307 +54 1 0.000905 1560.000000 0.000827 0.001457 0.005868 +55 1 0.001195 1560.000000 -0.001176 -0.000645 0.000798 +56 1 0.001253 1560.000000 0.002583 -0.001847 0.003310 +57 1 0.000982 1560.000000 0.001551 -0.002803 0.005076 +58 1 0.000945 1560.000000 -0.000481 0.000354 0.007220 +59 1 0.001040 1560.000000 -0.002736 0.001076 0.008769 +60 1 0.000917 1560.000000 0.000826 -0.001887 0.006449 +61 1 0.000914 1560.000000 -0.001171 -0.001592 0.007266 +62 1 0.000959 1560.000000 0.000834 -0.002671 0.007105 +63 1 0.000990 1560.000000 -0.000251 -0.001327 0.004339 +64 1 0.001220 1560.000000 0.001384 0.002896 0.005874 +65 1 0.000949 1560.000000 -0.001340 -0.000608 0.007496 +66 1 0.001306 1560.000000 0.002187 0.002068 0.002629 +67 1 0.001206 1560.000000 0.000148 0.001506 0.008517 +68 1 0.001123 1560.000000 0.001288 -0.000303 0.006613 +69 1 0.001151 1560.000000 -0.000876 0.001549 0.001740 +70 1 0.001315 1560.000000 -0.001902 -0.002590 0.001344 +71 1 0.000927 1560.000000 0.002285 -0.000866 0.006900 +72 1 0.001279 1560.000000 -0.000165 0.002689 0.007449 +73 1 0.000910 1560.000000 0.001009 0.001054 0.005049 +74 1 0.001148 1560.000000 -0.002229 -0.001285 0.008736 +75 1 0.001067 1560.000000 -0.000261 -0.002945 0.002157 +76 1 0.000993 1560.000000 -0.001641 0.002272 0.007601 +77 1 0.001228 1560.000000 0.001939 -0.000214 0.008903 +78 1 0.001076 1560.000000 0.000767 0.001172 0.003556 +79 1 0.001105 1560.000000 -0.000561 0.002493 0.004214 +80 1 0.001195 1560.000000 0.002694 -0.000817 0.007949 +81 1 0.001239 1560.000000 -0.000968 -0.003145 0.006096 +82 1 0.001083 1560.000000 -0.000808 0.001813 0.006396 +83 1 0.000923 1560.000000 0.000632 -0.001437 0.001310 +84 1 0.000981 1560.000000 -0.001842 0.002774 0.006508 +85 1 0.000998 1560.000000 -0.002775 0.001616 0.001453 +86 1 0.000979 1560.000000 -0.002520 0.001715 0.007741 +87 1 0.001002 1560.000000 -0.001465 -0.001931 0.006048 +88 1 0.000958 1560.000000 0.003264 0.000707 0.001189 +89 1 0.001052 1560.000000 -0.001314 -0.000701 0.002721 +90 1 0.001096 1560.000000 0.001154 0.002129 0.004403 +91 1 0.001104 1560.000000 0.002118 0.001977 0.000794 +92 1 0.001263 1560.000000 -0.001499 -0.002764 0.003441 +93 1 0.001086 1560.000000 -0.001096 0.002514 0.001154 +94 1 0.000895 1560.000000 0.001130 0.000029 0.001045 +95 1 0.000964 1560.000000 0.000905 -0.003200 0.000542 +96 1 0.000898 1560.000000 -0.000868 0.003148 0.008306 +97 1 0.000907 1560.000000 -0.001406 0.001144 0.007862 +98 1 0.001176 1560.000000 0.001246 -0.001074 0.004327 +99 1 0.001148 1560.000000 0.001512 -0.002739 0.003346 +100 1 0.000922 1560.000000 0.001470 -0.000036 0.007695 +101 1 0.001031 1560.000000 -0.002751 0.000928 0.004124 +102 1 0.001030 1560.000000 -0.000177 -0.002370 0.005374 +103 1 0.000915 1560.000000 0.000824 0.000521 0.007070 +104 1 0.001085 1560.000000 -0.002281 -0.000023 0.009123 +105 1 0.001004 1560.000000 -0.000167 0.002610 0.008905 +106 1 0.001060 1560.000000 -0.000389 -0.002220 0.007688 +107 1 0.000920 1560.000000 -0.000483 0.003231 0.006505 +108 1 0.001122 1560.000000 0.001781 -0.001547 0.002237 +109 1 0.001172 1560.000000 -0.002650 0.000830 0.005429 +110 1 0.001137 1560.000000 -0.000030 -0.003246 0.001024 +111 1 0.001315 1560.000000 0.001470 -0.001735 0.007580 +112 1 0.001245 1560.000000 0.000481 -0.003067 0.006025 +113 1 0.000904 1560.000000 0.000632 -0.000184 0.002010 +114 1 0.000883 1560.000000 -0.001828 0.002191 0.003819 +115 1 0.000974 1560.000000 0.002167 0.001616 0.006226 +116 1 0.001150 1560.000000 0.000871 -0.002731 0.002136 +117 1 0.001312 1560.000000 -0.000326 -0.001971 0.001000 +118 1 0.000914 1560.000000 0.001020 0.000810 0.002086 +119 1 0.001136 1560.000000 -0.000101 -0.003277 0.007246 +120 1 0.000991 1560.000000 -0.001944 0.000576 0.003215 +121 1 0.001216 1560.000000 -0.000913 -0.001165 0.008857 +122 1 0.001045 1560.000000 -0.003110 0.001062 0.002973 +123 1 0.000918 1560.000000 0.000348 0.000365 0.004046 +124 1 0.001279 1560.000000 -0.000884 0.003087 0.002268 +125 1 0.001065 1560.000000 -0.002238 0.001309 0.006452 +126 1 0.001012 1560.000000 -0.002059 -0.001354 0.001935 +127 1 0.001142 1560.000000 -0.003011 0.000567 0.001739 +128 1 0.000921 1560.000000 0.001764 0.002804 0.008177 +129 1 0.001151 1560.000000 -0.003105 -0.000384 0.006602 +130 1 0.000967 1560.000000 0.000932 0.000588 0.008823 +131 1 0.000908 1560.000000 -0.001873 -0.001947 0.007825 +132 1 0.000923 1560.000000 -0.002993 0.000883 0.007425 +133 1 0.001171 1560.000000 0.003310 -0.000405 0.006558 +134 1 0.000977 1560.000000 -0.000098 -0.000180 0.000492 +135 1 0.000938 1560.000000 -0.000706 -0.000129 0.006085 +136 1 0.001008 1560.000000 -0.000256 0.002333 0.000550 +137 1 0.001073 1560.000000 0.000534 -0.000055 0.008080 +138 1 0.000890 1560.000000 0.000351 0.001695 0.007195 +139 1 0.000973 1560.000000 0.002593 0.001907 0.005394 +140 1 0.001176 1560.000000 -0.001862 -0.000534 0.004494 +141 1 0.001306 1560.000000 -0.000951 0.001053 0.009299 +142 1 0.001103 1560.000000 -0.001937 -0.002711 0.008485 +143 1 0.001262 1560.000000 -0.002947 -0.001470 0.007682 +144 1 0.000914 1560.000000 0.002047 0.000811 0.005504 +145 1 0.000954 1560.000000 0.001935 -0.002349 0.006632 +146 1 0.001003 1560.000000 0.000766 -0.002635 0.008483 +147 1 0.001137 1560.000000 0.000102 0.003195 0.004922 +148 1 0.001006 1560.000000 -0.001982 0.001014 0.000685 +149 1 0.001255 1560.000000 -0.000718 0.001939 0.003056 +150 1 0.001057 1560.000000 -0.001189 -0.001717 0.003045 +151 1 0.001228 1560.000000 0.001581 0.002926 0.003510 +152 1 0.001052 1560.000000 -0.002172 0.001949 0.004831 +153 1 0.000979 1560.000000 -0.001817 0.000291 0.002048 +154 1 0.001286 1560.000000 -0.002647 -0.001839 0.004620 +155 1 0.001085 1560.000000 -0.000081 0.000850 0.002139 +156 1 0.000990 1560.000000 -0.000081 0.002105 0.005587 +157 1 0.001043 1560.000000 0.001636 -0.000112 0.001860 +158 1 0.001309 1560.000000 0.003216 -0.000851 0.002791 +159 1 0.000913 1560.000000 0.000608 0.003148 0.006565 +160 1 0.000919 1560.000000 0.000536 -0.003106 0.003249 +161 1 0.000943 1560.000000 0.003145 -0.000528 0.008915 +162 1 0.000993 1560.000000 -0.002811 -0.000099 0.008110 +163 1 0.001125 1560.000000 0.001415 -0.002271 0.000643 +164 1 0.000919 1560.000000 -0.001406 0.000223 0.006781 +165 1 0.001040 1560.000000 0.000690 0.003193 0.008329 +166 1 0.001055 1560.000000 0.001075 0.002584 0.009093 +167 1 0.001176 1560.000000 0.000851 0.003176 0.000591 +168 1 0.001003 1560.000000 -0.001462 0.001511 0.005544 +169 1 0.001126 1560.000000 -0.000077 0.003324 0.001347 +170 1 0.001068 1560.000000 0.003110 0.000810 0.008495 +171 1 0.001011 1560.000000 -0.001661 0.000117 0.008201 +172 1 0.001066 1560.000000 -0.000359 -0.003279 0.009094 +173 1 0.001303 1560.000000 0.003066 0.001188 0.004082 +174 1 0.000983 1560.000000 0.000354 0.002261 0.003558 +175 1 0.001137 1560.000000 0.002860 -0.001571 0.009180 +176 1 0.001070 1560.000000 0.001246 -0.001279 0.009104 +177 1 0.000886 1560.000000 0.002271 -0.000316 0.003675 +178 1 0.000983 1560.000000 -0.001987 -0.002490 0.005377 +179 1 0.000939 1560.000000 0.000601 -0.000861 0.003477 +180 1 0.001177 1560.000000 0.001522 0.002902 0.001690 +181 1 0.001036 1560.000000 -0.001200 -0.002874 0.004750 +182 1 0.000898 1560.000000 -0.001705 -0.001140 0.005503 +183 1 0.001315 1560.000000 0.002732 0.001766 0.007885 +184 1 0.001318 1560.000000 -0.002909 -0.001610 0.005936 +185 1 0.001218 1560.000000 0.003213 0.000884 0.002316 +186 1 0.001234 1560.000000 -0.002394 -0.002298 0.002575 +187 1 0.001160 1560.000000 -0.003313 -0.000065 0.003625 +188 1 0.001022 1560.000000 -0.003096 -0.001048 0.002151 +189 1 0.000966 1560.000000 0.001891 -0.002093 0.004404 +190 1 0.001048 1560.000000 -0.002367 0.002338 0.000697 +191 1 0.000995 1560.000000 -0.001204 -0.001912 0.002030 +192 1 0.001136 1560.000000 -0.001152 -0.002402 0.009223 +193 1 0.001083 1560.000000 -0.002588 -0.001768 0.000753 +194 1 0.000946 1560.000000 -0.001338 -0.000741 0.006527 +195 1 0.000943 1560.000000 -0.000073 0.003254 0.003663 +196 1 0.001059 1560.000000 0.000087 0.000958 0.006388 +197 1 0.001131 1560.000000 0.001030 0.001019 0.000752 +198 1 0.001257 1560.000000 -0.001365 0.002946 0.009266 +199 1 0.000891 1560.000000 -0.000445 -0.000273 0.002382 +200 1 0.001055 1560.000000 0.001781 0.000748 0.006583 diff --git a/examples/multi/in.granular b/examples/multi/in.granular index 468d0dcbf1..6abbd8da3b 100644 --- a/examples/multi/in.granular +++ b/examples/multi/in.granular @@ -1,4 +1,4 @@ -# Big colloid particles and small LJ particles +# Binary granular system units lj atom_style sphere diff --git a/examples/snap/README.md b/examples/snap/README.md index 305f920ae8..1df24acf1f 100644 --- a/examples/snap/README.md +++ b/examples/snap/README.md @@ -9,5 +9,11 @@ in.snap.Mo_Chen # SNAP linear Mo potential in.snap.compute # SNAP compute for training a linear model in.snap.compute.quadratic # SNAP compute for training a quadratic model in.snap.scale.Ni_Zuo_JCPA2020 # SNAP linear Ni potential with thermodynamic integration (fix adapt scale) +in.C_SNAP # SNAP carbon potential compute_snap_dgrad.py # SNAP compute with dgradflag (dBi/dRj) for training a non-linear model + +in.snap.grid # SNAP descriptors on a grid +in.snap.grid.triclinic # SNAP descriptors on a grid, triclinic +in.gaussian.grid # Gaussian descriptors on a grid + diff --git a/examples/snap/in.gaussian.grid b/examples/snap/in.gaussian.grid new file mode 100644 index 0000000000..48aeec1632 --- /dev/null +++ b/examples/snap/in.gaussian.grid @@ -0,0 +1,68 @@ +# Demonstrate calculation of Gaussian descriptors on a grid +# for a cell with two atoms of type 1 and type 2. +# The output in dump.glocal shows that for grid points +# sitting on an atom of type 1 or 2: +# val1 = 1.0/(0.1355*sqrt(2.0*pi))**3 = 25.5219 +# val2 = 1.0/(0.2 *sqrt(2.0*pi))**3 = 7.93670 +# These values are extracted to the log file +# + +variable nrep index 1 +variable a index 3.316 +variable ngrid index 2 + +units metal +atom_modify map hash + +# generate the box and atom positions using a BCC lattice + +variable nx equal ${nrep} +variable ny equal ${nrep} +variable nz equal ${nrep} + +boundary p p p + +lattice custom $a & + a1 1 0 0 & + a2 0 1 0 & + a3 0 0 1 & + basis 0 0 0 & + basis 0.5 0.5 0.5 & + +region box block 0 ${nx} 0 ${ny} 0 ${nz} +create_box 2 box +create_atoms 1 box basis 1 1 basis 2 2 + +mass * 180.88 + +# define atom compute and grid compute + +variable rcutfac equal 4.67637 +variable radelem1 equal 0.5 +variable radelem2 equal 0.5 +variable sigmaelem1 equal 0.1355 +variable sigmaelem2 equal 0.2 +variable gaussian_options string & + "${rcutfac} ${radelem1} ${radelem2} ${sigmaelem1} ${sigmaelem2}" + +# build zero potential to force ghost atom creation + +pair_style zero ${rcutfac} +pair_coeff * * + +# define atom and grid computes + +compute mygridlocal all gaussian/grid/local grid ${ngrid} ${ngrid} ${ngrid} & + ${gaussian_options} + +# define output + +dump 1 all local 1000 dump.glocal c_mygridlocal[*] +dump 2 all custom 1000 dump.gatom id x y z +compute val1 all reduce max c_mygridlocal[7] inputs local +compute val2 all reduce max c_mygridlocal[8] inputs local +thermo_style custom step c_val1 c_val2 + +# run + +run 0 diff --git a/examples/snap/in.grid.snap b/examples/snap/in.snap.grid similarity index 100% rename from examples/snap/in.grid.snap rename to examples/snap/in.snap.grid diff --git a/examples/snap/in.grid.tri b/examples/snap/in.snap.grid.triclinic similarity index 99% rename from examples/snap/in.grid.tri rename to examples/snap/in.snap.grid.triclinic index 95a14f3bb4..59063f576e 100644 --- a/examples/snap/in.grid.tri +++ b/examples/snap/in.snap.grid.triclinic @@ -47,7 +47,6 @@ lattice custom $a & basis 0.0 0.0 0.5 & spacing 1 1 1 -box tilt large region box prism 0 ${nx} 0 ${ny} 0 ${nz} ${ny} ${nz} ${nz} create_box 1 box create_atoms 1 box diff --git a/examples/snap/log.10Dec24.gaussian.grid.g++.1 b/examples/snap/log.10Dec24.gaussian.grid.g++.1 new file mode 100644 index 0000000000..b158ac07d0 --- /dev/null +++ b/examples/snap/log.10Dec24.gaussian.grid.g++.1 @@ -0,0 +1,129 @@ +LAMMPS (19 Nov 2024 - Development - patch_19Nov2024-59-g16e0a7788a) +OMP_NUM_THREADS environment is not set. Defaulting to 1 thread. (src/comm.cpp:99) + using 1 OpenMP thread(s) per MPI task +# Demonstrate calculation of Gaussian descriptors on a grid +# for a cell with two atoms of type 1 and type 2. +# The output in dump.glocal shows that for grid points +# sitting on an atom of type 1 or 2: +# val1 = 1.0/(0.1355*sqrt(2.0*pi))**3 = 25.5219 +# val2 = 1.0/(0.2 *sqrt(2.0*pi))**3 = 7.93670 +# These values are extracted to the log file +# + +variable nrep index 1 +variable a index 3.316 +variable ngrid index 2 + +units metal +atom_modify map hash + +# generate the box and atom positions using a BCC lattice + +variable nx equal ${nrep} +variable nx equal 1 +variable ny equal ${nrep} +variable ny equal 1 +variable nz equal ${nrep} +variable nz equal 1 + +boundary p p p + +lattice custom $a a1 1 0 0 a2 0 1 0 a3 0 0 1 basis 0 0 0 basis 0.5 0.5 0.5 +lattice custom 3.316 a1 1 0 0 a2 0 1 0 a3 0 0 1 basis 0 0 0 basis 0.5 0.5 0.5 +Lattice spacing in x,y,z = 3.316 3.316 3.316 +region box block 0 ${nx} 0 ${ny} 0 ${nz} +region box block 0 1 0 ${ny} 0 ${nz} +region box block 0 1 0 1 0 ${nz} +region box block 0 1 0 1 0 1 +create_box 2 box +Created orthogonal box = (0 0 0) to (3.316 3.316 3.316) + 1 by 1 by 1 MPI processor grid +create_atoms 1 box basis 1 1 basis 2 2 +Created 2 atoms + using lattice units in orthogonal box = (0 0 0) to (3.316 3.316 3.316) + create_atoms CPU = 0.001 seconds + +mass * 180.88 + +# define atom compute and grid compute + +variable rcutfac equal 4.67637 +variable radelem1 equal 0.5 +variable radelem2 equal 0.5 +variable sigmaelem1 equal 0.1355 +variable sigmaelem2 equal 0.2 +variable gaussian_options string "${rcutfac} ${radelem1} ${radelem2} ${sigmaelem1} ${sigmaelem2}" +4.67637 ${radelem1} ${radelem2} ${sigmaelem1} ${sigmaelem2} +4.67637 0.5 ${radelem2} ${sigmaelem1} ${sigmaelem2} +4.67637 0.5 0.5 ${sigmaelem1} ${sigmaelem2} +4.67637 0.5 0.5 0.1355 ${sigmaelem2} +4.67637 0.5 0.5 0.1355 0.2 + +# build zero potential to force ghost atom creation + +pair_style zero ${rcutfac} +pair_style zero 4.67637 +pair_coeff * * + +# define atom and grid computes + +compute mygridlocal all gaussian/grid/local grid ${ngrid} ${ngrid} ${ngrid} ${gaussian_options} +compute mygridlocal all gaussian/grid/local grid 2 ${ngrid} ${ngrid} ${gaussian_options} +compute mygridlocal all gaussian/grid/local grid 2 2 ${ngrid} ${gaussian_options} +compute mygridlocal all gaussian/grid/local grid 2 2 2 ${gaussian_options} +compute mygridlocal all gaussian/grid/local grid 2 2 2 4.67637 0.5 0.5 0.1355 0.2 + +# define output + +dump 1 all local 1000 dump.glocal c_mygridlocal[*] +dump 2 all custom 1000 dump.gatom id x y z +compute val1 all reduce max c_mygridlocal[7] inputs local +compute val2 all reduce max c_mygridlocal[8] inputs local +thermo_style custom step c_val1 c_val2 + +# run + +run 0 +WARNING: No fixes with time integration, atoms won't move (src/verlet.cpp:60) +Generated 0 of 1 mixed pair_coeff terms from geometric mixing rule +Neighbor list info ... + update: every = 1 steps, delay = 0 steps, check = yes + max neighbors/atom: 2000, page size: 100000 + master list distance cutoff = 6.67637 + ghost atom cutoff = 6.67637 + binsize = 3.338185, bins = 1 1 1 + 1 neighbor lists, perpetual/occasional/extra = 1 0 0 + (1) pair zero, perpetual + attributes: half, newton on + pair build: half/bin/atomonly/newton + stencil: half/bin/3d + bin: standard +Per MPI rank memory allocation (min/avg/max) = 3.492 | 3.492 | 3.492 Mbytes + Step c_val1 c_val2 + 0 25.521859 7.9367045 +Loop time of 1.088e-06 on 1 procs for 0 steps with 2 atoms + +183.8% CPU use with 1 MPI tasks x 1 OpenMP threads + +MPI task timing breakdown: +Section | min time | avg time | max time |%varavg| %total +--------------------------------------------------------------- +Pair | 0 | 0 | 0 | 0.0 | 0.00 +Neigh | 0 | 0 | 0 | 0.0 | 0.00 +Comm | 0 | 0 | 0 | 0.0 | 0.00 +Output | 0 | 0 | 0 | 0.0 | 0.00 +Modify | 0 | 0 | 0 | 0.0 | 0.00 +Other | | 1.088e-06 | | |100.00 + +Nlocal: 2 ave 2 max 2 min +Histogram: 1 0 0 0 0 0 0 0 0 0 +Nghost: 339 ave 339 max 339 min +Histogram: 1 0 0 0 0 0 0 0 0 0 +Neighs: 64 ave 64 max 64 min +Histogram: 1 0 0 0 0 0 0 0 0 0 + +Total # of neighbors = 64 +Ave neighs/atom = 32 +Neighbor list builds = 0 +Dangerous builds = 0 +Total wall time: 0:00:00 diff --git a/examples/snap/log.10Dec24.gaussian.grid.g++.4 b/examples/snap/log.10Dec24.gaussian.grid.g++.4 new file mode 100644 index 0000000000..54cc842bc7 --- /dev/null +++ b/examples/snap/log.10Dec24.gaussian.grid.g++.4 @@ -0,0 +1,130 @@ +LAMMPS (19 Nov 2024 - Development - patch_19Nov2024-59-g16e0a7788a) +OMP_NUM_THREADS environment is not set. Defaulting to 1 thread. (src/comm.cpp:99) + using 1 OpenMP thread(s) per MPI task +# Demonstrate calculation of Gaussian descriptors on a grid +# for a cell with two atoms of type 1 and type 2. +# The output in dump.glocal shows that for grid points +# sitting on an atom of type 1 or 2: +# val1 = 1.0/(0.1355*sqrt(2.0*pi))**3 = 25.5219 +# val2 = 1.0/(0.2 *sqrt(2.0*pi))**3 = 7.93670 +# These values are extracted to the log file +# + +variable nrep index 1 +variable a index 3.316 +variable ngrid index 2 + +units metal +atom_modify map hash + +# generate the box and atom positions using a BCC lattice + +variable nx equal ${nrep} +variable nx equal 1 +variable ny equal ${nrep} +variable ny equal 1 +variable nz equal ${nrep} +variable nz equal 1 + +boundary p p p + +lattice custom $a a1 1 0 0 a2 0 1 0 a3 0 0 1 basis 0 0 0 basis 0.5 0.5 0.5 +lattice custom 3.316 a1 1 0 0 a2 0 1 0 a3 0 0 1 basis 0 0 0 basis 0.5 0.5 0.5 +Lattice spacing in x,y,z = 3.316 3.316 3.316 +region box block 0 ${nx} 0 ${ny} 0 ${nz} +region box block 0 1 0 ${ny} 0 ${nz} +region box block 0 1 0 1 0 ${nz} +region box block 0 1 0 1 0 1 +create_box 2 box +Created orthogonal box = (0 0 0) to (3.316 3.316 3.316) + 1 by 2 by 2 MPI processor grid +create_atoms 1 box basis 1 1 basis 2 2 +Created 2 atoms + using lattice units in orthogonal box = (0 0 0) to (3.316 3.316 3.316) + create_atoms CPU = 0.001 seconds + +mass * 180.88 + +# define atom compute and grid compute + +variable rcutfac equal 4.67637 +variable radelem1 equal 0.5 +variable radelem2 equal 0.5 +variable sigmaelem1 equal 0.1355 +variable sigmaelem2 equal 0.2 +variable gaussian_options string "${rcutfac} ${radelem1} ${radelem2} ${sigmaelem1} ${sigmaelem2}" +4.67637 ${radelem1} ${radelem2} ${sigmaelem1} ${sigmaelem2} +4.67637 0.5 ${radelem2} ${sigmaelem1} ${sigmaelem2} +4.67637 0.5 0.5 ${sigmaelem1} ${sigmaelem2} +4.67637 0.5 0.5 0.1355 ${sigmaelem2} +4.67637 0.5 0.5 0.1355 0.2 + +# build zero potential to force ghost atom creation + +pair_style zero ${rcutfac} +pair_style zero 4.67637 +pair_coeff * * + +# define atom and grid computes + +compute mygridlocal all gaussian/grid/local grid ${ngrid} ${ngrid} ${ngrid} ${gaussian_options} +compute mygridlocal all gaussian/grid/local grid 2 ${ngrid} ${ngrid} ${gaussian_options} +compute mygridlocal all gaussian/grid/local grid 2 2 ${ngrid} ${gaussian_options} +compute mygridlocal all gaussian/grid/local grid 2 2 2 ${gaussian_options} +compute mygridlocal all gaussian/grid/local grid 2 2 2 4.67637 0.5 0.5 0.1355 0.2 + +# define output + +dump 1 all local 1000 dump.glocal c_mygridlocal[*] +dump 2 all custom 1000 dump.gatom id x y z +compute val1 all reduce max c_mygridlocal[7] inputs local +compute val2 all reduce max c_mygridlocal[8] inputs local +thermo_style custom step c_val1 c_val2 + +# run + +run 0 +WARNING: No fixes with time integration, atoms won't move (src/verlet.cpp:60) +Generated 0 of 1 mixed pair_coeff terms from geometric mixing rule +Neighbor list info ... + update: every = 1 steps, delay = 0 steps, check = yes + max neighbors/atom: 2000, page size: 100000 + master list distance cutoff = 6.67637 + ghost atom cutoff = 6.67637 + binsize = 3.338185, bins = 1 1 1 + 1 neighbor lists, perpetual/occasional/extra = 1 0 0 + (1) pair zero, perpetual + attributes: half, newton on + pair build: half/bin/atomonly/newton + stencil: half/bin/3d + bin: standard +WARNING: Proc sub-domain size < neighbor skin, could lead to lost atoms (src/domain.cpp:1202) +Per MPI rank memory allocation (min/avg/max) = 3.522 | 3.523 | 3.524 Mbytes + Step c_val1 c_val2 + 0 25.521859 7.9367045 +Loop time of 2.238e-06 on 4 procs for 0 steps with 2 atoms + +89.4% CPU use with 4 MPI tasks x 1 OpenMP threads + +MPI task timing breakdown: +Section | min time | avg time | max time |%varavg| %total +--------------------------------------------------------------- +Pair | 0 | 0 | 0 | 0.0 | 0.00 +Neigh | 0 | 0 | 0 | 0.0 | 0.00 +Comm | 0 | 0 | 0 | 0.0 | 0.00 +Output | 0 | 0 | 0 | 0.0 | 0.00 +Modify | 0 | 0 | 0 | 0.0 | 0.00 +Other | | 2.238e-06 | | |100.00 + +Nlocal: 0.5 ave 1 max 0 min +Histogram: 2 0 0 0 0 0 0 0 0 2 +Nghost: 274.5 ave 275 max 274 min +Histogram: 2 0 0 0 0 0 0 0 0 2 +Neighs: 16 ave 40 max 0 min +Histogram: 2 0 0 0 0 0 1 0 0 1 + +Total # of neighbors = 64 +Ave neighs/atom = 32 +Neighbor list builds = 0 +Dangerous builds = 0 +Total wall time: 0:00:00 diff --git a/examples/stress_vcm/README b/examples/stress_vcm/README new file mode 100644 index 0000000000..aeb065c678 --- /dev/null +++ b/examples/stress_vcm/README @@ -0,0 +1,32 @@ +README stress_vcm +================= + +Contents: + +- in.stress_vcm: Example script showing how to remove binned + velocities of center of mass (VCM) from stress calculations. +- stress_comparison.19Nov24.png: Plot shows the stress + calculated in bars on the y axis for each positional bin on + the x axis. Plotted are three different time steps from + stress profiles with and without the VCM removed. Plot + generated using Python. +- stress_xx.19Nov24.out: Output file generated by fix ave/time. +- log.19Nov24.stress_vcm.g++.1: LAMMPS log file with 1 proc. +- log.19Nov24.stress_vcm.g++.4: LAMMPS log file with 4 procs. + +Notes: + +- Running this script as-is will generate two files. A log + file with thermodynamic data and a stress_xx.out file + containing the binned stress profile with the VCM removed. +- To generate the binned stress profile without removing the + VCM then the compute stress/atom command at step three + needs the last keyword "ch_temp_vcm" to be replaced with + "NULL". +- Uncommenting the line under "Atom dump" will generate an + all atom dump file every 50 time steps containing atom ID, + type, and xyz coordinates. +- Uncommenting the lines under "Image dumps" will generate + .jpg image files every 250 timesteps. +- Uncommenting lines under "Movie dump" will generate a .avi + movie file showing timesteps every 125 timesteps. \ No newline at end of file diff --git a/examples/stress_vcm/in.stress_vcm b/examples/stress_vcm/in.stress_vcm new file mode 100644 index 0000000000..f2ecdb29a7 --- /dev/null +++ b/examples/stress_vcm/in.stress_vcm @@ -0,0 +1,113 @@ +# Removing Binned Velocities of Center of Mass (VCM) from Stress + +# This example shows how to remove rigid body motion from +# binned stress calculations. This uses a combination of commands +# from compute chunk/atom, compute temp/chunk, compute +# stress/atom and fix ave/time. We'll show how these commands +# work in the context of a shockwave experiment on a cube of +# atoms. To shock the cube, a rectangular region of atoms is +# frozen, moved into the cube with a constant velocity along the +# x direction, and then unfrozen. As the shockwave begins +# propagating, the body of the cube also moves along the x +# direction. To better understand the stress dynamics of the +# cube we remove the velocity component belonging to the overall +# motion of each bin. + +units metal +boundary p p p +atom_style atomic +lattice fcc 5.3589 +processors 1 * * + +# Defining regions for box and atoms. +# In this experiment an elongated simulation cell is +# defined in the x direction to allow for non-periodic +# motion of the atoms. + +region box1 block -3 24 0 12 0 12 units lattice +region box2 block 0 12 0 12 0 12 units lattice + +# Creating box and atoms + +create_box 1 box1 +create_atoms 1 region box2 + +mass 1 40.00 + +# Adding energy to the system + +velocity all create 600.0 9999 + +pair_style lj/cut 10 +pair_coeff 1 1 0.04 3.405 + +# Begin time integration + +timestep 2e-3 + +fix fix_nve all nve + +thermo 100 + +run 500 + +#--------------------------------------# +# Chunk, Stress, and VCM removal steps # +#--------------------------------------# + +# 1. Create 20 equispaced bins sliced along the x direction. +# -"units reduced" normalizes the distance from 0.0 to 1.0 +variable nbins index 20 +variable fraction equal 1.0/v_nbins +variable volfrac equal 1/(vol*${fraction}) +compute ch_id all chunk/atom bin/1d x lower ${fraction} units reduced + +# 2. Calculate temperature bins with VCM aka COM velocities removed. +compute ch_temp_vcm all temp/chunk ch_id com yes + +# 3. Compute per atom stress with VCM removed via temp-ID. +# -The velocities from specified temp-ID are used to compute stress. +# -Stress/atom units are pressure*volume! Optionally handled next step. +compute atom_stress_vcm all stress/atom ch_temp_vcm + +# 4. Divide out bin volume from xx stress component. +variable stress atom -(c_atom_stress_vcm[1])/(vol*${fraction}) + +# 5. Sum the per atom stresses in each bin. +compute ch_stress_vcm all reduce/chunk ch_id sum v_stress + +# 6. Average and output to file. +# -The average output is every 100 steps with samples collected 20 times with 5 step intervals. +fix ave_stress_vcm all ave/time 5 20 100 c_ch_stress_vcm mode vector file stress_xx.out + +#--------------------------------------# + +# Piston compressing along x direction + +region piston block -1 1 INF INF INF INF units lattice +group piston region piston +fix fix_piston piston move linear 5 0 0 units box # strain rate ~ 8e10 1/s + +thermo_style custom step temp ke pe lx ly lz pxx pyy pzz econserve + +# Atom dump + +# dump atom_dump all atom 50 dump.vcm + +# # Image dumps + +# dump 2 all image 250 image.*.jpg type type & +# axes yes 0.8 0.02 view 60 -30 +# dump_modify 2 pad 1 + +# # Movie dump + +# dump 3 all movie 125 movie.avi type type & +# axes yes 0.8 0.02 view 60 -30 +# dump_modify 3 pad 1 + +run 500 + +unfix fix_piston + +run 1500 diff --git a/examples/stress_vcm/log.19Nov24.stress_vcm.g++.1 b/examples/stress_vcm/log.19Nov24.stress_vcm.g++.1 new file mode 100644 index 0000000000..93ae029334 --- /dev/null +++ b/examples/stress_vcm/log.19Nov24.stress_vcm.g++.1 @@ -0,0 +1,253 @@ +LAMMPS (19 Nov 2024) +OMP_NUM_THREADS environment is not set. Defaulting to 1 thread. (src/comm.cpp:99) + using 1 OpenMP thread(s) per MPI task +# Removing Binned Velocities of Center of Mass (VCM) from Stress + +# This example shows how to remove rigid body motion from +# binned stress calculations. This uses a combination of commands +# from compute chunk/atom, compute temp/chunk, compute +# stress/atom and fix ave/time. We'll show how these commands +# work in the context of a shockwave experiment on a cube of +# atoms. To shock the cube, a rectangular region of atoms is +# frozen, moved into the cube with a constant velocity along the +# x direction, and then unfrozen. As the shockwave begins +# propagating, the body of the cube also moves along the x +# direction. To better understand the stress dynamics of the +# cube we remove the velocity component belonging to the overall +# motion of each bin. + +units metal +boundary p p p +atom_style atomic +lattice fcc 5.3589 +Lattice spacing in x,y,z = 5.3589 5.3589 5.3589 +processors 1 * * + +# Defining regions for box and atoms. +# In this experiment an elongated simulation cell is +# defined in the x direction to allow for non-periodic +# motion of the atoms. + +region box1 block -3 24 0 12 0 12 units lattice +region box2 block 0 12 0 12 0 12 units lattice + +# Creating box and atoms + +create_box 1 box1 +Created orthogonal box = (-16.0767 0 0) to (128.6136 64.3068 64.3068) + 1 by 1 by 1 MPI processor grid +create_atoms 1 region box2 +Created 7200 atoms + using lattice units in orthogonal box = (-16.0767 0 0) to (128.6136 64.3068 64.3068) + create_atoms CPU = 0.002 seconds + +mass 1 40.00 + +# Adding energy to the system + +velocity all create 600.0 9999 + +pair_style lj/cut 10 +pair_coeff 1 1 0.04 3.405 + +# Begin time integration + +timestep 2e-3 + +fix fix_nve all nve + +thermo 100 + +run 500 +Generated 0 of 0 mixed pair_coeff terms from geometric mixing rule +Neighbor list info ... + update: every = 1 steps, delay = 0 steps, check = yes + max neighbors/atom: 2000, page size: 100000 + master list distance cutoff = 12 + ghost atom cutoff = 12 + binsize = 6, bins = 25 11 11 + 1 neighbor lists, perpetual/occasional/extra = 1 0 0 + (1) pair lj/cut, perpetual + attributes: half, newton on + pair build: half/bin/atomonly/newton + stencil: half/bin/3d + bin: standard +Per MPI rank memory allocation (min/avg/max) = 5.721 | 5.721 | 5.721 Mbytes + Step Temp E_pair E_mol TotEng Press + 0 600 -2252.7567 0 -1694.4304 -974.62456 + 100 284.72172 -1977.4291 0 -1712.483 2453.7429 + 200 304.44519 -1994.7937 0 -1711.4941 1822.2699 + 300 304.28012 -1993.2958 0 -1710.1498 1498.3794 + 400 296.76492 -1985.1364 0 -1708.9836 1259.9474 + 500 295.00895 -1982.4224 0 -1707.9036 964.9526 +Loop time of 3.01696 on 1 procs for 500 steps with 7200 atoms + +Performance: 28.638 ns/day, 0.838 hours/ns, 165.730 timesteps/s, 1.193 Matom-step/s +99.4% CPU use with 1 MPI tasks x 1 OpenMP threads + +MPI task timing breakdown: +Section | min time | avg time | max time |%varavg| %total +--------------------------------------------------------------- +Pair | 2.8439 | 2.8439 | 2.8439 | 0.0 | 94.26 +Neigh | 0.11212 | 0.11212 | 0.11212 | 0.0 | 3.72 +Comm | 0.015585 | 0.015585 | 0.015585 | 0.0 | 0.52 +Output | 0.003747 | 0.003747 | 0.003747 | 0.0 | 0.12 +Modify | 0.026097 | 0.026097 | 0.026097 | 0.0 | 0.87 +Other | | 0.01551 | | | 0.51 + +Nlocal: 7200 ave 7200 max 7200 min +Histogram: 1 0 0 0 0 0 0 0 0 0 +Nghost: 6410 ave 6410 max 6410 min +Histogram: 1 0 0 0 0 0 0 0 0 0 +Neighs: 615095 ave 615095 max 615095 min +Histogram: 1 0 0 0 0 0 0 0 0 0 + +Total # of neighbors = 615095 +Ave neighs/atom = 85.429861 +Neighbor list builds = 9 +Dangerous builds = 0 + +#------------------------------------# +# Chunk, Stress, and VCM removal steps +#------------------------------------# + +# 1. Create 20 equispaced bins sliced along the x direction. +# "units reduced" normalizes the distance from 0 to 1 +variable nbins index 20 +variable fraction equal 1.0/v_nbins +variable volfrac equal 1/(vol*${fraction}) +variable volfrac equal 1/(vol*0.05) +compute ch_id all chunk/atom bin/1d x lower ${fraction} units reduced +compute ch_id all chunk/atom bin/1d x lower 0.05 units reduced + +# 2. Calculate temperature bins with VCM aka COM velocities removed. +compute ch_temp_vcm all temp/chunk ch_id com yes + +# 3. Compute per atom stress with VCM removed via temp-ID. +# The velocities from specified temp-ID are used to compute stress +# Stress/atom units are pressure*volume! Optionally handled next step. +compute atom_stress_vcm all stress/atom ch_temp_vcm + +# 4. Divide out bin volume from xx stress component. +variable stress atom -(c_atom_stress_vcm[1])/(vol*${fraction}) +variable stress atom -(c_atom_stress_vcm[1])/(vol*0.05) + +# 5. Sum the per atom stresses in each bin. +compute ch_stress_vcm all reduce/chunk ch_id sum v_stress + +# 6. Average and output to file. +# The average output is every 100 steps with samples collected 20 times with 5 step intervals +# fix ave_stress_vcm all ave/time 5 20 100 c_ch_stress_vcm mode vector file stress_xx.out + +#------------------------------------# + +# Piston compressing along x direction + +region piston block -1 1 INF INF INF INF units lattice +group piston region piston +863 atoms in group piston +fix fix_piston piston move linear 5 0 0 units box # strain rate ~ 8e10 1/s + +thermo_style custom step temp ke pe lx ly lz pxx pyy pzz econserve + +# Atom dump + +# dump atom_dump all atom 50 dump.vcm + +# # Image dumps + +# dump 2 all image 250 image.*.jpg type type # axes yes 0.8 0.02 view 60 -30 +# dump_modify 2 pad 1 + +# # Movie dump + +# dump 3 all movie 125 movie.avi type type # axes yes 0.8 0.02 view 60 -30 +# dump_modify 3 pad 1 + +run 500 +Generated 0 of 0 mixed pair_coeff terms from geometric mixing rule +WARNING: One or more atoms are time integrated more than once (src/modify.cpp:296) +Per MPI rank memory allocation (min/avg/max) = 6.975 | 6.975 | 6.975 Mbytes + Step Temp KinEng PotEng Lx Ly Lz Pxx Pyy Pzz Econserve + 500 295.00895 274.51875 -1982.4224 144.6903 64.3068 64.3068 631.89976 1127.2965 1135.6616 -1707.9036 + 600 357.38902 332.56613 -1951.3422 144.6903 64.3068 64.3068 2236.6706 2003.2726 1943.6815 -1618.7761 + 700 420.30268 391.11005 -1911.8178 144.6903 64.3068 64.3068 3761.5011 3065.4699 3140.3169 -1520.7077 + 800 484.96279 451.27911 -1875.379 144.6903 64.3068 64.3068 5362.254 4174.4201 4166.0818 -1424.0999 + 900 587.78954 546.96391 -1871.217 144.6903 64.3068 64.3068 6481.4714 4875.705 4676.6083 -1324.2531 + 1000 684.07997 636.56636 -1868.1639 144.6903 64.3068 64.3068 7734.6158 5271.3524 5272.1276 -1231.5975 +Loop time of 3.09383 on 1 procs for 500 steps with 7200 atoms + +Performance: 27.927 ns/day, 0.859 hours/ns, 161.612 timesteps/s, 1.164 Matom-step/s +100.0% CPU use with 1 MPI tasks x 1 OpenMP threads + +MPI task timing breakdown: +Section | min time | avg time | max time |%varavg| %total +--------------------------------------------------------------- +Pair | 2.8485 | 2.8485 | 2.8485 | 0.0 | 92.07 +Neigh | 0.18767 | 0.18767 | 0.18767 | 0.0 | 6.07 +Comm | 0.011533 | 0.011533 | 0.011533 | 0.0 | 0.37 +Output | 0.003323 | 0.003323 | 0.003323 | 0.0 | 0.11 +Modify | 0.031777 | 0.031777 | 0.031777 | 0.0 | 1.03 +Other | | 0.01107 | | | 0.36 + +Nlocal: 7200 ave 7200 max 7200 min +Histogram: 1 0 0 0 0 0 0 0 0 0 +Nghost: 6409 ave 6409 max 6409 min +Histogram: 1 0 0 0 0 0 0 0 0 0 +Neighs: 646408 ave 646408 max 646408 min +Histogram: 1 0 0 0 0 0 0 0 0 0 + +Total # of neighbors = 646408 +Ave neighs/atom = 89.778889 +Neighbor list builds = 15 +Dangerous builds = 0 + +unfix fix_piston + +run 1500 +Generated 0 of 0 mixed pair_coeff terms from geometric mixing rule +Per MPI rank memory allocation (min/avg/max) = 6.6 | 6.6 | 6.6 Mbytes + Step Temp KinEng PotEng Lx Ly Lz Pxx Pyy Pzz Econserve + 1000 684.07997 636.56636 -1868.1639 144.6903 64.3068 64.3068 7734.6158 5271.3524 5272.1276 -1231.5975 + 1100 710.19886 660.87113 -1894.0485 144.6903 64.3068 64.3068 8048.3485 5396.6668 5376.5956 -1233.1774 + 1200 717.16487 667.35331 -1901.3849 144.6903 64.3068 64.3068 8009.7984 5634.5121 5349.4113 -1234.0316 + 1300 710.26037 660.92837 -1894.9802 144.6903 64.3068 64.3068 8063.4125 5572.1245 5530.174 -1234.0519 + 1400 715.93921 666.21278 -1898.8885 144.6903 64.3068 64.3068 7752.0927 5293.5463 5322.2312 -1232.6757 + 1500 748.85411 696.84154 -1926.4891 144.6903 64.3068 64.3068 6030.5428 4076.8886 4012.7653 -1229.6475 + 1600 767.98982 714.64815 -1939.8556 144.6903 64.3068 64.3068 4200.3475 2532.5711 2530.5518 -1225.2075 + 1700 757.22042 704.62675 -1925.553 144.6903 64.3068 64.3068 2686.7843 1482.2796 1505.8073 -1220.9262 + 1800 727.30327 676.78754 -1894.6635 144.6903 64.3068 64.3068 1764.2793 781.37451 801.18668 -1217.8759 + 1900 688.82146 640.97853 -1856.5007 144.6903 64.3068 64.3068 1022.805 417.32394 359.74951 -1215.5221 + 2000 655.91228 610.35509 -1823.954 144.6903 64.3068 64.3068 551.98825 -20.148643 -56.976652 -1213.5989 + 2100 620.22468 577.14622 -1789.1761 144.6903 64.3068 64.3068 264.05975 -266.8323 -314.45533 -1212.0299 + 2200 589.13325 548.21428 -1758.9252 144.6903 64.3068 64.3068 41.369707 -533.503 -525.69401 -1210.7109 + 2300 563.20394 524.08593 -1733.6036 144.6903 64.3068 64.3068 -220.99189 -810.90513 -774.65084 -1209.5176 + 2400 540.44236 502.90528 -1711.3384 144.6903 64.3068 64.3068 -358.01508 -962.31635 -977.3253 -1208.4332 + 2500 523.5718 487.20648 -1694.7088 144.6903 64.3068 64.3068 -521.87444 -1152.8386 -1231.7615 -1207.5023 +Loop time of 9.34327 on 1 procs for 1500 steps with 7200 atoms + +Performance: 27.742 ns/day, 0.865 hours/ns, 160.543 timesteps/s, 1.156 Matom-step/s +98.5% CPU use with 1 MPI tasks x 1 OpenMP threads + +MPI task timing breakdown: +Section | min time | avg time | max time |%varavg| %total +--------------------------------------------------------------- +Pair | 8.4692 | 8.4692 | 8.4692 | 0.0 | 90.65 +Neigh | 0.7512 | 0.7512 | 0.7512 | 0.0 | 8.04 +Comm | 0.031189 | 0.031189 | 0.031189 | 0.0 | 0.33 +Output | 0.010584 | 0.010584 | 0.010584 | 0.0 | 0.11 +Modify | 0.053052 | 0.053052 | 0.053052 | 0.0 | 0.57 +Other | | 0.02803 | | | 0.30 + +Nlocal: 7200 ave 7200 max 7200 min +Histogram: 1 0 0 0 0 0 0 0 0 0 +Nghost: 6380 ave 6380 max 6380 min +Histogram: 1 0 0 0 0 0 0 0 0 0 +Neighs: 515773 ave 515773 max 515773 min +Histogram: 1 0 0 0 0 0 0 0 0 0 + +Total # of neighbors = 515773 +Ave neighs/atom = 71.635139 +Neighbor list builds = 57 +Dangerous builds = 0 +Total wall time: 0:00:15 diff --git a/examples/stress_vcm/log.19Nov24.stress_vcm.g++.4 b/examples/stress_vcm/log.19Nov24.stress_vcm.g++.4 new file mode 100644 index 0000000000..d8e7c07536 --- /dev/null +++ b/examples/stress_vcm/log.19Nov24.stress_vcm.g++.4 @@ -0,0 +1,253 @@ +LAMMPS (19 Nov 2024) +OMP_NUM_THREADS environment is not set. Defaulting to 1 thread. (src/comm.cpp:99) + using 1 OpenMP thread(s) per MPI task +# Removing Binned Velocities of Center of Mass (VCM) from Stress + +# This example shows how to remove rigid body motion from +# binned stress calculations. This uses a combination of commands +# from compute chunk/atom, compute temp/chunk, compute +# stress/atom and fix ave/time. We'll show how these commands +# work in the context of a shockwave experiment on a cube of +# atoms. To shock the cube, a rectangular region of atoms is +# frozen, moved into the cube with a constant velocity along the +# x direction, and then unfrozen. As the shockwave begins +# propagating, the body of the cube also moves along the x +# direction. To better understand the stress dynamics of the +# cube we remove the velocity component belonging to the overall +# motion of each bin. + +units metal +boundary p p p +atom_style atomic +lattice fcc 5.3589 +Lattice spacing in x,y,z = 5.3589 5.3589 5.3589 +processors 1 * * + +# Defining regions for box and atoms. +# In this experiment an elongated simulation cell is +# defined in the x direction to allow for non-periodic +# motion of the atoms. + +region box1 block -3 24 0 12 0 12 units lattice +region box2 block 0 12 0 12 0 12 units lattice + +# Creating box and atoms + +create_box 1 box1 +Created orthogonal box = (-16.0767 0 0) to (128.6136 64.3068 64.3068) + 1 by 2 by 2 MPI processor grid +create_atoms 1 region box2 +Created 7200 atoms + using lattice units in orthogonal box = (-16.0767 0 0) to (128.6136 64.3068 64.3068) + create_atoms CPU = 0.001 seconds + +mass 1 40.00 + +# Adding energy to the system + +velocity all create 600.0 9999 + +pair_style lj/cut 10 +pair_coeff 1 1 0.04 3.405 + +# Begin time integration + +timestep 2e-3 + +fix fix_nve all nve + +thermo 100 + +run 500 +Generated 0 of 0 mixed pair_coeff terms from geometric mixing rule +Neighbor list info ... + update: every = 1 steps, delay = 0 steps, check = yes + max neighbors/atom: 2000, page size: 100000 + master list distance cutoff = 12 + ghost atom cutoff = 12 + binsize = 6, bins = 25 11 11 + 1 neighbor lists, perpetual/occasional/extra = 1 0 0 + (1) pair lj/cut/opt, perpetual + attributes: half, newton on + pair build: half/bin/atomonly/newton + stencil: half/bin/3d + bin: standard +Per MPI rank memory allocation (min/avg/max) = 3.662 | 3.662 | 3.662 Mbytes + Step Temp E_pair E_mol TotEng Press + 0 600 -2252.7567 0 -1694.4304 -974.62456 + 100 284.1896 -1976.961 0 -1712.5101 2462.6396 + 200 308.58965 -1998.6349 0 -1711.4787 1789.0033 + 300 300.55093 -1989.9838 0 -1710.308 1545.8576 + 400 297.91491 -1986.2519 0 -1709.029 1247.7121 + 500 294.66041 -1982.1097 0 -1707.9153 961.03073 +Loop time of 0.942408 on 4 procs for 500 steps with 7200 atoms + +Performance: 91.680 ns/day, 0.262 hours/ns, 530.556 timesteps/s, 3.820 Matom-step/s +82.1% CPU use with 4 MPI tasks x 1 OpenMP threads + +MPI task timing breakdown: +Section | min time | avg time | max time |%varavg| %total +--------------------------------------------------------------- +Pair | 0.61287 | 0.63781 | 0.65858 | 2.1 | 67.68 +Neigh | 0.030246 | 0.031529 | 0.034546 | 1.0 | 3.35 +Comm | 0.23074 | 0.25145 | 0.27819 | 3.7 | 26.68 +Output | 0.000282 | 0.0003735 | 0.000463 | 0.0 | 0.04 +Modify | 0.005566 | 0.0057635 | 0.005989 | 0.2 | 0.61 +Other | | 0.01548 | | | 1.64 + +Nlocal: 1800 ave 1814 max 1787 min +Histogram: 1 0 1 0 0 0 0 1 0 1 +Nghost: 3713.5 ave 3727 max 3699 min +Histogram: 1 0 1 0 0 0 0 1 0 1 +Neighs: 153532 ave 154995 max 152312 min +Histogram: 1 0 1 0 0 1 0 0 0 1 + +Total # of neighbors = 614128 +Ave neighs/atom = 85.295556 +Neighbor list builds = 9 +Dangerous builds = 0 + +#------------------------------------# +# Chunk, Stress, and VCM removal steps +#------------------------------------# + +# 1. Create 20 equispaced bins sliced along the x direction. +# "units reduced" normalizes the distance from 0 to 1 +variable nbins index 20 +variable fraction equal 1.0/v_nbins +variable volfrac equal 1/(vol*${fraction}) +variable volfrac equal 1/(vol*0.05) +compute ch_id all chunk/atom bin/1d x lower ${fraction} units reduced +compute ch_id all chunk/atom bin/1d x lower 0.05 units reduced + +# 2. Calculate temperature bins with VCM aka COM velocities removed. +compute ch_temp_vcm all temp/chunk ch_id com yes + +# 3. Compute per atom stress with VCM removed via temp-ID. +# The velocities from specified temp-ID are used to compute stress +# Stress/atom units are pressure*volume! Optionally handled next step. +compute atom_stress_vcm all stress/atom ch_temp_vcm + +# 4. Divide out bin volume from xx stress component. +variable stress atom -(c_atom_stress_vcm[1])/(vol*${fraction}) +variable stress atom -(c_atom_stress_vcm[1])/(vol*0.05) + +# 5. Sum the per atom stresses in each bin. +compute ch_stress_vcm all reduce/chunk ch_id sum v_stress + +# 6. Average and output to file. +# The average output is every 100 steps with samples collected 20 times with 5 step intervals +# fix ave_stress_vcm all ave/time 5 20 100 c_ch_stress_vcm mode vector file stress_xx.out + +#------------------------------------# + +# Piston compressing along x direction + +region piston block -1 1 INF INF INF INF units lattice +group piston region piston +864 atoms in group piston +fix fix_piston piston move linear 5 0 0 units box # strain rate ~ 8e10 1/s + +thermo_style custom step temp ke pe lx ly lz pxx pyy pzz econserve + +# Atom dump + +# dump atom_dump all atom 50 dump.vcm + +# # Image dumps + +# dump 2 all image 250 image.*.jpg type type # axes yes 0.8 0.02 view 60 -30 +# dump_modify 2 pad 1 + +# # Movie dump + +# dump 3 all movie 125 movie.avi type type # axes yes 0.8 0.02 view 60 -30 +# dump_modify 3 pad 1 + +run 500 +Generated 0 of 0 mixed pair_coeff terms from geometric mixing rule +WARNING: One or more atoms are time integrated more than once (src/modify.cpp:296) +Per MPI rank memory allocation (min/avg/max) = 4.916 | 4.916 | 4.916 Mbytes + Step Temp KinEng PotEng Lx Ly Lz Pxx Pyy Pzz Econserve + 500 294.66041 274.19441 -1982.1097 144.6903 64.3068 64.3068 645.25795 1119.5337 1118.3006 -1707.9153 + 600 357.88641 333.02897 -1951.8158 144.6903 64.3068 64.3068 2176.0343 1929.2787 1981.8479 -1618.7869 + 700 418.41159 389.3503 -1912.8337 144.6903 64.3068 64.3068 3702.2875 3043.7607 3081.1607 -1523.4834 + 800 483.71102 450.11428 -1875.7955 144.6903 64.3068 64.3068 5254.3875 4190.9789 4158.3561 -1425.6813 + 900 586.0893 545.38176 -1870.9313 144.6903 64.3068 64.3068 6509.1439 4756.2216 4724.7086 -1325.5495 + 1000 686.32946 638.65962 -1874.811 144.6903 64.3068 64.3068 7515.1606 5193.049 5261.8688 -1236.1514 +Loop time of 0.656417 on 4 procs for 500 steps with 7200 atoms + +Performance: 131.624 ns/day, 0.182 hours/ns, 761.711 timesteps/s, 5.484 Matom-step/s +92.8% CPU use with 4 MPI tasks x 1 OpenMP threads + +MPI task timing breakdown: +Section | min time | avg time | max time |%varavg| %total +--------------------------------------------------------------- +Pair | 0.51672 | 0.52334 | 0.53259 | 0.8 | 79.73 +Neigh | 0.045091 | 0.045915 | 0.047402 | 0.4 | 6.99 +Comm | 0.060735 | 0.071794 | 0.079302 | 2.6 | 10.94 +Output | 0.000208 | 0.000389 | 0.000926 | 0.0 | 0.06 +Modify | 0.006007 | 0.0061595 | 0.00626 | 0.1 | 0.94 +Other | | 0.008815 | | | 1.34 + +Nlocal: 1800 ave 1811 max 1785 min +Histogram: 1 0 0 1 0 0 0 0 0 2 +Nghost: 3713.25 ave 3727 max 3702 min +Histogram: 2 0 0 0 0 0 0 1 0 1 +Neighs: 161477 ave 162958 max 159732 min +Histogram: 1 0 0 0 1 0 0 1 0 1 + +Total # of neighbors = 645909 +Ave neighs/atom = 89.709583 +Neighbor list builds = 15 +Dangerous builds = 0 + +unfix fix_piston + +run 1500 +Generated 0 of 0 mixed pair_coeff terms from geometric mixing rule +Per MPI rank memory allocation (min/avg/max) = 4.541 | 4.541 | 4.541 Mbytes + Step Temp KinEng PotEng Lx Ly Lz Pxx Pyy Pzz Econserve + 1000 686.32946 638.65962 -1874.811 144.6903 64.3068 64.3068 7515.1606 5193.049 5261.8688 -1236.1514 + 1100 709.7333 660.43791 -1898.2844 144.6903 64.3068 64.3068 7932.8638 5334.6171 5364.5335 -1237.8465 + 1200 713.27253 663.73132 -1902.4588 144.6903 64.3068 64.3068 7957.2574 5500.6231 5538.0516 -1238.7275 + 1300 705.44796 656.45022 -1895.1575 144.6903 64.3068 64.3068 7996.7265 5584.6233 5538.2494 -1238.7072 + 1400 711.86463 662.42121 -1899.8416 144.6903 64.3068 64.3068 7674.2462 5292.4915 5294.5366 -1237.4204 + 1500 742.18946 690.63979 -1924.9562 144.6903 64.3068 64.3068 6047.915 4056.6156 4014.4446 -1234.3164 + 1600 762.81764 709.83522 -1939.8563 144.6903 64.3068 64.3068 4185.5873 2530.0572 2576.1943 -1230.0211 + 1700 754.40428 702.00621 -1927.7337 144.6903 64.3068 64.3068 2662.7604 1509.1985 1484.7252 -1225.7275 + 1800 721.03504 670.95468 -1893.5556 144.6903 64.3068 64.3068 1765.8783 835.89765 861.9432 -1222.6009 + 1900 689.64162 641.74172 -1861.8886 144.6903 64.3068 64.3068 941.58148 312.93205 409.79901 -1220.1469 + 2000 650.79664 605.59477 -1823.9889 144.6903 64.3068 64.3068 543.39234 28.48735 80.396505 -1218.3941 + 2100 616.04072 573.25286 -1790.1764 144.6903 64.3068 64.3068 308.16444 -235.20997 -248.22531 -1216.9235 + 2200 587.18712 546.40333 -1761.8878 144.6903 64.3068 64.3068 37.044801 -476.50396 -470.83059 -1215.4845 + 2300 562.84178 523.74892 -1738.2239 144.6903 64.3068 64.3068 -139.28348 -711.17273 -730.80877 -1214.475 + 2400 540.48362 502.94367 -1716.3529 144.6903 64.3068 64.3068 -320.98222 -951.2066 -943.93966 -1213.4093 + 2500 519.80431 483.70067 -1696.1896 144.6903 64.3068 64.3068 -471.61317 -1088.8457 -1131.5396 -1212.4889 +Loop time of 1.97213 on 4 procs for 1500 steps with 7200 atoms + +Performance: 131.431 ns/day, 0.183 hours/ns, 760.598 timesteps/s, 5.476 Matom-step/s +95.3% CPU use with 4 MPI tasks x 1 OpenMP threads + +MPI task timing breakdown: +Section | min time | avg time | max time |%varavg| %total +--------------------------------------------------------------- +Pair | 1.5455 | 1.5599 | 1.5723 | 0.8 | 79.10 +Neigh | 0.16844 | 0.1704 | 0.17237 | 0.4 | 8.64 +Comm | 0.19002 | 0.2047 | 0.22068 | 2.4 | 10.38 +Output | 0.000525 | 0.0006785 | 0.001077 | 0.0 | 0.03 +Modify | 0.012434 | 0.012601 | 0.012777 | 0.1 | 0.64 +Other | | 0.02388 | | | 1.21 + +Nlocal: 1800 ave 1833 max 1776 min +Histogram: 1 0 1 0 1 0 0 0 0 1 +Nghost: 3702 ave 3732 max 3674 min +Histogram: 1 0 0 1 0 0 1 0 0 1 +Neighs: 129380 ave 132578 max 127003 min +Histogram: 1 0 0 2 0 0 0 0 0 1 + +Total # of neighbors = 517520 +Ave neighs/atom = 71.877778 +Neighbor list builds = 54 +Dangerous builds = 0 +Total wall time: 0:00:03 diff --git a/examples/stress_vcm/stress_comparison.19Nov24.png b/examples/stress_vcm/stress_comparison.19Nov24.png new file mode 100644 index 0000000000..c37a3ba345 Binary files /dev/null and b/examples/stress_vcm/stress_comparison.19Nov24.png differ diff --git a/examples/stress_vcm/stress_xx.19Nov24.out b/examples/stress_vcm/stress_xx.19Nov24.out new file mode 100644 index 0000000000..6e65250a96 --- /dev/null +++ b/examples/stress_vcm/stress_xx.19Nov24.out @@ -0,0 +1,423 @@ +# Time-averaged data for fix ave_stress_vcm +# TimeStep Number-of-rows +# Row c_ch_stress_vcm +600 20 +1 0 +2 -142.965 +3 2142.79 +4 12968.3 +5 -336.7 +6 2638.09 +7 4214.83 +8 3187.61 +9 -488.891 +10 -49.3553 +11 151.373 +12 -317.663 +13 0 +14 0 +15 0 +16 0 +17 0 +18 0 +19 0 +20 0 +700 20 +1 0 +2 -14.3195 +3 -1238.9 +4 30664.3 +5 18805.2 +6 498.562 +7 930.874 +8 660.655 +9 -266.903 +10 -317.877 +11 -386.989 +12 -304.697 +13 0 +14 0 +15 0 +16 0 +17 0 +18 0 +19 0 +20 0 +800 20 +1 0 +2 0 +3 -1656.7 +4 30424.3 +5 37003.5 +6 15562.5 +7 -2441.9 +8 -1766.09 +9 272.718 +10 -664.774 +11 -72.6933 +12 -469.765 +13 0 +14 0 +15 0 +16 0 +17 0 +18 0 +19 0 +20 0 +900 20 +1 0 +2 0 +3 -1567.21 +4 24987.6 +5 38068.9 +6 31595 +7 8864.94 +8 -3423.99 +9 -753.063 +10 125.21 +11 -50.4895 +12 -172.14 +13 0 +14 0 +15 0 +16 0 +17 0 +18 0 +19 0 +20 0 +1000 20 +1 0 +2 0 +3 -893.168 +4 15591.6 +5 32690.6 +6 30183 +7 27172 +8 9459.75 +9 -1416.35 +10 -432.731 +11 444.323 +12 -424.357 +13 0 +14 0 +15 0 +16 0 +17 0 +18 0 +19 0 +20 0 +1100 20 +1 0 +2 0 +3 -601.805 +4 8890.79 +5 23345.1 +6 28529.2 +7 29111.9 +8 25846.2 +9 7451.83 +10 -1624.2 +11 320.704 +12 -50.9865 +13 -5.50481 +14 0 +15 0 +16 0 +17 0 +18 0 +19 0 +20 0 +1200 20 +1 0 +2 0 +3 1435.39 +4 8818.29 +5 7129.61 +6 20281.7 +7 28026.1 +8 28327.7 +9 26918.6 +10 8277.12 +11 -249.644 +12 -171.806 +13 -7.19065 +14 0 +15 0 +16 0 +17 0 +18 0 +19 0 +20 0 +1300 20 +1 0 +2 0 +3 -718.118 +4 3021.9 +5 9010.51 +6 9500.87 +7 19432.8 +8 27254.3 +9 28638.5 +10 25568.5 +11 8094.66 +12 -368.293 +13 -2.20997 +14 0 +15 0 +16 0 +17 0 +18 0 +19 0 +20 0 +1400 20 +1 0 +2 0 +3 -650.581 +4 190.19 +5 5465.38 +6 7489.23 +7 7575.16 +8 18433.5 +9 26975.3 +10 28981.5 +11 26987.9 +12 7502.07 +13 0.117312 +14 0 +15 0 +16 0 +17 0 +18 0 +19 0 +20 0 +1500 20 +1 0 +2 0 +3 -619.311 +4 561.257 +5 461.5 +6 4105.68 +7 9272.68 +8 10445.6 +9 18826.1 +10 25434.8 +11 25653.8 +12 10981.2 +13 33.682 +14 0 +15 0 +16 0 +17 0 +18 0 +19 0 +20 0 +1600 20 +1 0 +2 0 +3 -349.345 +4 513.579 +5 -471.384 +6 1257.81 +7 7122.9 +8 8659.35 +9 8452.08 +10 16013.5 +11 17091 +12 5476.24 +13 -136.183 +14 0 +15 0 +16 0 +17 0 +18 0 +19 0 +20 0 +1700 20 +1 0 +2 0 +3 -273.839 +4 -907.407 +5 -272.136 +6 594.363 +7 3302.77 +8 5564.07 +9 8689.92 +10 6446.06 +11 1779.37 +12 338.998 +13 -171.408 +14 -1.21548 +15 0 +16 0 +17 0 +18 0 +19 0 +20 0 +1800 20 +1 0 +2 0 +3 -164.819 +4 383.877 +5 -140.681 +6 -10.0153 +7 907.937 +8 3269.05 +9 5325.22 +10 395.73 +11 -4103.73 +12 -2787.16 +13 -1357.04 +14 -35.2044 +15 0 +16 0 +17 0 +18 0 +19 0 +20 0 +1900 20 +1 0 +2 0 +3 -80.813 +4 334.225 +5 248.55 +6 82.0566 +7 207.763 +8 185.714 +9 -55.8635 +10 -2758.51 +11 -4619.33 +12 -5521.92 +13 -2346.36 +14 -415.324 +15 0 +16 0 +17 0 +18 0 +19 0 +20 0 +2000 20 +1 0 +2 0 +3 -83.1832 +4 264.023 +5 596.087 +6 40.8157 +7 -267.093 +8 -2288.15 +9 -3387.64 +10 -5566.79 +11 -5640.76 +12 -4925.74 +13 -3096.01 +14 -757.817 +15 -1.13042 +16 0 +17 0 +18 0 +19 0 +20 0 +2100 20 +1 0 +2 0 +3 -17.4378 +4 62.1251 +5 740.988 +6 357.467 +7 -1137.61 +8 -4266.83 +9 -4962.9 +10 -5322.45 +11 -5437.58 +12 -4846.56 +13 -3651.28 +14 -1151.01 +15 -28.3074 +16 0 +17 0 +18 0 +19 0 +20 0 +2200 20 +1 0 +2 0 +3 -10.8779 +4 -56.7926 +5 400.261 +6 -568.63 +7 -2193.36 +8 -3856.71 +9 -6603 +10 -5717.11 +11 -4868.64 +12 -4173.5 +13 -3402.64 +14 -1712.44 +15 -80.6771 +16 -0.123189 +17 0 +18 0 +19 0 +20 0 +2300 20 +1 0 +2 0 +3 -22.8402 +4 -44.5496 +5 -365.476 +6 -1285.6 +7 -2887.76 +8 -4022.77 +9 -6280.86 +10 -6055.26 +11 -4921.51 +12 -4445.37 +13 -3531.69 +14 -1360.49 +15 -258.99 +16 0.196931 +17 0 +18 0 +19 0 +20 0 +2400 20 +1 0 +2 0 +3 -0.594396 +4 -148.921 +5 -1118.18 +6 -2071.85 +7 -3989.41 +8 -4567.01 +9 -4939.36 +10 -5170.94 +11 -4922.25 +12 -4587.5 +13 -3748.19 +14 -1785.46 +15 -460.491 +16 2.54038 +17 0 +18 0 +19 0 +20 0 +2500 20 +1 0 +2 0 +3 5.64755 +4 -485.854 +5 -2525.68 +6 -2642.35 +7 -5066.15 +8 -4546.03 +9 -4429.45 +10 -4579.15 +11 -4829.56 +12 -4384.77 +13 -3525.99 +14 -1708.9 +15 -627.176 +16 -23.5581 +17 0 +18 0 +19 0 +20 0 diff --git a/fortran/lammps.f90 b/fortran/lammps.f90 index 1cc6992c27..2cfd4422b0 100644 --- a/fortran/lammps.f90 +++ b/fortran/lammps.f90 @@ -126,6 +126,17 @@ MODULE LIBLAMMPS PROCEDURE :: set_variable => lmp_set_variable PROCEDURE :: set_string_variable => lmp_set_string_variable PROCEDURE :: set_internal_variable => lmp_set_internal_variable + PROCEDURE :: eval => lmp_eval + + PROCEDURE :: clearstep_compute => lmp_clearstep_compute + PROCEDURE, PRIVATE :: lmp_addstep_compute_smallint + PROCEDURE, PRIVATE :: lmp_addstep_compute_bigint + GENERIC :: addstep_compute => lmp_addstep_compute_smallint, lmp_addstep_compute_bigint + PROCEDURE, PRIVATE :: lmp_addstep_compute_all_smallint + PROCEDURE, PRIVATE :: lmp_addstep_compute_all_bigint + GENERIC :: addstep_compute_all => lmp_addstep_compute_all_smallint, & + lmp_addstep_compute_all_bigint + PROCEDURE, PRIVATE :: lmp_gather_atoms_int PROCEDURE, PRIVATE :: lmp_gather_atoms_double GENERIC :: gather_atoms => lmp_gather_atoms_int, & @@ -618,7 +629,32 @@ MODULE LIBLAMMPS INTEGER(c_int) :: lammps_set_internal_variable END FUNCTION lammps_set_internal_variable - SUBROUTINE lammps_gather_atoms(handle, name, type, count, data) BIND(C) + FUNCTION lammps_eval(handle, expr) BIND(C) + IMPORT :: c_ptr, c_double + IMPLICIT NONE + TYPE(c_ptr), VALUE :: handle, expr + REAL(c_double) :: lammps_eval + END FUNCTION lammps_eval + + SUBROUTINE lammps_clearstep_compute(handle) BIND(C) + IMPORT :: c_ptr + IMPLICIT NONE + TYPE(c_ptr), VALUE :: handle + END SUBROUTINE lammps_clearstep_compute + + SUBROUTINE lammps_addstep_compute(handle, step) BIND(C) + IMPORT :: c_ptr + IMPLICIT NONE + TYPE(c_ptr), VALUE :: handle, step + END SUBROUTINE lammps_addstep_compute + + SUBROUTINE lammps_addstep_compute_all(handle, step) BIND(C) + IMPORT :: c_ptr + IMPLICIT NONE + TYPE(c_ptr), VALUE :: handle, step + END SUBROUTINE lammps_addstep_compute_all + + SUBROUTINE lammps_gather_atoms(handle, name, TYPE, count, DATA) BIND(C) IMPORT :: c_int, c_ptr IMPLICIT NONE TYPE(c_ptr), VALUE :: handle, name, data @@ -1812,7 +1848,7 @@ CONTAINS SUBROUTINE lmp_set_internal_variable(self, name, val) CLASS(lammps), INTENT(IN) :: self CHARACTER(LEN=*), INTENT(IN) :: name - REAL(KIND=c_double), INTENT(IN) :: val + REAL(c_double), INTENT(IN) :: val INTEGER :: err TYPE(c_ptr) :: Cname @@ -1826,6 +1862,92 @@ CONTAINS END IF END SUBROUTINE lmp_set_internal_variable + ! equivalent function to lammps_eval + FUNCTION lmp_eval(self, expr) + CLASS(lammps), INTENT(IN) :: self + CHARACTER(LEN=*), INTENT(IN) :: expr + REAL(c_double) :: lmp_eval + TYPE(c_ptr) :: Cexpr + + Cexpr = f2c_string(expr) + lmp_eval = lammps_eval(self%handle, Cexpr) + CALL lammps_free(Cexpr) + END FUNCTION lmp_eval + + ! equivalent subroutine to lammps_clearstep_compute + SUBROUTINE lmp_clearstep_compute(self) + CLASS(lammps), INTENT(IN) :: self + CALL lammps_clearstep_compute(self%handle) + END SUBROUTINE lmp_clearstep_compute + + ! equivalent subroutine to lammps_addstep_compute + SUBROUTINE lmp_addstep_compute_bigint(self, nextstep) + CLASS(lammps), INTENT(IN) :: self + INTEGER(kind=8), INTENT(IN) :: nextstep + INTEGER(c_int), TARGET :: smallstep + INTEGER(c_int64_t), TARGET :: bigstep + TYPE(c_ptr) :: ptrstep + IF (SIZE_BIGINT == 4_c_int) THEN + smallstep = INT(nextstep,kind=c_int) + ptrstep = C_LOC(smallstep) + ELSE + bigstep = nextstep + ptrstep = C_LOC(bigstep) + END IF + CALL lammps_addstep_compute(self%handle, ptrstep) + END SUBROUTINE lmp_addstep_compute_bigint + + ! equivalent subroutine to lammps_addstep_compute + SUBROUTINE lmp_addstep_compute_smallint(self, nextstep) + CLASS(lammps), INTENT(IN) :: self + INTEGER(kind=4), INTENT(IN) :: nextstep + INTEGER(c_int), TARGET :: smallstep + INTEGER(c_int64_t), TARGET :: bigstep + TYPE(c_ptr) :: ptrstep + IF (SIZE_BIGINT == 4_c_int) THEN + smallstep = nextstep + ptrstep = C_LOC(smallstep) + ELSE + bigstep = nextstep + ptrstep = C_LOC(bigstep) + END IF + CALL lammps_addstep_compute(self%handle, ptrstep) + END SUBROUTINE lmp_addstep_compute_smallint + + ! equivalent subroutine to lammps_addstep_compute_all + SUBROUTINE lmp_addstep_compute_all_bigint(self, nextstep) + CLASS(lammps), INTENT(IN) :: self + INTEGER(kind=8), INTENT(IN) :: nextstep + INTEGER(c_int), TARGET :: smallstep + INTEGER(c_int64_t), TARGET :: bigstep + TYPE(c_ptr) :: ptrstep + IF (SIZE_BIGINT == 4_c_int) THEN + smallstep = INT(nextstep,kind=c_int) + ptrstep = C_LOC(smallstep) + ELSE + bigstep = nextstep + ptrstep = C_LOC(bigstep) + END IF + CALL lammps_addstep_compute_all(self%handle, ptrstep) + END SUBROUTINE lmp_addstep_compute_all_bigint + + ! equivalent subroutine to lammps_addstep_compute_all + SUBROUTINE lmp_addstep_compute_all_smallint(self, nextstep) + CLASS(lammps), INTENT(IN) :: self + INTEGER(kind=4), INTENT(IN) :: nextstep + INTEGER(c_int), TARGET :: smallstep + INTEGER(c_int64_t), TARGET :: bigstep + TYPE(c_ptr) :: ptrstep + IF (SIZE_BIGINT == 4_c_int) THEN + smallstep = nextstep + ptrstep = C_LOC(smallstep) + ELSE + bigstep = nextstep + ptrstep = C_LOC(bigstep) + END IF + CALL lammps_addstep_compute_all(self%handle, ptrstep) + END SUBROUTINE lmp_addstep_compute_all_smallint + ! equivalent function to lammps_gather_atoms (for integers) SUBROUTINE lmp_gather_atoms_int(self, name, count, data) CLASS(lammps), INTENT(IN) :: self diff --git a/lib/kokkos/CHANGELOG.md b/lib/kokkos/CHANGELOG.md index 7b1d69e566..84bbd03585 100644 --- a/lib/kokkos/CHANGELOG.md +++ b/lib/kokkos/CHANGELOG.md @@ -1,7 +1,112 @@ # CHANGELOG +## 4.5.01 + +[Full Changelog](https://github.com/kokkos/kokkos/compare/4.5.00...4.5.01) + +### Bug Fixes + +* Fix re-builds after cleaning the binary tree when doing `add_subdirectory` on the Kokkos source [\#7557](https://github.com/kokkos/kokkos/pull/7557) +* Update mdspan to include fix for submdspan and bracket operator with clang 15&16 [\#7559](https://github.com/kokkos/kokkos/pull/7559) +* Fix DynRankView performance regression by re-introducing shortcut operator() impls [\#7606](https://github.com/kokkos/kokkos/pull/7606) +* Add missing MI300A (`GFX942_APU`) option to Makefile build-system + +## 4.5.00 + +[Full Changelog](https://github.com/kokkos/kokkos/compare/4.4.01...4.5.00) + +### Features + +* SYCL backend graduated to production ready +* Introduce new `SequentialHostInit` view allocation property [\#7229](https://github.com/kokkos/kokkos/pull/7229) (backported in 4.4.01) +* Support building with Run-Time Type Information (RTTI) disabled +* Add new `KOKKOS_RELOCATABLE_FUNCTION` function annotation macro [\#5993](https://github.com/kokkos/kokkos/pull/5993) + +### Backend and Architecture Enhancements + +#### CUDA + +* Adding occupancy tuning for CUDA architectures [\#6788](https://github.com/kokkos/kokkos/pull/6788) +* By default disable `cudaMallocAsync` (i.e., revert the change made in version 4.2) [\#7353](https://github.com/kokkos/kokkos/pull/7353) + +#### HIP + +* Add support for AMD Phoenix APUs with Radeon 740M/760M/780M/880M/890M [\#7162](https://github.com/kokkos/kokkos/pull/7162) +* Update maximum waves per CU values for consumer card [\#7347](https://github.com/kokkos/kokkos/pull/7347) +* Check that Kokkos is running on the architecture it was compiled for [\#7379](https://github.com/kokkos/kokkos/pull/7379) +* Add opt-in option to use `hipMallocAsync` instead of `hipMalloc` [\#7324](https://github.com/kokkos/kokkos/pull/7324) +* Introduce new architecture option `AMD_GFX942_APU` for MI300A [\#7462](https://github.com/kokkos/kokkos/pull/7462) + +#### SYCL + +* Move the `SYCL` backend out of the `Experimental` namespace [\#7171](https://github.com/kokkos/kokkos/pull/7171) +* Introduce `KOKKOS_ENABLE_SYCL_RELOCATABLE_DEVICE_CODE` as CMake option [\#5993](https://github.com/kokkos/kokkos/pull/5993) + +#### OpenACC + +* Add support for building with the Clacc compiler [\#7198](https://github.com/kokkos/kokkos/pull/7198) +* Workaround NVHPC collapse clause bug for `MDRangePolicy` [\#7425](https://github.com/kokkos/kokkos/pull/7425) + +#### HPX + +* Implement `Experimental::partition_space` to produce truly independent execution spaces [\#7287](https://github.com/kokkos/kokkos/pull/7287) + +#### Threads + +* Fix compilation for `parallel_reduce` `MDRange` with `Dynamic` scheduling [\#7478](https://github.com/kokkos/kokkos/pull/7478) +* Fix race conditions on ARM architectures [\#7498](https://github.com/kokkos/kokkos/pull/7498) + +#### OpenMP + +* Fix run time behavior when compiling with `-fvisibility-hidden` [\#7284](https://github.com/kokkos/kokkos/pull/7284) (backported in 4.4.01) +* Fix linking with Cray Clang compiler [\#7341](https://github.com/kokkos/kokkos/pull/7341) + +#### Serial + +* Allow `Kokkos_ENABLE_ATOMICS_BYPASS` to skip mutexes to remediate performance regression in 4.4 [\#7369](https://github.com/kokkos/kokkos/pull/7369) + +### General Enhancements + +* Improve `View` initialization/destruction for non-scalar trivial and trivially-destructible types [\#7219](https://github.com/kokkos/kokkos/pull/7219) [\#7225](https://github.com/kokkos/kokkos/pull/7225) +* Add getters for default tile sizes used in `MDRangePolicy` [\#6839](https://github.com/kokkos/kokkos/pull/6839) +* Improve performance of `Kokkos::sort` when `std::sort` is used [\#7264](https://github.com/kokkos/kokkos/pull/7264) +* Add range-based for loop support for `Array` [\#7293](https://github.com/kokkos/kokkos/pull/7293) +* Allow functors as reducers for nested team parallel reduce [\#6921](https://github.com/kokkos/kokkos/pull/6921) +* Avoid making copies of string rvalue reference arguments to `view_alloc()` [\#7364](https://github.com/kokkos/kokkos/pull/7364) +* Add `atomic_{mod,xor,nand,lshift,rshift}` [\#7458](https://github.com/kokkos/kokkos/pull/7458) +* Allow using `SequentialHostInit` with `Kokkos::DualView` [\#7456](https://github.com/kokkos/kokkos/pull/7456) +* Add `Graph::instantiate()` [\#7240](https://github.com/kokkos/kokkos/pull/7240) +* Allow an arbitrary execution space instance to be used in `Kokkos::Graph::submit()` [\#7249](https://github.com/kokkos/kokkos/pull/7249) +* Enable compile-time diagnostic of illegal reduction target for graphs [\#7460](https://github.com/kokkos/kokkos/pull/7460) + +### Build System Changes + +* Make sure backend-specific options such as `IMPL_CUDA_MALLOC_ASYNC` only show when that backend is actually enabled [\#7228](https://github.com/kokkos/kokkos/pull/7228) +* Major refactoring removing `TriBITS` paths [\#6164](https://github.com/kokkos/kokkos/pull/6164) +* Add support for SpacemiT K60 (RISC-V) [\#7160](https://github.com/kokkos/kokkos/pull/7160) + +### Deprecations + +* Deprecate Tasking interface [\#7393](https://github.com/kokkos/kokkos/pull/7393) +* Deprecate `atomic_query_version`, `atomic_assign`, `atomic_compare_exchange_strong`, `atomic_{inc, dec}rement` [\#7458](https://github.com/kokkos/kokkos/pull/7458) +* Deprecate `{OpenMP,HPX}::is_asynchronous()` [\#7322](https://github.com/kokkos/kokkos/pull/7322) + +### Bug Fixes + +* Fix undefined behavior in `BinSort` when sorting within bins on host [\#7223](https://github.com/kokkos/kokkos/pull/7223) +* Using CUDA limits to set extents for blocks, grids [\#7235](https://github.com/kokkos/kokkos/pull/7235) +* Fix `deep_copy (serial_exec, dst, src)` with multiple host backends [\#7245](https://github.com/kokkos/kokkos/pull/7245) +* Skip `RangePolicy` bounds conversion checks if roundtrip convertibility is not provided [\#7172](https://github.com/kokkos/kokkos/pull/7172) +* Allow extracting host and device views from `DualView` with `const` value type [\#7242](https://github.com/kokkos/kokkos/pull/7242) +* Fix `TeamPolicy` array reduction for CUDA and HIP [\#6296](https://github.com/kokkos/kokkos/pull/6296) +* Fix implicit copy assignment operators in few AVX2 masks being deleted [\#7296](https://github.com/kokkos/kokkos/pull/7296) +* Fix configuring without architecture flags for SYCL [\#7303](https://github.com/kokkos/kokkos/pull/7303) +* Set an initial value index during join of `MinLoc`, `MaxLoc` or `MinMaxLoc` [\#7330](https://github.com/kokkos/kokkos/pull/7330) +* Fix storage lifetime of driver for global launch of graph nodes for CUDA and HIP [\#7365](https://github.com/kokkos/kokkos/pull/7365) +* Make `value_type` for `RandomAccessIterator` non-`const` [\#7485](https://github.com/kokkos/kokkos/pull/7485) + ## [4.4.01](https://github.com/kokkos/kokkos/tree/4.4.01) -[Full Changelog](https://github.com/kokkos/kokkos/compare/4.0.00...4.4.01) +[Full Changelog](https://github.com/kokkos/kokkos/compare/4.4.00...4.4.01) ### Features: * Introduce new SequentialHostInit view allocation property [\#7229](https://github.com/kokkos/kokkos/pull/7229) @@ -13,7 +118,7 @@ ### Bug Fixes * OpenMP: Fix issue related to the visibility of an internal symbol with shared libraries that affected `ScatterView` in particular [\#7284](https://github.com/kokkos/kokkos/pull/7284) -* Fix implicit copy assignment operators in few AVX2 masks being deleted [#7296](https://github.com/kokkos/kokkos/pull/7296) +* Fix implicit copy assignment operators in few AVX2 masks being deleted [\#7296](https://github.com/kokkos/kokkos/pull/7296) ## [4.4.00](https://github.com/kokkos/kokkos/tree/4.4.00) [Full Changelog](https://github.com/kokkos/kokkos/compare/4.3.01...4.4.00) @@ -57,6 +162,7 @@ * SIMD: Allow flexible vector width for 32 bit types [\#6802](https://github.com/kokkos/kokkos/pull/6802) * Updates for `Kokkos::Array`: add `kokkos_swap(Array)` specialization [\#6943](https://github.com/kokkos/kokkos/pull/6943), add `Kokkos::to_array` [\#6375](https://github.com/kokkos/kokkos/pull/6375), make `Kokkos::Array` equality-comparable [\#7148](https://github.com/kokkos/kokkos/pull/7148) * Structured binding support for `Kokkos::complex` [\#7040](https://github.com/kokkos/kokkos/pull/7040) +* Introduce `KOKKOS_DEDUCTION_GUIDE` macro to allow for portable user-defined deduction guides [\#6954](https://github.com/kokkos/kokkos/pull/6954) ### Build System Changes * Do not require OpenMP support for languages other than CXX [\#6965](https://github.com/kokkos/kokkos/pull/6965) @@ -1388,7 +1494,7 @@ **Closed issues:** - Silent error (Validate storage level arg to set_scratch_size) [\#3097](https://github.com/kokkos/kokkos/issues/3097) -- Remove KOKKKOS\_ENABLE\_PROFILING Option [\#3095](https://github.com/kokkos/kokkos/issues/3095) +- Remove KOKKOS\_ENABLE\_PROFILING Option [\#3095](https://github.com/kokkos/kokkos/issues/3095) - Cuda 11 -\> allow C++17 [\#3083](https://github.com/kokkos/kokkos/issues/3083) - In source build failure not explained [\#3081](https://github.com/kokkos/kokkos/issues/3081) - Allow naming of Views for initialization kernel [\#3070](https://github.com/kokkos/kokkos/issues/3070) diff --git a/lib/kokkos/CMakeLists.txt b/lib/kokkos/CMakeLists.txt index 736cbac218..6a70bea149 100644 --- a/lib/kokkos/CMakeLists.txt +++ b/lib/kokkos/CMakeLists.txt @@ -1,12 +1,11 @@ cmake_minimum_required(VERSION 3.16 FATAL_ERROR) # Disable in-source builds to prevent source tree corruption. -if( "${CMAKE_SOURCE_DIR}" STREQUAL "${CMAKE_BINARY_DIR}" ) - message( FATAL_ERROR "FATAL: In-source builds are not allowed. You should create a separate directory for build files and delete CMakeCache.txt." ) -endif() - -if (COMMAND TRIBITS_PACKAGE) - TRIBITS_PACKAGE(Kokkos) +if("${CMAKE_SOURCE_DIR}" STREQUAL "${CMAKE_BINARY_DIR}") + message( + FATAL_ERROR + "FATAL: In-source builds are not allowed. You should create a separate directory for build files and delete CMakeCache.txt." + ) endif() # We want to determine if options are given with the wrong case @@ -15,142 +14,141 @@ endif() # form a list of all the given variables. If it begins with any # case of KoKkOS, we add it to the list. -GET_CMAKE_PROPERTY(_variableNames VARIABLES) -SET(KOKKOS_GIVEN_VARIABLES) -FOREACH (var ${_variableNames}) - STRING(TOUPPER ${var} UC_VAR) - STRING(FIND ${UC_VAR} KOKKOS IDX) - IF (${IDX} EQUAL 0) - LIST(APPEND KOKKOS_GIVEN_VARIABLES ${var}) - ENDIF() -ENDFOREACH() +get_cmake_property(_variableNames VARIABLES) +set(KOKKOS_GIVEN_VARIABLES) +foreach(var ${_variableNames}) + string(TOUPPER ${var} UC_VAR) + string(FIND ${UC_VAR} KOKKOS IDX) + if(${IDX} EQUAL 0) + list(APPEND KOKKOS_GIVEN_VARIABLES ${var}) + endif() +endforeach() # Basic initialization (Used in KOKKOS_SETTINGS) -SET(Kokkos_SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}) -SET(KOKKOS_SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}) -SET(KOKKOS_SRC_PATH ${Kokkos_SOURCE_DIR}) -SET(KOKKOS_PATH ${Kokkos_SOURCE_DIR}) -SET(KOKKOS_TOP_BUILD_DIR ${CMAKE_CURRENT_BINARY_DIR}) +set(Kokkos_SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}) +set(KOKKOS_SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}) +set(KOKKOS_SRC_PATH ${Kokkos_SOURCE_DIR}) +set(KOKKOS_PATH ${Kokkos_SOURCE_DIR}) +set(KOKKOS_TOP_BUILD_DIR ${CMAKE_CURRENT_BINARY_DIR}) + +set(PACKAGE_NAME Kokkos) +set(PACKAGE_SOURCE_DIR "${CMAKE_CURRENT_SOURCE_DIR}") -# Is this a build as part of Trilinos? -IF(COMMAND TRIBITS_PACKAGE_DECL) - SET(KOKKOS_HAS_TRILINOS ON) -ELSE() - SET(KOKKOS_HAS_TRILINOS OFF) - SET(PACKAGE_NAME Kokkos) - SET(PACKAGE_SOURCE_DIR "${CMAKE_CURRENT_SOURCE_DIR}") -ENDIF() # Is this build a subdirectory of another project -GET_DIRECTORY_PROPERTY(HAS_PARENT PARENT_DIRECTORY) +get_directory_property(HAS_PARENT PARENT_DIRECTORY) +include(${KOKKOS_SRC_PATH}/cmake/kokkos_functions.cmake) +include(${KOKKOS_SRC_PATH}/cmake/kokkos_pick_cxx_std.cmake) -INCLUDE(${KOKKOS_SRC_PATH}/cmake/kokkos_functions.cmake) -INCLUDE(${KOKKOS_SRC_PATH}/cmake/kokkos_pick_cxx_std.cmake) - -SET(KOKKOS_ENABLED_OPTIONS) #exported in config file -SET(KOKKOS_ENABLED_DEVICES) #exported in config file -SET(KOKKOS_ENABLED_TPLS) #exported in config file -SET(KOKKOS_ENABLED_ARCH_LIST) #exported in config file +set(KOKKOS_ENABLED_OPTIONS) #exported in config file +set(KOKKOS_ENABLED_DEVICES) #exported in config file +set(KOKKOS_ENABLED_TPLS) #exported in config file +set(KOKKOS_ENABLED_ARCH_LIST) #exported in config file #These are helper flags used for sanity checks during config #Certain features should depend on other features being configured first -SET(KOKKOS_CFG_DAG_NONE On) #sentinel to indicate no dependencies -SET(KOKKOS_CFG_DAG_DEVICES_DONE Off) -SET(KOKKOS_CFG_DAG_OPTIONS_DONE Off) -SET(KOKKOS_CFG_DAG_ARCH_DONE Off) -SET(KOKKOS_CFG_DAG_CXX_STD_DONE Off) -SET(KOKKOS_CFG_DAG_COMPILER_ID_DONE Off) -FUNCTION(KOKKOS_CFG_DEPENDS SUCCESSOR PRECURSOR) - SET(PRE_FLAG KOKKOS_CFG_DAG_${PRECURSOR}) - SET(POST_FLAG KOKKOS_CFG_DAG_${SUCCESSOR}) - IF (NOT ${PRE_FLAG}) - MESSAGE(FATAL_ERROR "Bad CMake refactor: feature ${SUCCESSOR} cannot be configured until ${PRECURSOR} is configured") - ENDIF() - GLOBAL_SET(${POST_FLAG} On) -ENDFUNCTION() +set(KOKKOS_CFG_DAG_NONE On) #sentinel to indicate no dependencies +set(KOKKOS_CFG_DAG_DEVICES_DONE Off) +set(KOKKOS_CFG_DAG_OPTIONS_DONE Off) +set(KOKKOS_CFG_DAG_ARCH_DONE Off) +set(KOKKOS_CFG_DAG_CXX_STD_DONE Off) +set(KOKKOS_CFG_DAG_COMPILER_ID_DONE Off) +function(KOKKOS_CFG_DEPENDS SUCCESSOR PRECURSOR) + set(PRE_FLAG KOKKOS_CFG_DAG_${PRECURSOR}) + set(POST_FLAG KOKKOS_CFG_DAG_${SUCCESSOR}) + if(NOT ${PRE_FLAG}) + message( + FATAL_ERROR "Bad CMake refactor: feature ${SUCCESSOR} cannot be configured until ${PRECURSOR} is configured" + ) + endif() + global_set(${POST_FLAG} On) +endfunction() +list(APPEND CMAKE_MODULE_PATH cmake/Modules) -LIST(APPEND CMAKE_MODULE_PATH cmake/Modules) +set(CMAKE_DISABLE_SOURCE_CHANGES ON) +set(CMAKE_DISABLE_IN_SOURCE_BUILD ON) -IF(NOT KOKKOS_HAS_TRILINOS) - set(CMAKE_DISABLE_SOURCE_CHANGES ON) - set(CMAKE_DISABLE_IN_SOURCE_BUILD ON) +# What language are we compiling Kokkos as +# downstream dependencies need to match this! +set(KOKKOS_COMPILE_LANGUAGE CXX) +# use lower case here since we didn't parse options yet +if(Kokkos_ENABLE_COMPILE_AS_CMAKE_LANGUAGE AND Kokkos_ENABLE_CUDA) - # What language are we compiling Kokkos as - # downstream dependencies need to match this! - SET(KOKKOS_COMPILE_LANGUAGE CXX) - # use lower case here since we didn't parse options yet - IF (Kokkos_ENABLE_COMPILE_AS_CMAKE_LANGUAGE AND Kokkos_ENABLE_CUDA) + # Without this as a language for the package we would get a C++ compiler enabled. + # but we still need a C++ compiler even if we build all our cpp files as CUDA only + # because otherwise the C++ features don't work etc. + # This is just the rather odd way CMake does this, since CUDA doesn't imply C++ even + # though it is a C++ extension ... (but I guess it didn't use to be back in CUDA 4 or 5 + # days. + set(KOKKOS_INTERNAL_EXTRA_COMPILE_LANGUAGE CXX) - # Without this as a language for the package we would get a C++ compiler enabled. - # but we still need a C++ compiler even if we build all our cpp files as CUDA only - # because otherwise the C++ features don't work etc. - # This is just the rather odd way CMake does this, since CUDA doesn't imply C++ even - # though it is a C++ extension ... (but I guess it didn't use to be back in CUDA 4 or 5 - # days. - SET(KOKKOS_INTERNAL_EXTRA_COMPILE_LANGUAGE CXX) + set(KOKKOS_COMPILE_LANGUAGE CUDA) +endif() +# use lower case here since we haven't parsed options yet +if(Kokkos_ENABLE_COMPILE_AS_CMAKE_LANGUAGE AND Kokkos_ENABLE_HIP) - SET(KOKKOS_COMPILE_LANGUAGE CUDA) - ENDIF() - # use lower case here since we haven't parsed options yet - IF (Kokkos_ENABLE_COMPILE_AS_CMAKE_LANGUAGE AND Kokkos_ENABLE_HIP) + # Without this as a language for the package we would get a C++ compiler enabled. + # but we still need a C++ compiler even if we build all our cpp files as HIP only + # because otherwise the C++ features don't work etc. + set(KOKKOS_INTERNAL_EXTRA_COMPILE_LANGUAGE CXX) - # Without this as a language for the package we would get a C++ compiler enabled. - # but we still need a C++ compiler even if we build all our cpp files as HIP only - # because otherwise the C++ features don't work etc. - SET(KOKKOS_INTERNAL_EXTRA_COMPILE_LANGUAGE CXX) + set(KOKKOS_COMPILE_LANGUAGE HIP) +endif() - SET(KOKKOS_COMPILE_LANGUAGE HIP) - ENDIF() +if(Spack_WORKAROUND) + if(Kokkos_ENABLE_COMPILE_AS_CMAKE_LANGUAGE) + message(FATAL_ERROR "Can't currently use Kokkos_ENABLE_COMPILER_AS_CMAKE_LANGUAGE in a spack installation!") + endif() - IF (Spack_WORKAROUND) - IF (Kokkos_ENABLE_COMPILE_AS_CMAKE_LANGUAGE) - MESSAGE(FATAL_ERROR "Can't currently use Kokkos_ENABLE_COMPILER_AS_CMAKE_LANGUAGE in a spack installation!") - ENDIF() - - #if we are explicitly using Spack for development, - #nuke the Spack compiler - SET(SPACK_CXX $ENV{SPACK_CXX}) - IF(SPACK_CXX) - SET(CMAKE_CXX_COMPILER ${SPACK_CXX} CACHE STRING "the C++ compiler" FORCE) - SET(ENV{CXX} ${SPACK_CXX}) - ENDIF() - ENDIF() - # Always call the project command to define Kokkos_ variables - # and to make sure that C++ is an enabled language - PROJECT(Kokkos ${KOKKOS_COMPILE_LANGUAGE} ${KOKKOS_INTERNAL_EXTRA_COMPILE_LANGUAGE}) - IF(NOT HAS_PARENT) - IF (NOT CMAKE_BUILD_TYPE) - SET(DEFAULT_BUILD_TYPE "RelWithDebInfo") - MESSAGE(STATUS "Setting build type to '${DEFAULT_BUILD_TYPE}' as none was specified.") - SET(CMAKE_BUILD_TYPE "${DEFAULT_BUILD_TYPE}" CACHE STRING - "Choose the type of build, options are: Debug, Release, RelWithDebInfo and MinSizeRel." - FORCE) - ENDIF() - ENDIF() -ELSE() - SET(KOKKOS_COMPILE_LANGUAGE CXX) -ENDIF() - -IF (NOT CMAKE_SIZEOF_VOID_P) - STRING(FIND ${CMAKE_CXX_COMPILER} nvcc_wrapper FIND_IDX) - IF (NOT FIND_IDX STREQUAL -1) - MESSAGE(FATAL_ERROR "Kokkos did not configure correctly and failed to validate compiler. The most likely cause is CUDA linkage using nvcc_wrapper. Please ensure your CUDA environment is correctly configured.") - ELSE() - MESSAGE(FATAL_ERROR "Kokkos did not configure correctly and failed to validate compiler. The most likely cause is linkage errors during CMake compiler validation. Please consult the CMake error log shown below for the exact error during compiler validation") - ENDIF() -ELSEIF (NOT CMAKE_SIZEOF_VOID_P EQUAL 8) - IF(CMAKE_SIZEOF_VOID_P EQUAL 4) - MESSAGE(WARNING "32-bit builds are experimental and not officially supported.") - SET(KOKKOS_IMPL_32BIT ON) - ELSE() - MESSAGE(FATAL_ERROR "Kokkos assumes a 64-bit build, i.e., 8-byte pointers, but found ${CMAKE_SIZEOF_VOID_P}-byte pointers instead;") - ENDIF() -ENDIF() + #if we are explicitly using Spack for development, + #nuke the Spack compiler + set(SPACK_CXX $ENV{SPACK_CXX}) + if(SPACK_CXX) + set(CMAKE_CXX_COMPILER ${SPACK_CXX} CACHE STRING "the C++ compiler" FORCE) + set(ENV{CXX} ${SPACK_CXX}) + endif() +endif() +# Always call the project command to define Kokkos_ variables +# and to make sure that C++ is an enabled language +project(Kokkos ${KOKKOS_COMPILE_LANGUAGE} ${KOKKOS_INTERNAL_EXTRA_COMPILE_LANGUAGE}) +if(NOT HAS_PARENT) + if(NOT CMAKE_BUILD_TYPE) + set(DEFAULT_BUILD_TYPE "RelWithDebInfo") + message(STATUS "Setting build type to '${DEFAULT_BUILD_TYPE}' as none was specified.") + set(CMAKE_BUILD_TYPE "${DEFAULT_BUILD_TYPE}" + CACHE STRING "Choose the type of build, options are: Debug, Release, RelWithDebInfo and MinSizeRel." FORCE + ) + endif() +endif() +if(NOT CMAKE_SIZEOF_VOID_P) + string(FIND ${CMAKE_CXX_COMPILER} nvcc_wrapper FIND_IDX) + if(NOT FIND_IDX STREQUAL -1) + message( + FATAL_ERROR + "Kokkos did not configure correctly and failed to validate compiler. The most likely cause is CUDA linkage using nvcc_wrapper. Please ensure your CUDA environment is correctly configured." + ) + else() + message( + FATAL_ERROR + "Kokkos did not configure correctly and failed to validate compiler. The most likely cause is linkage errors during CMake compiler validation. Please consult the CMake error log shown below for the exact error during compiler validation" + ) + endif() +elseif(NOT CMAKE_SIZEOF_VOID_P EQUAL 8) + if(CMAKE_SIZEOF_VOID_P EQUAL 4) + message(WARNING "32-bit builds are experimental and not officially supported.") + set(KOKKOS_IMPL_32BIT ON) + else() + message( + FATAL_ERROR + "Kokkos assumes a 64-bit build, i.e., 8-byte pointers, but found ${CMAKE_SIZEOF_VOID_P}-byte pointers instead;" + ) + endif() +endif() set(Kokkos_VERSION_MAJOR 4) -set(Kokkos_VERSION_MINOR 4) +set(Kokkos_VERSION_MINOR 5) set(Kokkos_VERSION_PATCH 1) set(Kokkos_VERSION "${Kokkos_VERSION_MAJOR}.${Kokkos_VERSION_MINOR}.${Kokkos_VERSION_PATCH}") message(STATUS "Kokkos version: ${Kokkos_VERSION}") @@ -164,58 +162,54 @@ math(EXPR KOKKOS_VERSION_PATCH "${KOKKOS_VERSION} % 100") # Load either the real TriBITS or a TriBITS wrapper # for certain utility functions that are universal (like GLOBAL_SET) -INCLUDE(${KOKKOS_SRC_PATH}/cmake/fake_tribits.cmake) +include(${KOKKOS_SRC_PATH}/cmake/fake_tribits.cmake) -IF (Kokkos_ENABLE_CUDA) +if(Kokkos_ENABLE_CUDA) # If we are building CUDA, we have tricked CMake because we declare a CXX project # If the default C++ standard for a given compiler matches the requested # standard, then CMake just omits the -std flag in later versions of CMake # This breaks CUDA compilation (CUDA compiler can have a different default # -std then the underlying host compiler by itself). Setting this variable # forces CMake to always add the -std flag even if it thinks it doesn't need it - GLOBAL_SET(CMAKE_CXX_STANDARD_DEFAULT 98) -ENDIF() + global_set(CMAKE_CXX_STANDARD_DEFAULT 98) +endif() # These are the variables we will append to as we go # I really wish these were regular variables # but scoping issues can make it difficult -GLOBAL_SET(KOKKOS_COMPILE_OPTIONS) -GLOBAL_SET(KOKKOS_LINK_OPTIONS) -GLOBAL_SET(KOKKOS_AMDGPU_OPTIONS) -GLOBAL_SET(KOKKOS_CUDA_OPTIONS) -GLOBAL_SET(KOKKOS_CUDAFE_OPTIONS) -GLOBAL_SET(KOKKOS_XCOMPILER_OPTIONS) +global_set(KOKKOS_COMPILE_OPTIONS) +global_set(KOKKOS_LINK_OPTIONS) +global_set(KOKKOS_AMDGPU_OPTIONS) +global_set(KOKKOS_CUDA_OPTIONS) +global_set(KOKKOS_CUDAFE_OPTIONS) +global_set(KOKKOS_XCOMPILER_OPTIONS) # We need to append text here for making sure TPLs # we import are available for an installed Kokkos -GLOBAL_SET(KOKKOS_TPL_EXPORTS) +global_set(KOKKOS_TPL_EXPORTS) # KOKKOS_DEPENDENCE is used by kokkos_launch_compiler -GLOBAL_SET(KOKKOS_COMPILE_DEFINITIONS KOKKOS_DEPENDENCE) +global_set(KOKKOS_COMPILE_DEFINITIONS KOKKOS_DEPENDENCE) # MSVC never goes through kokkos_launch_compiler -IF(NOT MSVC) - GLOBAL_APPEND(KOKKOS_LINK_OPTIONS -DKOKKOS_DEPENDENCE) -ENDIF() +if(NOT MSVC) + global_append(KOKKOS_LINK_OPTIONS -DKOKKOS_DEPENDENCE) +endif() -IF(Kokkos_ENABLE_TESTS AND NOT KOKKOS_HAS_TRILINOS) +include(${CMAKE_CURRENT_SOURCE_DIR}/cmake/kokkos_configure_trilinos.cmake) + +if(Kokkos_ENABLE_TESTS) find_package(GTest QUIET) -ENDIF() +endif() # Include a set of Kokkos-specific wrapper functions that # will either call raw CMake or TriBITS # These are functions like KOKKOS_INCLUDE_DIRECTORIES -INCLUDE(${KOKKOS_SRC_PATH}/cmake/kokkos_tribits.cmake) - +include(${KOKKOS_SRC_PATH}/cmake/kokkos_tribits.cmake) # Check the environment and set certain variables # to allow platform-specific checks -INCLUDE(${KOKKOS_SRC_PATH}/cmake/kokkos_check_env.cmake) +include(${KOKKOS_SRC_PATH}/cmake/kokkos_check_env.cmake) -IF(NOT KOKKOS_HAS_TRILINOS) - # This does not work in Trilinos and we simply don't care - # to fix it for Trilinos - # Gather information about the runtime environment - INCLUDE(${KOKKOS_SRC_PATH}/cmake/build_env_info.cmake) - check_git_setup() -ENDIF() +include(${KOKKOS_SRC_PATH}/cmake/build_env_info.cmake) +check_git_setup() # The build environment setup goes in the following steps # 1) Check all the enable options. This includes checking Kokkos_DEVICES @@ -223,102 +217,54 @@ ENDIF() # 3) Check the CXX standard and select important CXX flags # 4) Check for any third-party libraries (TPLs) like hwloc # 5) Check if optimizing for a particular architecture and add arch-specific flags -KOKKOS_SETUP_BUILD_ENVIRONMENT() +kokkos_setup_build_environment() # Finish off the build # 6) Recurse into subdirectories and configure individual libraries # 7) Export and install targets -OPTION(BUILD_SHARED_LIBS "Build shared libraries" OFF) +option(BUILD_SHARED_LIBS "Build shared libraries" OFF) -SET(KOKKOS_COMPONENT_LIBRARIES kokkoscore kokkoscontainers kokkosalgorithms kokkossimd) -SET_PROPERTY(GLOBAL PROPERTY KOKKOS_INT_LIBRARIES kokkos ${KOKKOS_COMPONENT_LIBRARIES}) +set(KOKKOS_COMPONENT_LIBRARIES kokkoscore kokkoscontainers kokkosalgorithms kokkossimd) +set_property(GLOBAL PROPERTY KOKKOS_INT_LIBRARIES kokkos ${KOKKOS_COMPONENT_LIBRARIES}) -IF (KOKKOS_HAS_TRILINOS) - SET(TRILINOS_INCDIR ${${PROJECT_NAME}_INSTALL_INCLUDE_DIR}) - SET(KOKKOS_HEADER_DIR ${TRILINOS_INCDIR}) - SET(KOKKOS_IS_SUBDIRECTORY TRUE) -ELSEIF(HAS_PARENT) - SET(KOKKOS_HEADER_DIR "include/kokkos") - SET(KOKKOS_IS_SUBDIRECTORY TRUE) -ELSE() - SET(KOKKOS_HEADER_DIR "${CMAKE_INSTALL_INCLUDEDIR}") - SET(KOKKOS_IS_SUBDIRECTORY FALSE) -ENDIF() +if(HAS_PARENT) + set(KOKKOS_HEADER_DIR "include/kokkos") + set(KOKKOS_IS_SUBDIRECTORY TRUE) +else() + set(KOKKOS_HEADER_DIR "${CMAKE_INSTALL_INCLUDEDIR}") + set(KOKKOS_IS_SUBDIRECTORY FALSE) +endif() #------------------------------------------------------------------------------ # # A) Forward declare the package so that certain options are also defined for # subpackages -## This restores the old behavior of ProjectCompilerPostConfig.cmake -# We must do this before KOKKOS_PACKAGE_DECL -IF (KOKKOS_HAS_TRILINOS) - # Overwrite the old flags at the top-level - # Because Tribits doesn't use lists, it uses spaces for the list of CXX flags - # we have to match the annoying behavior, also we have to preserve quotes - # which needs another workaround. - SET(KOKKOS_COMPILE_OPTIONS_TMP) - IF (KOKKOS_ENABLE_HIP) - LIST(APPEND KOKKOS_COMPILE_OPTIONS ${KOKKOS_AMDGPU_OPTIONS}) - ENDIF() - FOREACH(OPTION ${KOKKOS_COMPILE_OPTIONS}) - STRING(FIND "${OPTION}" " " OPTION_HAS_WHITESPACE) - IF(OPTION_HAS_WHITESPACE EQUAL -1) - LIST(APPEND KOKKOS_COMPILE_OPTIONS_TMP "${OPTION}") - ELSE() - LIST(APPEND KOKKOS_COMPILE_OPTIONS_TMP "\"${OPTION}\"") - ENDIF() - ENDFOREACH() - STRING(REPLACE ";" " " KOKKOSCORE_COMPILE_OPTIONS "${KOKKOS_COMPILE_OPTIONS_TMP}") - LIST(APPEND KOKKOS_ALL_COMPILE_OPTIONS ${KOKKOS_COMPILE_OPTIONS}) - IF (KOKKOS_ENABLE_CUDA) - LIST(APPEND KOKKOS_ALL_COMPILE_OPTIONS ${KOKKOS_CUDA_OPTIONS}) - ENDIF() - FOREACH(XCOMP_FLAG ${KOKKOS_XCOMPILER_OPTIONS}) - SET(KOKKOSCORE_XCOMPILER_OPTIONS "${KOKKOSCORE_XCOMPILER_OPTIONS} -Xcompiler ${XCOMP_FLAG}") - LIST(APPEND KOKKOS_ALL_COMPILE_OPTIONS -Xcompiler ${XCOMP_FLAG}) - ENDFOREACH() - IF (KOKKOS_ENABLE_CUDA) - STRING(REPLACE ";" " " KOKKOSCORE_CUDA_OPTIONS "${KOKKOS_CUDA_OPTIONS}") - FOREACH(CUDAFE_FLAG ${KOKKOS_CUDAFE_OPTIONS}) - SET(KOKKOSCORE_CUDAFE_OPTIONS "${KOKKOSCORE_CUDAFE_OPTIONS} -Xcudafe ${CUDAFE_FLAG}") - LIST(APPEND KOKKOS_ALL_COMPILE_OPTIONS -Xcudafe ${CUDAFE_FLAG}) - ENDFOREACH() - ENDIF() - #These flags get set up in KOKKOS_PACKAGE_DECL, which means they - #must be configured before KOKKOS_PACKAGE_DECL - SET(KOKKOS_ALL_COMPILE_OPTIONS - $<$:${KOKKOS_ALL_COMPILE_OPTIONS}>) -ENDIF() - - #------------------------------------------------------------------------------ # # D) Process the subpackages (subdirectories) for Kokkos # -KOKKOS_PROCESS_SUBPACKAGES() - +kokkos_process_subpackages() #------------------------------------------------------------------------------ # # E) If Kokkos itself is enabled, process the Kokkos package # -KOKKOS_PACKAGE_POSTPROCESS() -KOKKOS_CONFIGURE_CORE() +kokkos_configure_core() -IF (NOT KOKKOS_HAS_TRILINOS AND NOT Kokkos_INSTALL_TESTING) - ADD_LIBRARY(kokkos INTERFACE) +if(NOT Kokkos_INSTALL_TESTING) + add_library(kokkos INTERFACE) #Make sure in-tree projects can reference this as Kokkos:: #to match the installed target names - ADD_LIBRARY(Kokkos::kokkos ALIAS kokkos) + add_library(Kokkos::kokkos ALIAS kokkos) # all_libs target is required for TriBITS-compliance - ADD_LIBRARY(Kokkos::all_libs ALIAS kokkos) - TARGET_LINK_LIBRARIES(kokkos INTERFACE ${KOKKOS_COMPONENT_LIBRARIES}) - KOKKOS_INTERNAL_ADD_LIBRARY_INSTALL(kokkos) -ENDIF() -INCLUDE(${KOKKOS_SRC_PATH}/cmake/kokkos_install.cmake) + add_library(Kokkos::all_libs ALIAS kokkos) + target_link_libraries(kokkos INTERFACE ${KOKKOS_COMPONENT_LIBRARIES}) + kokkos_internal_add_library_install(kokkos) +endif() +include(${KOKKOS_SRC_PATH}/cmake/kokkos_install.cmake) # nvcc_wrapper is Kokkos' wrapper for NVIDIA's NVCC CUDA compiler. # Kokkos needs nvcc_wrapper in order to build. Other libraries and @@ -327,16 +273,15 @@ INCLUDE(${KOKKOS_SRC_PATH}/cmake/kokkos_install.cmake) # as relative to ${CMAKE_INSTALL_PATH}. # KOKKOS_INSTALL_ADDITIONAL_FILES will install nvcc wrapper and other generated # files -KOKKOS_INSTALL_ADDITIONAL_FILES() - +kokkos_install_additional_files() # Finally - if we are a subproject - make sure the enabled devices are visible -IF (HAS_PARENT) - FOREACH(DEV Kokkos_ENABLED_DEVICES) +if(HAS_PARENT) + foreach(DEV Kokkos_ENABLED_DEVICES) #I would much rather not make these cache variables or global properties, but I can't #make any guarantees on whether PARENT_SCOPE is good enough to make #these variables visible where I need them - SET(Kokkos_ENABLE_${DEV} ON PARENT_SCOPE) - SET_PROPERTY(GLOBAL PROPERTY Kokkos_ENABLE_${DEV} ON) - ENDFOREACH() -ENDIF() + set(Kokkos_ENABLE_${DEV} ON PARENT_SCOPE) + set_property(GLOBAL PROPERTY Kokkos_ENABLE_${DEV} ON) + endforeach() +endif() diff --git a/lib/kokkos/CONTRIBUTING.md b/lib/kokkos/CONTRIBUTING.md index b4f3057cef..e97f8c4d89 100644 --- a/lib/kokkos/CONTRIBUTING.md +++ b/lib/kokkos/CONTRIBUTING.md @@ -7,6 +7,8 @@ We actively welcome pull requests. 3. If you've changed APIs, update the documentation. 4. Ensure the test suite passes. +Before sending your patch for review, please try to ensure that it is formatted properly. We use clang-format version 16 for this. + ## Issues We use GitHub issues to track public bugs. Please ensure your description is clear and has sufficient instructions to be able to reproduce the issue. diff --git a/lib/kokkos/HOW_TO_SNAPSHOT b/lib/kokkos/HOW_TO_SNAPSHOT deleted file mode 100644 index ad3f78efb4..0000000000 --- a/lib/kokkos/HOW_TO_SNAPSHOT +++ /dev/null @@ -1,73 +0,0 @@ - -Developers of Kokkos (those who commit modifications to Kokkos) -must maintain the snapshot of Kokkos in the Trilinos repository. - -This file contains instructions for how to -snapshot Kokkos from github.com/kokkos to Trilinos. - ------------------------------------------------------------------------- -*** EVERYTHING GOES RIGHT WORKFLOW *** - -1) Given a 'git clone' of Kokkos and of Trilinos repositories. -1.1) Let ${KOKKOS} be the absolute path to the Kokkos clone. - This path *must* terminate with the directory name 'kokkos'; - e.g., ${HOME}/kokkos . -1.2) Let ${TRILINOS} be the absolute path to the Trilinos directory. - -2) Given that the Kokkos build & test is clean and - changes are committed to the Kokkos clone. - -3) Snapshot the current commit in the Kokkos clone into the Trilinos clone. - This overwrites ${TRILINOS}/packages/kokkos with the content of ${KOKKOS}: - ${KOKKOS}/scripts/snapshot.py --verbose ${KOKKOS} ${TRILINOS}/packages - -4) Verify the snapshot commit happened as expected - cd ${TRILINOS}/packages/kokkos - git log -1 --name-only - -5) Modify, build, and test Trilinos with the Kokkos snapshot. - -6) Given that that the Trilinos build & test is clean and - changes are committed to the Trilinos clone. - -7) Attempt push to the Kokkos repository. - If push fails then you must 'remove the Kokkos snapshot' - from your Trilinos clone. - See below. - -8) Attempt to push to the Trilinos repository. - If updating for a failed push requires you to change Kokkos you must - 'remove the Kokkos snapshot' from your Trilinos clone. - See below. - ------------------------------------------------------------------------- -*** WHEN SOMETHING GOES WRONG AND YOU MUST *** -*** REMOVE THE KOKKOS SNAPSHOT FROM YOUR TRILINOS CLONE *** - -1) Query the Trilinos clone commit log. - git log --oneline - -2) Note the of the commit to the Trillinos clone - immediately BEFORE the Kokkos snapshot commit. - Copy this for use in the next command. - -3) IF more than one outstanding commit then you can remove just the - Kokkos snapshot commit with 'git rebase -i'. Edit the rebase file. - Remove or comment out the Kokkos snapshot commit entry. - git rebase -i - -4) IF the Kokkos snapshot commit is the one and only - outstanding commit then remove just than commit. - git reset --hard HEAD~1 - ------------------------------------------------------------------------- -*** REGARDING 'snapshot.py' TOOL *** - -The 'snapshot.py' tool is developed and maintained by the -Center for Computing Research (CCR) -Software Engineering, Maintenance, and Support (SEMS) team. - -Contact Brent Perschbacher for questions> - ------------------------------------------------------------------------- - diff --git a/lib/kokkos/Makefile.kokkos b/lib/kokkos/Makefile.kokkos index 6b627dcc36..abdfb7a316 100644 --- a/lib/kokkos/Makefile.kokkos +++ b/lib/kokkos/Makefile.kokkos @@ -1,6 +1,6 @@ # Default settings common options. -#LAMMPS specific settings: +#SPARTA specific settings: ifndef KOKKOS_PATH KOKKOS_PATH=../../lib/kokkos endif @@ -11,7 +11,7 @@ CXXFLAGS += $(SHFLAGS) endif KOKKOS_VERSION_MAJOR = 4 -KOKKOS_VERSION_MINOR = 4 +KOKKOS_VERSION_MINOR = 5 KOKKOS_VERSION_PATCH = 1 KOKKOS_VERSION = $(shell echo $(KOKKOS_VERSION_MAJOR)*10000+$(KOKKOS_VERSION_MINOR)*100+$(KOKKOS_VERSION_PATCH) | bc) @@ -23,7 +23,7 @@ KOKKOS_DEVICES ?= "OpenMP" # NVIDIA: Kepler,Kepler30,Kepler32,Kepler35,Kepler37,Maxwell,Maxwell50,Maxwell52,Maxwell53,Pascal60,Pascal61,Volta70,Volta72,Turing75,Ampere80,Ampere86,Ada89,Hopper90 # ARM: ARMv80,ARMv81,ARMv8-ThunderX,ARMv8-TX2,A64FX,ARMv9-Grace # IBM: Power8,Power9 -# AMD-GPUS: AMD_GFX906,AMD_GFX908,AMD_GFX90A,AMD_GFX940,AMD_GFX942,AMD_GFX1030,AMD_GFX1100,AMD_GFX1103 +# AMD-GPUS: AMD_GFX906,AMD_GFX908,AMD_GFX90A,AMD_GFX940,AMD_GFX942,AMD_GFX942_APU,AMD_GFX1030,AMD_GFX1100,AMD_GFX1103 # AMD-CPUS: AMDAVX,Zen,Zen2,Zen3 # Intel-GPUs: Intel_Gen,Intel_Gen9,Intel_Gen11,Intel_Gen12LP,Intel_DG1,Intel_XeHP,Intel_PVC KOKKOS_ARCH ?= "" @@ -40,16 +40,19 @@ KOKKOS_TRIBITS ?= "no" KOKKOS_STANDALONE_CMAKE ?= "no" # Default settings specific options. -# Options: force_uvm,use_ldg,rdc,enable_lambda,enable_constexpr,disable_malloc_async -KOKKOS_CUDA_OPTIONS ?= "disable_malloc_async" +# Options: force_uvm,use_ldg,rdc,enable_lambda,enable_constexpr,enable_malloc_async +KOKKOS_CUDA_OPTIONS ?= "" -# Options: rdc +# Options: rdc,enable_malloc_async KOKKOS_HIP_OPTIONS ?= "" # Default settings specific options. # Options: enable_async_dispatch KOKKOS_HPX_OPTIONS ?= "" +#Options : force_host_as_device +KOKKOS_OPENACC_OPTIONS ?= "" + # Helper functions for conversion to upper case uppercase_TABLE:=a,A b,B c,C d,D e,E f,F g,G h,H i,I j,J k,K l,L m,M n,N o,O p,P q,Q r,R s,S t,T u,U v,V w,W x,X y,Y z,Z uppercase_internal=$(if $1,$$(subst $(firstword $1),$(call uppercase_internal,$(wordlist 2,$(words $1),$1),$2)),$2) @@ -92,7 +95,7 @@ KOKKOS_INTERNAL_CUDA_USE_UVM := $(call kokkos_has_string,$(KOKKOS_CUDA_OPTIONS), KOKKOS_INTERNAL_CUDA_USE_RELOC := $(call kokkos_has_string,$(KOKKOS_CUDA_OPTIONS),rdc) KOKKOS_INTERNAL_CUDA_USE_LAMBDA := $(call kokkos_has_string,$(KOKKOS_CUDA_OPTIONS),enable_lambda) KOKKOS_INTERNAL_CUDA_USE_CONSTEXPR := $(call kokkos_has_string,$(KOKKOS_CUDA_OPTIONS),enable_constexpr) -KOKKOS_INTERNAL_CUDA_DISABLE_MALLOC_ASYNC := $(call kokkos_has_string,$(KOKKOS_CUDA_OPTIONS),disable_malloc_async) +KOKKOS_INTERNAL_CUDA_ENABLE_MALLOC_ASYNC := $(call kokkos_has_string,$(KOKKOS_CUDA_OPTIONS),enable_malloc_async) KOKKOS_INTERNAL_HPX_ENABLE_ASYNC_DISPATCH := $(call kokkos_has_string,$(KOKKOS_HPX_OPTIONS),enable_async_dispatch) # deprecated KOKKOS_INTERNAL_ENABLE_DESUL_ATOMICS := $(call kokkos_has_string,$(KOKKOS_OPTIONS),enable_desul_atomics) @@ -103,6 +106,8 @@ KOKKOS_INTERNAL_DISABLE_DEPRECATED_CODE := $(call kokkos_has_string,$(KOKKOS_OPT KOKKOS_INTERNAL_ENABLE_DEPRECATION_WARNINGS := $(call kokkos_has_string,$(KOKKOS_OPTIONS),enable_deprecation_warnings) KOKKOS_INTERNAL_HIP_USE_RELOC := $(call kokkos_has_string,$(KOKKOS_HIP_OPTIONS),rdc) +KOKKOS_INTERNAL_HIP_ENABLE_MALLOC_ASYNC := $(call kokkos_has_string,$(KOKKOS_HIP_OPTIONS),enable_malloc_async) +KOKKOS_INTERNAL_OPENACC_FORCE_HOST_AS_DEVICE := $(call kokkos_has_string,$(KOKKOS_OPENACC_OPTIONS),force_host_as_device) # Check for Kokkos Host Execution Spaces one of which must be on. KOKKOS_INTERNAL_USE_OPENMP := $(call kokkos_has_string,$(subst OpenMPTarget,,$(KOKKOS_DEVICES)),OpenMP) @@ -178,7 +183,7 @@ KOKKOS_INTERNAL_COMPILER_CRAY := $(strip $(shell $(CXX) -craype-verbose 2 KOKKOS_INTERNAL_COMPILER_NVCC := $(strip $(shell echo "$(shell export OMPI_CXX=$(OMPI_CXX); export MPICH_CXX=$(MPICH_CXX); $(CXX) --version 2>&1 | grep -c nvcc)>0" | bc)) KOKKOS_INTERNAL_COMPILER_NVHPC := $(strip $(shell $(CXX) --version 2>&1 | grep -c "nvc++")) KOKKOS_INTERNAL_COMPILER_CLANG := $(call kokkos_has_string,$(KOKKOS_CXX_VERSION),clang) -KOKKOS_INTERNAL_COMPILER_CRAY_CLANG := $(strip $(shell $(CXX) -craype-verbose 2>&1 | grep -c "clang++")) +KOKKOS_INTERNAL_COMPILER_CRAY_CLANG := $(strip $(shell $(CXX) -craype-verbose 2>&1 | grep -v "error:" | grep -c "clang++")) KOKKOS_INTERNAL_COMPILER_INTEL_CLANG := $(call kokkos_has_string,$(KOKKOS_CXX_VERSION),oneAPI) KOKKOS_INTERNAL_COMPILER_APPLE_CLANG := $(call kokkos_has_string,$(KOKKOS_CXX_VERSION),Apple clang) KOKKOS_INTERNAL_COMPILER_HCC := $(call kokkos_has_string,$(KOKKOS_CXX_VERSION),HCC) @@ -292,6 +297,8 @@ ifeq ($(KOKKOS_INTERNAL_USE_OPENACC), 1) # Set OpenACC flags. ifeq ($(KOKKOS_INTERNAL_COMPILER_NVHPC), 1) KOKKOS_INTERNAL_OPENACC_FLAG := -acc + else ifeq ($(KOKKOS_INTERNAL_COMPILER_CLANG), 1) + KOKKOS_INTERNAL_OPENACC_FLAG := -fopenacc -fopenacc-fake-async-wait -fopenacc-implicit-worker=vector -Wno-openacc-and-cxx -Wno-openmp-mapping -Wno-unknown-cuda-version -Wno-pass-failed else $(error Makefile.kokkos: OpenACC is enabled but the compiler must be NVHPC (got version string $(KOKKOS_CXX_VERSION))) endif @@ -411,8 +418,8 @@ ifeq ($(KOKKOS_INTERNAL_USE_ARCH_NVIDIA), 0) endif ifeq ($(KOKKOS_INTERNAL_USE_ARCH_NVIDIA), 1) + KOKKOS_INTERNAL_NVCC_PATH := $(shell which nvcc) ifeq ($(KOKKOS_INTERNAL_COMPILER_CLANG), 1) - KOKKOS_INTERNAL_NVCC_PATH := $(shell which nvcc) CUDA_PATH ?= $(KOKKOS_INTERNAL_NVCC_PATH:/bin/nvcc=) ifeq ($(KOKKOS_INTERNAL_USE_OPENMPTARGET), 1) KOKKOS_INTERNAL_OPENMPTARGET_FLAG := $(KOKKOS_INTERNAL_OPENMPTARGET_FLAG) --cuda-path=$(CUDA_PATH) @@ -457,6 +464,7 @@ ifeq ($(KOKKOS_INTERNAL_USE_ARCH_AMD_GFX90A), 0) endif KOKKOS_INTERNAL_USE_ARCH_AMD_GFX940 := $(call kokkos_has_string,$(KOKKOS_ARCH),AMD_GFX940) KOKKOS_INTERNAL_USE_ARCH_AMD_GFX942 := $(call kokkos_has_string,$(KOKKOS_ARCH),AMD_GFX942) +KOKKOS_INTERNAL_USE_ARCH_AMD_GFX942_APU := $(call kokkos_has_string,$(KOKKOS_ARCH),AMD_GFX942_APU) KOKKOS_INTERNAL_USE_ARCH_AMD_GFX1030 := $(call kokkos_has_string,$(KOKKOS_ARCH),AMD_GFX1030) ifeq ($(KOKKOS_INTERNAL_USE_ARCH_AMD_GFX1030), 0) KOKKOS_INTERNAL_USE_ARCH_AMD_GFX1030 := $(call kokkos_has_string,$(KOKKOS_ARCH),NAVI1030) @@ -466,6 +474,15 @@ ifeq ($(KOKKOS_INTERNAL_USE_ARCH_AMD_GFX1100), 0) KOKKOS_INTERNAL_USE_ARCH_AMD_GFX1100 := $(call kokkos_has_string,$(KOKKOS_ARCH),NAVI1100) endif KOKKOS_INTERNAL_USE_ARCH_AMD_GFX1103 := $(call kokkos_has_string,$(KOKKOS_ARCH),AMD_GFX1103) +KOKKOS_INTERNAL_USE_ARCH_AMD := $(shell expr $(KOKKOS_INTERNAL_USE_ARCH_AMD_GFX906) \ + + $(KOKKOS_INTERNAL_USE_ARCH_AMD_GFX908) \ + + $(KOKKOS_INTERNAL_USE_ARCH_AMD_GFX90A) \ + + $(KOKKOS_INTERNAL_USE_ARCH_AMD_GFX940) \ + + $(KOKKOS_INTERNAL_USE_ARCH_AMD_GFX942) \ + + $(KOKKOS_INTERNAL_USE_ARCH_AMD_GFX942_APU) \ + + $(KOKKOS_INTERNAL_USE_ARCH_AMD_GFX1030) \ + + $(KOKKOS_INTERNAL_USE_ARCH_AMD_GFX1100) \ + + $(KOKKOS_INTERNAL_USE_ARCH_AMD_GFX1103)) # Any AVX? KOKKOS_INTERNAL_USE_ARCH_AVX := $(shell expr $(KOKKOS_INTERNAL_USE_ARCH_SNB) + $(KOKKOS_INTERNAL_USE_ARCH_AMDAVX)) @@ -561,6 +578,9 @@ endif ifeq ($(KOKKOS_INTERNAL_USE_OPENACC), 1) tmp := $(call kokkos_append_header,"$H""define KOKKOS_ENABLE_OPENACC") + ifeq ($(KOKKOS_INTERNAL_OPENACC_FORCE_HOST_AS_DEVICE), 1) + tmp := $(call kokkos_append_header,"$H""define KOKKOS_ENABLE_OPENACC_FORCE_HOST_AS_DEVICE") + endif endif ifeq ($(KOKKOS_INTERNAL_USE_OPENMP), 1) @@ -733,7 +753,7 @@ ifeq ($(KOKKOS_INTERNAL_USE_CUDA), 1) endif endif - ifeq ($(KOKKOS_INTERNAL_CUDA_DISABLE_MALLOC_ASYNC), 0) + ifeq ($(KOKKOS_INTERNAL_CUDA_ENABLE_MALLOC_ASYNC), 1) tmp := $(call kokkos_append_header,"$H""define KOKKOS_ENABLE_IMPL_CUDA_MALLOC_ASYNC") else tmp := $(call kokkos_append_header,"/* $H""undef KOKKOS_ENABLE_IMPL_CUDA_MALLOC_ASYNC */") @@ -1024,86 +1044,122 @@ ifeq ($(KOKKOS_INTERNAL_USE_OPENMPTARGET), 1) endif endif +ifeq ($(KOKKOS_INTERNAL_USE_OPENACC), 1) + ifeq ($(KOKKOS_INTERNAL_COMPILER_CLANG), 1) + KOKKOS_INTERNAL_CUDA_ARCH_FLAG=--offload-arch + endif +endif + # Do not add this flag if its the cray compiler or the nvhpc compiler. ifeq ($(KOKKOS_INTERNAL_COMPILER_CRAY_CLANG), 0) - ifeq ($(KOKKOS_INTERNAL_COMPILER_NVHPC), 0) - # Lets start with adding architecture defines - ifeq ($(KOKKOS_INTERNAL_USE_ARCH_KEPLER30), 1) - tmp := $(call kokkos_append_header,"$H""define KOKKOS_ARCH_KEPLER") - tmp := $(call kokkos_append_header,"$H""define KOKKOS_ARCH_KEPLER30") + # Lets start with adding architecture defines + ifeq ($(KOKKOS_INTERNAL_USE_ARCH_KEPLER30), 1) + tmp := $(call kokkos_append_header,"$H""define KOKKOS_ARCH_KEPLER") + tmp := $(call kokkos_append_header,"$H""define KOKKOS_ARCH_KEPLER30") + ifeq ($(KOKKOS_INTERNAL_COMPILER_NVHPC), 0) KOKKOS_INTERNAL_CUDA_ARCH_FLAG := $(KOKKOS_INTERNAL_CUDA_ARCH_FLAG)=sm_30 endif - ifeq ($(KOKKOS_INTERNAL_USE_ARCH_KEPLER32), 1) - tmp := $(call kokkos_append_header,"$H""define KOKKOS_ARCH_KEPLER") - tmp := $(call kokkos_append_header,"$H""define KOKKOS_ARCH_KEPLER32") + endif + ifeq ($(KOKKOS_INTERNAL_USE_ARCH_KEPLER32), 1) + tmp := $(call kokkos_append_header,"$H""define KOKKOS_ARCH_KEPLER") + tmp := $(call kokkos_append_header,"$H""define KOKKOS_ARCH_KEPLER32") + ifeq ($(KOKKOS_INTERNAL_COMPILER_NVHPC), 0) KOKKOS_INTERNAL_CUDA_ARCH_FLAG := $(KOKKOS_INTERNAL_CUDA_ARCH_FLAG)=sm_32 endif - ifeq ($(KOKKOS_INTERNAL_USE_ARCH_KEPLER35), 1) - tmp := $(call kokkos_append_header,"$H""define KOKKOS_ARCH_KEPLER") - tmp := $(call kokkos_append_header,"$H""define KOKKOS_ARCH_KEPLER35") + endif + ifeq ($(KOKKOS_INTERNAL_USE_ARCH_KEPLER35), 1) + tmp := $(call kokkos_append_header,"$H""define KOKKOS_ARCH_KEPLER") + tmp := $(call kokkos_append_header,"$H""define KOKKOS_ARCH_KEPLER35") + ifeq ($(KOKKOS_INTERNAL_COMPILER_NVHPC), 0) KOKKOS_INTERNAL_CUDA_ARCH_FLAG := $(KOKKOS_INTERNAL_CUDA_ARCH_FLAG)=sm_35 endif - ifeq ($(KOKKOS_INTERNAL_USE_ARCH_KEPLER37), 1) - tmp := $(call kokkos_append_header,"$H""define KOKKOS_ARCH_KEPLER") - tmp := $(call kokkos_append_header,"$H""define KOKKOS_ARCH_KEPLER37") + endif + ifeq ($(KOKKOS_INTERNAL_USE_ARCH_KEPLER37), 1) + tmp := $(call kokkos_append_header,"$H""define KOKKOS_ARCH_KEPLER") + tmp := $(call kokkos_append_header,"$H""define KOKKOS_ARCH_KEPLER37") + ifeq ($(KOKKOS_INTERNAL_COMPILER_NVHPC), 0) KOKKOS_INTERNAL_CUDA_ARCH_FLAG := $(KOKKOS_INTERNAL_CUDA_ARCH_FLAG)=sm_37 endif - ifeq ($(KOKKOS_INTERNAL_USE_ARCH_MAXWELL50), 1) - tmp := $(call kokkos_append_header,"$H""define KOKKOS_ARCH_MAXWELL") - tmp := $(call kokkos_append_header,"$H""define KOKKOS_ARCH_MAXWELL50") + endif + ifeq ($(KOKKOS_INTERNAL_USE_ARCH_MAXWELL50), 1) + tmp := $(call kokkos_append_header,"$H""define KOKKOS_ARCH_MAXWELL") + tmp := $(call kokkos_append_header,"$H""define KOKKOS_ARCH_MAXWELL50") + ifeq ($(KOKKOS_INTERNAL_COMPILER_NVHPC), 0) KOKKOS_INTERNAL_CUDA_ARCH_FLAG := $(KOKKOS_INTERNAL_CUDA_ARCH_FLAG)=sm_50 endif - ifeq ($(KOKKOS_INTERNAL_USE_ARCH_MAXWELL52), 1) - tmp := $(call kokkos_append_header,"$H""define KOKKOS_ARCH_MAXWELL") - tmp := $(call kokkos_append_header,"$H""define KOKKOS_ARCH_MAXWELL52") + endif + ifeq ($(KOKKOS_INTERNAL_USE_ARCH_MAXWELL52), 1) + tmp := $(call kokkos_append_header,"$H""define KOKKOS_ARCH_MAXWELL") + tmp := $(call kokkos_append_header,"$H""define KOKKOS_ARCH_MAXWELL52") + ifeq ($(KOKKOS_INTERNAL_COMPILER_NVHPC), 0) KOKKOS_INTERNAL_CUDA_ARCH_FLAG := $(KOKKOS_INTERNAL_CUDA_ARCH_FLAG)=sm_52 endif - ifeq ($(KOKKOS_INTERNAL_USE_ARCH_MAXWELL53), 1) - tmp := $(call kokkos_append_header,"$H""define KOKKOS_ARCH_MAXWELL") - tmp := $(call kokkos_append_header,"$H""define KOKKOS_ARCH_MAXWELL53") + endif + ifeq ($(KOKKOS_INTERNAL_USE_ARCH_MAXWELL53), 1) + tmp := $(call kokkos_append_header,"$H""define KOKKOS_ARCH_MAXWELL") + tmp := $(call kokkos_append_header,"$H""define KOKKOS_ARCH_MAXWELL53") + ifeq ($(KOKKOS_INTERNAL_COMPILER_NVHPC), 0) KOKKOS_INTERNAL_CUDA_ARCH_FLAG := $(KOKKOS_INTERNAL_CUDA_ARCH_FLAG)=sm_53 endif - ifeq ($(KOKKOS_INTERNAL_USE_ARCH_PASCAL60), 1) - tmp := $(call kokkos_append_header,"$H""define KOKKOS_ARCH_PASCAL") - tmp := $(call kokkos_append_header,"$H""define KOKKOS_ARCH_PASCAL60") + endif + ifeq ($(KOKKOS_INTERNAL_USE_ARCH_PASCAL60), 1) + tmp := $(call kokkos_append_header,"$H""define KOKKOS_ARCH_PASCAL") + tmp := $(call kokkos_append_header,"$H""define KOKKOS_ARCH_PASCAL60") + ifeq ($(KOKKOS_INTERNAL_COMPILER_NVHPC), 0) KOKKOS_INTERNAL_CUDA_ARCH_FLAG := $(KOKKOS_INTERNAL_CUDA_ARCH_FLAG)=sm_60 endif - ifeq ($(KOKKOS_INTERNAL_USE_ARCH_PASCAL61), 1) - tmp := $(call kokkos_append_header,"$H""define KOKKOS_ARCH_PASCAL") - tmp := $(call kokkos_append_header,"$H""define KOKKOS_ARCH_PASCAL61") + endif + ifeq ($(KOKKOS_INTERNAL_USE_ARCH_PASCAL61), 1) + tmp := $(call kokkos_append_header,"$H""define KOKKOS_ARCH_PASCAL") + tmp := $(call kokkos_append_header,"$H""define KOKKOS_ARCH_PASCAL61") + ifeq ($(KOKKOS_INTERNAL_COMPILER_NVHPC), 0) KOKKOS_INTERNAL_CUDA_ARCH_FLAG := $(KOKKOS_INTERNAL_CUDA_ARCH_FLAG)=sm_61 endif - ifeq ($(KOKKOS_INTERNAL_USE_ARCH_VOLTA70), 1) - tmp := $(call kokkos_append_header,"$H""define KOKKOS_ARCH_VOLTA") - tmp := $(call kokkos_append_header,"$H""define KOKKOS_ARCH_VOLTA70") + endif + ifeq ($(KOKKOS_INTERNAL_USE_ARCH_VOLTA70), 1) + tmp := $(call kokkos_append_header,"$H""define KOKKOS_ARCH_VOLTA") + tmp := $(call kokkos_append_header,"$H""define KOKKOS_ARCH_VOLTA70") + ifeq ($(KOKKOS_INTERNAL_COMPILER_NVHPC), 0) KOKKOS_INTERNAL_CUDA_ARCH_FLAG := $(KOKKOS_INTERNAL_CUDA_ARCH_FLAG)=sm_70 endif - ifeq ($(KOKKOS_INTERNAL_USE_ARCH_VOLTA72), 1) - tmp := $(call kokkos_append_header,"$H""define KOKKOS_ARCH_VOLTA") - tmp := $(call kokkos_append_header,"$H""define KOKKOS_ARCH_VOLTA72") + endif + ifeq ($(KOKKOS_INTERNAL_USE_ARCH_VOLTA72), 1) + tmp := $(call kokkos_append_header,"$H""define KOKKOS_ARCH_VOLTA") + tmp := $(call kokkos_append_header,"$H""define KOKKOS_ARCH_VOLTA72") + ifeq ($(KOKKOS_INTERNAL_COMPILER_NVHPC), 0) KOKKOS_INTERNAL_CUDA_ARCH_FLAG := $(KOKKOS_INTERNAL_CUDA_ARCH_FLAG)=sm_72 endif - ifeq ($(KOKKOS_INTERNAL_USE_ARCH_TURING75), 1) - tmp := $(call kokkos_append_header,"$H""define KOKKOS_ARCH_TURING75") + endif + ifeq ($(KOKKOS_INTERNAL_USE_ARCH_TURING75), 1) + tmp := $(call kokkos_append_header,"$H""define KOKKOS_ARCH_TURING75") + ifeq ($(KOKKOS_INTERNAL_COMPILER_NVHPC), 0) KOKKOS_INTERNAL_CUDA_ARCH_FLAG := $(KOKKOS_INTERNAL_CUDA_ARCH_FLAG)=sm_75 endif - ifeq ($(KOKKOS_INTERNAL_USE_ARCH_AMPERE80), 1) - tmp := $(call kokkos_append_header,"$H""define KOKKOS_ARCH_AMPERE") - tmp := $(call kokkos_append_header,"$H""define KOKKOS_ARCH_AMPERE80") + endif + ifeq ($(KOKKOS_INTERNAL_USE_ARCH_AMPERE80), 1) + tmp := $(call kokkos_append_header,"$H""define KOKKOS_ARCH_AMPERE") + tmp := $(call kokkos_append_header,"$H""define KOKKOS_ARCH_AMPERE80") + ifeq ($(KOKKOS_INTERNAL_COMPILER_NVHPC), 0) KOKKOS_INTERNAL_CUDA_ARCH_FLAG := $(KOKKOS_INTERNAL_CUDA_ARCH_FLAG)=sm_80 endif - ifeq ($(KOKKOS_INTERNAL_USE_ARCH_AMPERE86), 1) - tmp := $(call kokkos_append_header,"$H""define KOKKOS_ARCH_AMPERE") - tmp := $(call kokkos_append_header,"$H""define KOKKOS_ARCH_AMPERE86") + endif + ifeq ($(KOKKOS_INTERNAL_USE_ARCH_AMPERE86), 1) + tmp := $(call kokkos_append_header,"$H""define KOKKOS_ARCH_AMPERE") + tmp := $(call kokkos_append_header,"$H""define KOKKOS_ARCH_AMPERE86") + ifeq ($(KOKKOS_INTERNAL_COMPILER_NVHPC), 0) KOKKOS_INTERNAL_CUDA_ARCH_FLAG := $(KOKKOS_INTERNAL_CUDA_ARCH_FLAG)=sm_86 endif - ifeq ($(KOKKOS_INTERNAL_USE_ARCH_ADA89), 1) - tmp := $(call kokkos_append_header,"$H""define KOKKOS_ARCH_ADA89") + endif + ifeq ($(KOKKOS_INTERNAL_USE_ARCH_ADA89), 1) + tmp := $(call kokkos_append_header,"$H""define KOKKOS_ARCH_ADA89") + ifeq ($(KOKKOS_INTERNAL_COMPILER_NVHPC), 0) KOKKOS_INTERNAL_CUDA_ARCH_FLAG := $(KOKKOS_INTERNAL_CUDA_ARCH_FLAG)=sm_89 endif - ifeq ($(KOKKOS_INTERNAL_USE_ARCH_HOPPER90), 1) - tmp := $(call kokkos_append_header,"$H""define KOKKOS_ARCH_HOPPER") - tmp := $(call kokkos_append_header,"$H""define KOKKOS_ARCH_HOPPER90") + endif + ifeq ($(KOKKOS_INTERNAL_USE_ARCH_HOPPER90), 1) + tmp := $(call kokkos_append_header,"$H""define KOKKOS_ARCH_HOPPER") + tmp := $(call kokkos_append_header,"$H""define KOKKOS_ARCH_HOPPER90") + ifeq ($(KOKKOS_INTERNAL_COMPILER_NVHPC), 0) KOKKOS_INTERNAL_CUDA_ARCH_FLAG := $(KOKKOS_INTERNAL_CUDA_ARCH_FLAG)=sm_90 endif endif @@ -1119,6 +1175,9 @@ ifneq ($(KOKKOS_INTERNAL_USE_ARCH_NVIDIA), 0) ifeq ($(KOKKOS_INTERNAL_USE_OPENMPTARGET), 1) KOKKOS_LDFLAGS += $(KOKKOS_INTERNAL_CUDA_ARCH_FLAG) endif + ifeq ($(KOKKOS_INTERNAL_USE_OPENACC), 1) + KOKKOS_LDFLAGS += $(KOKKOS_INTERNAL_CUDA_ARCH_FLAG) + endif endif endif @@ -1126,43 +1185,48 @@ endif # Figure out the architecture flag for ROCm. ifeq ($(KOKKOS_INTERNAL_USE_ARCH_AMD_GFX906), 1) tmp := $(call kokkos_append_header,"$H""define KOKKOS_ARCH_AMD_GFX906") - tmp := $(call kokkos_append_header,"$H""define KOKKOS_ARCH_AMD_GPU") - KOKKOS_INTERNAL_HIP_ARCH_FLAG := --offload-arch=gfx906 + tmp := $(call kokkos_append_header,"$H""define KOKKOS_ARCH_AMD_GPU \"gfx906\"") + KOKKOS_INTERNAL_AMD_ARCH_FLAG := --offload-arch=gfx906 endif ifeq ($(KOKKOS_INTERNAL_USE_ARCH_AMD_GFX908), 1) tmp := $(call kokkos_append_header,"$H""define KOKKOS_ARCH_AMD_GFX908") - tmp := $(call kokkos_append_header,"$H""define KOKKOS_ARCH_AMD_GPU") - KOKKOS_INTERNAL_HIP_ARCH_FLAG := --offload-arch=gfx908 + tmp := $(call kokkos_append_header,"$H""define KOKKOS_ARCH_AMD_GPU \"gfx908\"") + KOKKOS_INTERNAL_AMD_ARCH_FLAG := --offload-arch=gfx908 endif ifeq ($(KOKKOS_INTERNAL_USE_ARCH_AMD_GFX90A), 1) tmp := $(call kokkos_append_header,"$H""define KOKKOS_ARCH_AMD_GFX90A") - tmp := $(call kokkos_append_header,"$H""define KOKKOS_ARCH_AMD_GPU") - KOKKOS_INTERNAL_HIP_ARCH_FLAG := --offload-arch=gfx90a + tmp := $(call kokkos_append_header,"$H""define KOKKOS_ARCH_AMD_GPU \"gfx90A\"") + KOKKOS_INTERNAL_AMD_ARCH_FLAG := --offload-arch=gfx90a endif ifeq ($(KOKKOS_INTERNAL_USE_ARCH_AMD_GFX940), 1) tmp := $(call kokkos_append_header,"$H""define KOKKOS_ARCH_AMD_GFX940") - tmp := $(call kokkos_append_header,"$H""define KOKKOS_ARCH_AMD_GPU") - KOKKOS_INTERNAL_HIP_ARCH_FLAG := --offload-arch=gfx940 + tmp := $(call kokkos_append_header,"$H""define KOKKOS_ARCH_AMD_GPU \"gfx940\"") + KOKKOS_INTERNAL_AMD_ARCH_FLAG := --offload-arch=gfx940 endif ifeq ($(KOKKOS_INTERNAL_USE_ARCH_AMD_GFX942), 1) tmp := $(call kokkos_append_header,"$H""define KOKKOS_ARCH_AMD_GFX942") - tmp := $(call kokkos_append_header,"$H""define KOKKOS_ARCH_AMD_GPU") - KOKKOS_INTERNAL_HIP_ARCH_FLAG := --offload-arch=gfx942 + tmp := $(call kokkos_append_header,"$H""define KOKKOS_ARCH_AMD_GPU \"gfx942\"") + KOKKOS_INTERNAL_AMD_ARCH_FLAG := --offload-arch=gfx942 +endif +ifeq ($(KOKKOS_INTERNAL_USE_ARCH_AMD_GFX942_APU), 1) + tmp := $(call kokkos_append_header,"$H""define KOKKOS_ARCH_AMD_GFX942_APU") + tmp := $(call kokkos_append_header,"$H""define KOKKOS_ARCH_AMD_GPU \"gfx942\"") + KOKKOS_INTERNAL_AMD_ARCH_FLAG := --offload-arch=gfx942 endif ifeq ($(KOKKOS_INTERNAL_USE_ARCH_AMD_GFX1030), 1) tmp := $(call kokkos_append_header,"$H""define KOKKOS_ARCH_AMD_GFX1030") - tmp := $(call kokkos_append_header,"$H""define KOKKOS_ARCH_AMD_GPU") - KOKKOS_INTERNAL_HIP_ARCH_FLAG := --offload-arch=gfx1030 + tmp := $(call kokkos_append_header,"$H""define KOKKOS_ARCH_AMD_GPU \"gfx1030\"") + KOKKOS_INTERNAL_AMD_ARCH_FLAG := --offload-arch=gfx1030 endif ifeq ($(KOKKOS_INTERNAL_USE_ARCH_AMD_GFX1100), 1) tmp := $(call kokkos_append_header,"$H""define KOKKOS_ARCH_AMD_GFX1100") - tmp := $(call kokkos_append_header,"$H""define KOKKOS_ARCH_AMD_GPU") - KOKKOS_INTERNAL_HIP_ARCH_FLAG := --offload-arch=gfx1100 + tmp := $(call kokkos_append_header,"$H""define KOKKOS_ARCH_AMD_GPU \"gfx1100\"") + KOKKOS_INTERNAL_AMD_ARCH_FLAG := --offload-arch=gfx1100 endif ifeq ($(KOKKOS_INTERNAL_USE_ARCH_AMD_GFX1103), 1) tmp := $(call kokkos_append_header,"$H""define KOKKOS_ARCH_AMD_GFX1103") - tmp := $(call kokkos_append_header,"$H""define KOKKOS_ARCH_AMD_GPU") - KOKKOS_INTERNAL_HIP_ARCH_FLAG := --offload-arch=gfx1103 + tmp := $(call kokkos_append_header,"$H""define KOKKOS_ARCH_AMD_GPU \"gfx1103\"") + KOKKOS_INTERNAL_AMD_ARCH_FLAG := --offload-arch=gfx1103 endif @@ -1171,8 +1235,8 @@ ifeq ($(KOKKOS_INTERNAL_USE_HIP), 1) KOKKOS_SRC += $(KOKKOS_PATH)/tpls/desul/src/Lock_Array_HIP.cpp KOKKOS_HEADERS += $(wildcard $(KOKKOS_PATH)/core/src/HIP/*.hpp) - KOKKOS_CXXFLAGS+=$(KOKKOS_INTERNAL_HIP_ARCH_FLAG) - KOKKOS_LDFLAGS+=$(KOKKOS_INTERNAL_HIP_ARCH_FLAG) + KOKKOS_CXXFLAGS+=$(KOKKOS_INTERNAL_AMD_ARCH_FLAG) + KOKKOS_LDFLAGS+=$(KOKKOS_INTERNAL_AMD_ARCH_FLAG) ifeq ($(KOKKOS_INTERNAL_HIP_USE_RELOC), 1) tmp := $(call kokkos_append_header,"$H""define KOKKOS_ENABLE_HIP_RELOCATABLE_DEVICE_CODE") @@ -1182,6 +1246,21 @@ ifeq ($(KOKKOS_INTERNAL_USE_HIP), 1) KOKKOS_CXXFLAGS+=-fno-gpu-rdc KOKKOS_LDFLAGS+=-fno-gpu-rdc endif + + ifeq ($(KOKKOS_INTERNAL_HIP_ENABLE_MALLOC_ASYNC), 1) + tmp := $(call kokkos_append_header,"$H""define KOKKOS_ENABLE_IMPL_HIP_MALLOC_ASYNC") + else + tmp := $(call kokkos_append_header,"/* $H""undef KOKKOS_ENABLE_IMPL_HIP_MALLOC_ASYNC */") + endif +endif + +ifneq ($(KOKKOS_INTERNAL_USE_ARCH_AMD), 0) + ifeq ($(KOKKOS_INTERNAL_COMPILER_CLANG), 1) + ifeq ($(KOKKOS_INTERNAL_USE_OPENACC), 1) + KOKKOS_CXXFLAGS += $(KOKKOS_INTERNAL_AMD_ARCH_FLAG) + KOKKOS_LDFLAGS += $(KOKKOS_INTERNAL_AMD_ARCH_FLAG) + endif + endif endif # Figure out Intel architecture flags. @@ -1235,6 +1314,8 @@ ifeq ($(KOKKOS_INTERNAL_USE_SYCL), 1) KOKKOS_CXXFLAGS+=$(KOKKOS_INTERNAL_INTEL_ARCH_FLAG) KOKKOS_LDFLAGS+=-fsycl KOKKOS_LDFLAGS+=$(KOKKOS_INTERNAL_INTEL_ARCH_FLAG) + + tmp := $(call kokkos_append_header,"$H""define KOKKOS_ENABLE_SYCL_RELOCATABLE_DEVICE_CODE") endif ifeq ($(KOKKOS_INTERNAL_USE_OPENMPTARGET), 1) @@ -1322,6 +1403,8 @@ ifneq ($(KOKKOS_INTERNAL_NEW_CONFIG), 0) endif KOKKOS_HEADERS += $(wildcard $(KOKKOS_PATH)/core/src/*.hpp) +KOKKOS_HEADERS += $(wildcard $(KOKKOS_PATH)/core/src/View/*.hpp) +KOKKOS_HEADERS += $(wildcard $(KOKKOS_PATH)/core/src/View/MDSpan/*.hpp) KOKKOS_HEADERS += $(wildcard $(KOKKOS_PATH)/core/src/impl/*.hpp) KOKKOS_HEADERS += $(wildcard $(KOKKOS_PATH)/containers/src/*.hpp) KOKKOS_HEADERS += $(wildcard $(KOKKOS_PATH)/containers/src/impl/*.hpp) @@ -1374,6 +1457,48 @@ ifeq ($(KOKKOS_INTERNAL_USE_OPENACC), 1) KOKKOS_CXXFLAGS += $(KOKKOS_INTERNAL_OPENACC_FLAG) KOKKOS_LDFLAGS += $(KOKKOS_INTERNAL_OPENACC_FLAG) KOKKOS_LIBS += $(KOKKOS_INTERNAL_OPENACC_LIB) + ifeq ($(KOKKOS_INTERNAL_USE_ARCH_NVIDIA), 1) + ifneq ($(CUDA_PATH),) + ifeq ($(call kokkos_path_exists,$(CUDA_PATH)/lib), 1) + CUDA_PATH := $(CUDA_PATH:/compilers=/cuda) + endif + endif + ifeq ($(KOKKOS_INTERNAL_COMPILER_CLANG), 1) + ifneq ($(CUDA_PATH),) + KOKKOS_LDFLAGS += -L$(CUDA_PATH)/lib64 + endif + KOKKOS_LIBS += -lcudart + endif + ifeq ($(KOKKOS_INTERNAL_COMPILER_NVHPC), 1) + KOKKOS_LIBS += -cuda + endif + ifeq ($(KOKKOS_INTERNAL_OPENACC_FORCE_HOST_AS_DEVICE), 1) + $(error If a GPU architecture is specified, KOKKOS_OPENACC_OPTIONS = force_host_as_device cannot be used. Disable the force_host_as_device option) + endif + else ifeq ($(KOKKOS_INTERNAL_USE_ARCH_AMD), 1) + ifeq ($(KOKKOS_INTERNAL_COMPILER_CLANG), 1) + ifneq ($(ROCM_PATH),) + KOKKOS_CPPFLAGS += -I$(ROCM_PATH)/include + KOKKOS_LDFLAGS += -L$(ROCM_PATH)/lib + endif + KOKKOS_LIBS += -lamdhip64 + endif + ifeq ($(KOKKOS_INTERNAL_OPENACC_FORCE_HOST_AS_DEVICE), 1) + $(error If a GPU architecture is specified, KOKKOS_OPENACC_OPTIONS = force_host_as_device cannot be used. Disable the force_host_as_device option) + endif + else ifeq ($(KOKKOS_INTERNAL_OPENACC_FORCE_HOST_AS_DEVICE), 1) + # Compile for kernel execution on the host. In that case, + # memory is shared between the OpenACC space and the host space. + ifeq ($(KOKKOS_INTERNAL_COMPILER_NVHPC), 1) + KOKKOS_CXXFLAGS += -acc=multicore + endif + else + # Automatic fallback mode; try to offload any available GPU, and fall back + # to the host CPU if no available GPU is found. + ifeq ($(KOKKOS_INTERNAL_COMPILER_NVHPC), 1) + KOKKOS_CXXFLAGS += -acc=gpu,multicore + endif + endif endif ifeq ($(KOKKOS_INTERNAL_USE_OPENMP), 1) @@ -1484,7 +1609,11 @@ else endif ifeq ($(KOKKOS_INTERNAL_USE_OPENACC), 1) - tmp := $(call desul_append_header,"$H""define DESUL_ATOMICS_ENABLE_OPENACC") + ifeq ($(KOKKOS_INTERNAL_COMPILER_NVHPC), 1) + tmp := $(call desul_append_header,"$H""define DESUL_ATOMICS_ENABLE_OPENACC") + else + tmp := $(call desul_append_header,"/* $H""undef DESUL_ATOMICS_ENABLE_OPENACC */") + endif else tmp := $(call desul_append_header,"/* $H""undef DESUL_ATOMICS_ENABLE_OPENACC */") endif @@ -1512,6 +1641,12 @@ $(DESUL_CONFIG_HEADER): KOKKOS_CPP_DEPENDS := $(DESUL_CONFIG_HEADER) KokkosCore_config.h $(KOKKOS_HEADERS) +# Tasking is deprecated +ifeq ($(KOKKOS_INTERNAL_DISABLE_DEPRECATED_CODE), 1) + TMP_KOKKOS_SRC := $(KOKKOS_SRC) + KOKKOS_SRC = $(patsubst %Task.cpp,, $(TMP_KOKKOS_SRC)) +endif + KOKKOS_OBJ = $(KOKKOS_SRC:.cpp=.o) KOKKOS_OBJ_LINK = $(notdir $(KOKKOS_OBJ)) diff --git a/lib/kokkos/Makefile.targets b/lib/kokkos/Makefile.targets index e8e429e027..be535eea3e 100644 --- a/lib/kokkos/Makefile.targets +++ b/lib/kokkos/Makefile.targets @@ -16,8 +16,6 @@ Kokkos_HostSpace.o: $(KOKKOS_CPP_DEPENDS) $(KOKKOS_PATH)/core/src/impl/Kokkos_Ho $(CXX) $(KOKKOS_CPPFLAGS) $(KOKKOS_CXXFLAGS) $(CXXFLAGS) -c $(KOKKOS_PATH)/core/src/impl/Kokkos_HostSpace.cpp Kokkos_hwloc.o: $(KOKKOS_CPP_DEPENDS) $(KOKKOS_PATH)/core/src/impl/Kokkos_hwloc.cpp $(CXX) $(KOKKOS_CPPFLAGS) $(KOKKOS_CXXFLAGS) $(CXXFLAGS) -c $(KOKKOS_PATH)/core/src/impl/Kokkos_hwloc.cpp -Kokkos_TaskQueue.o: $(KOKKOS_CPP_DEPENDS) $(KOKKOS_PATH)/core/src/impl/Kokkos_TaskQueue.cpp - $(CXX) $(KOKKOS_CPPFLAGS) $(KOKKOS_CXXFLAGS) $(CXXFLAGS) -c $(KOKKOS_PATH)/core/src/impl/Kokkos_TaskQueue.cpp Kokkos_HostThreadTeam.o: $(KOKKOS_CPP_DEPENDS) $(KOKKOS_PATH)/core/src/impl/Kokkos_HostThreadTeam.cpp $(CXX) $(KOKKOS_CPPFLAGS) $(KOKKOS_CXXFLAGS) $(CXXFLAGS) -c $(KOKKOS_PATH)/core/src/impl/Kokkos_HostThreadTeam.cpp Kokkos_HostBarrier.o: $(KOKKOS_CPP_DEPENDS) $(KOKKOS_PATH)/core/src/impl/Kokkos_HostBarrier.cpp @@ -38,17 +36,21 @@ Kokkos_Abort.o: $(KOKKOS_CPP_DEPENDS) $(KOKKOS_PATH)/core/src/impl/Kokkos_Abort. ifeq ($(KOKKOS_INTERNAL_USE_SERIAL), 1) Kokkos_Serial.o: $(KOKKOS_CPP_DEPENDS) $(KOKKOS_PATH)/core/src/Serial/Kokkos_Serial.cpp $(CXX) $(KOKKOS_CPPFLAGS) $(KOKKOS_CXXFLAGS) $(CXXFLAGS) -c $(KOKKOS_PATH)/core/src/Serial/Kokkos_Serial.cpp +ifneq ($(KOKKOS_INTERNAL_DISABLE_DEPRECATED_CODE), 1) Kokkos_Serial_Task.o: $(KOKKOS_CPP_DEPENDS) $(KOKKOS_PATH)/core/src/Serial/Kokkos_Serial_Task.cpp $(CXX) $(KOKKOS_CPPFLAGS) $(KOKKOS_CXXFLAGS) $(CXXFLAGS) -c $(KOKKOS_PATH)/core/src/Serial/Kokkos_Serial_Task.cpp endif +endif ifeq ($(KOKKOS_INTERNAL_USE_CUDA), 1) Kokkos_Cuda_Instance.o: $(KOKKOS_CPP_DEPENDS) $(KOKKOS_PATH)/core/src/Cuda/Kokkos_Cuda_Instance.cpp $(CXX) $(KOKKOS_CPPFLAGS) $(KOKKOS_CXXFLAGS) $(CXXFLAGS) -c $(KOKKOS_PATH)/core/src/Cuda/Kokkos_Cuda_Instance.cpp Kokkos_CudaSpace.o: $(KOKKOS_CPP_DEPENDS) $(KOKKOS_PATH)/core/src/Cuda/Kokkos_CudaSpace.cpp $(CXX) $(KOKKOS_CPPFLAGS) $(KOKKOS_CXXFLAGS) $(CXXFLAGS) -c $(KOKKOS_PATH)/core/src/Cuda/Kokkos_CudaSpace.cpp +ifneq ($(KOKKOS_INTERNAL_DISABLE_DEPRECATED_CODE), 1) Kokkos_Cuda_Task.o: $(KOKKOS_CPP_DEPENDS) $(KOKKOS_PATH)/core/src/Cuda/Kokkos_Cuda_Task.cpp $(CXX) $(KOKKOS_CPPFLAGS) $(KOKKOS_CXXFLAGS) $(CXXFLAGS) -c $(KOKKOS_PATH)/core/src/Cuda/Kokkos_Cuda_Task.cpp +endif Lock_Array_CUDA.o: $(KOKKOS_CPP_DEPENDS) $(KOKKOS_PATH)/tpls/desul/src/Lock_Array_CUDA.cpp $(CXX) $(KOKKOS_CPPFLAGS) $(KOKKOS_CXXFLAGS) $(CXXFLAGS) -c $(KOKKOS_PATH)/tpls/desul/src/Lock_Array_CUDA.cpp endif @@ -73,6 +75,8 @@ Kokkos_HIP_Space.o: $(KOKKOS_CPP_DEPENDS) $(KOKKOS_PATH)/core/src/HIP/Kokkos_HIP $(CXX) $(KOKKOS_CPPFLAGS) $(KOKKOS_CXXFLAGS) $(CXXFLAGS) -c $(KOKKOS_PATH)/core/src/HIP/Kokkos_HIP_Space.cpp Kokkos_HIP_Instance.o: $(KOKKOS_CPP_DEPENDS) $(KOKKOS_PATH)/core/src/HIP/Kokkos_HIP_Instance.cpp $(CXX) $(KOKKOS_CPPFLAGS) $(KOKKOS_CXXFLAGS) $(CXXFLAGS) -c $(KOKKOS_PATH)/core/src/HIP/Kokkos_HIP_Instance.cpp +Kokkos_HIP_ZeroMemset.o: $(KOKKOS_CPP_DEPENDS) $(KOKKOS_PATH)/core/src/HIP/Kokkos_HIP_ZeroMemset.cpp + $(CXX) $(KOKKOS_CPPFLAGS) $(KOKKOS_CXXFLAGS) $(CXXFLAGS) -c $(KOKKOS_PATH)/core/src/HIP/Kokkos_HIP_ZeroMemset.cpp Lock_Array_HIP.o: $(KOKKOS_CPP_DEPENDS) $(KOKKOS_PATH)/tpls/desul/src/Lock_Array_HIP.cpp $(CXX) $(KOKKOS_CPPFLAGS) $(KOKKOS_CXXFLAGS) $(CXXFLAGS) -c $(KOKKOS_PATH)/tpls/desul/src/Lock_Array_HIP.cpp endif @@ -89,26 +93,26 @@ Kokkos_OpenMP.o: $(KOKKOS_CPP_DEPENDS) $(KOKKOS_PATH)/core/src/OpenMP/Kokkos_Ope $(CXX) $(KOKKOS_CPPFLAGS) $(KOKKOS_CXXFLAGS) $(CXXFLAGS) -c $(KOKKOS_PATH)/core/src/OpenMP/Kokkos_OpenMP.cpp Kokkos_OpenMP_Instance.o: $(KOKKOS_CPP_DEPENDS) $(KOKKOS_PATH)/core/src/OpenMP/Kokkos_OpenMP_Instance.cpp $(CXX) $(KOKKOS_CPPFLAGS) $(KOKKOS_CXXFLAGS) $(CXXFLAGS) -c $(KOKKOS_PATH)/core/src/OpenMP/Kokkos_OpenMP_Instance.cpp +ifneq ($(KOKKOS_INTERNAL_DISABLE_DEPRECATED_CODE), 1) Kokkos_OpenMP_Task.o: $(KOKKOS_CPP_DEPENDS) $(KOKKOS_PATH)/core/src/OpenMP/Kokkos_OpenMP_Task.cpp $(CXX) $(KOKKOS_CPPFLAGS) $(KOKKOS_CXXFLAGS) $(CXXFLAGS) -c $(KOKKOS_PATH)/core/src/OpenMP/Kokkos_OpenMP_Task.cpp endif +endif ifeq ($(KOKKOS_INTERNAL_USE_HPX), 1) Kokkos_HPX.o: $(KOKKOS_CPP_DEPENDS) $(KOKKOS_PATH)/core/src/HPX/Kokkos_HPX.cpp $(CXX) $(KOKKOS_CPPFLAGS) $(KOKKOS_CXXFLAGS) $(CXXFLAGS) -c $(KOKKOS_PATH)/core/src/HPX/Kokkos_HPX.cpp +ifneq ($(KOKKOS_INTERNAL_DISABLE_DEPRECATED_CODE), 1) Kokkos_HPX_Task.o: $(KOKKOS_CPP_DEPENDS) $(KOKKOS_PATH)/core/src/HPX/Kokkos_HPX_Task.cpp $(CXX) $(KOKKOS_CPPFLAGS) $(KOKKOS_CXXFLAGS) $(CXXFLAGS) -c $(KOKKOS_PATH)/core/src/HPX/Kokkos_HPX_Task.cpp endif +endif ifeq ($(KOKKOS_INTERNAL_USE_OPENMPTARGET), 1) -Kokkos_OpenMPTarget_Exec.o: $(KOKKOS_CPP_DEPENDS) $(KOKKOS_PATH)/core/src/OpenMPTarget/Kokkos_OpenMPTarget_Exec.cpp - $(CXX) $(KOKKOS_CPPFLAGS) $(KOKKOS_CXXFLAGS) $(CXXFLAGS) -c $(KOKKOS_PATH)/core/src/OpenMPTarget/Kokkos_OpenMPTarget_Exec.cpp Kokkos_OpenMPTarget_Instance.o: $(KOKKOS_CPP_DEPENDS) $(KOKKOS_PATH)/core/src/OpenMPTarget/Kokkos_OpenMPTarget_Instance.cpp $(CXX) $(KOKKOS_CPPFLAGS) $(KOKKOS_CXXFLAGS) $(CXXFLAGS) -c $(KOKKOS_PATH)/core/src/OpenMPTarget/Kokkos_OpenMPTarget_Instance.cpp Kokkos_OpenMPTargetSpace.o: $(KOKKOS_CPP_DEPENDS) $(KOKKOS_PATH)/core/src/OpenMPTarget/Kokkos_OpenMPTargetSpace.cpp $(CXX) $(KOKKOS_CPPFLAGS) $(KOKKOS_CXXFLAGS) $(CXXFLAGS) -c $(KOKKOS_PATH)/core/src/OpenMPTarget/Kokkos_OpenMPTargetSpace.cpp -Kokkos_OpenMPTarget_Task.o: $(KOKKOS_CPP_DEPENDS) $(KOKKOS_PATH)/core/src/OpenMPTarget/Kokkos_OpenMPTarget_Task.cpp - $(CXX) $(KOKKOS_CPPFLAGS) $(KOKKOS_CXXFLAGS) $(CXXFLAGS) -c $(KOKKOS_PATH)/core/src/OpenMPTarget/Kokkos_OpenMPTarget_Task.cpp endif ifeq ($(KOKKOS_INTERNAL_USE_OPENACC), 1) diff --git a/lib/kokkos/README.md b/lib/kokkos/README.md index c8c6f8f7cf..56159b35c2 100644 --- a/lib/kokkos/README.md +++ b/lib/kokkos/README.md @@ -30,12 +30,12 @@ To start learning about Kokkos: The latest release of Kokkos can be obtained from the [GitHub releases page](https://github.com/kokkos/kokkos/releases/latest). -The current release is [4.3.01](https://github.com/kokkos/kokkos/releases/tag/4.3.01). +The current release is [4.5.01](https://github.com/kokkos/kokkos/releases/tag/4.5.01). ```bash -curl -OJ -L https://github.com/kokkos/kokkos/archive/refs/tags/4.3.01.tar.gz +curl -OJ -L https://github.com/kokkos/kokkos/releases/download/4.5.01/kokkos-4.5.01.tar.gz # Or with wget -wget https://github.com/kokkos/kokkos/archive/refs/tags/4.3.01.tar.gz +wget https://github.com/kokkos/kokkos/releases/download/4.5.01/kokkos-4.5.01.tar.gz ``` To clone the latest development version of Kokkos from GitHub: diff --git a/lib/kokkos/algorithms/CMakeLists.txt b/lib/kokkos/algorithms/CMakeLists.txt index 368984647e..73ce9f7ec5 100644 --- a/lib/kokkos/algorithms/CMakeLists.txt +++ b/lib/kokkos/algorithms/CMakeLists.txt @@ -1,7 +1,7 @@ -IF (NOT Kokkos_INSTALL_TESTING) - ADD_SUBDIRECTORY(src) -ENDIF() +if(NOT Kokkos_INSTALL_TESTING) + add_subdirectory(src) +endif() # FIXME_OPENACC: temporarily disabled due to unimplemented features -IF(NOT ((KOKKOS_ENABLE_OPENMPTARGET AND KOKKOS_CXX_COMPILER_ID STREQUAL NVHPC) OR KOKKOS_ENABLE_OPENACC)) - KOKKOS_ADD_TEST_DIRECTORIES(unit_tests) -ENDIF() +if(NOT ((KOKKOS_ENABLE_OPENMPTARGET AND KOKKOS_CXX_COMPILER_ID STREQUAL NVHPC) OR KOKKOS_ENABLE_OPENACC)) + kokkos_add_test_directories(unit_tests) +endif() diff --git a/lib/kokkos/algorithms/src/CMakeLists.txt b/lib/kokkos/algorithms/src/CMakeLists.txt index b490caca62..9f10b85e02 100644 --- a/lib/kokkos/algorithms/src/CMakeLists.txt +++ b/lib/kokkos/algorithms/src/CMakeLists.txt @@ -1,34 +1,29 @@ #I have to leave these here for tribits -KOKKOS_INCLUDE_DIRECTORIES(${CMAKE_CURRENT_BINARY_DIR}) -KOKKOS_INCLUDE_DIRECTORIES(${CMAKE_CURRENT_SOURCE_DIR}) +kokkos_include_directories(${CMAKE_CURRENT_BINARY_DIR}) +kokkos_include_directories(${CMAKE_CURRENT_SOURCE_DIR}) #----------------------------------------------------------------------------- -FILE(GLOB ALGO_HEADERS *.hpp) -FILE(GLOB ALGO_SOURCES *.cpp) -APPEND_GLOB(ALGO_HEADERS ${CMAKE_CURRENT_SOURCE_DIR}/std_algorithms/*.hpp) -APPEND_GLOB(ALGO_HEADERS ${CMAKE_CURRENT_SOURCE_DIR}/std_algorithms/impl/*.hpp) +file(GLOB ALGO_HEADERS *.hpp) +file(GLOB ALGO_SOURCES *.cpp) +append_glob(ALGO_HEADERS ${CMAKE_CURRENT_SOURCE_DIR}/std_algorithms/*.hpp) +append_glob(ALGO_HEADERS ${CMAKE_CURRENT_SOURCE_DIR}/std_algorithms/impl/*.hpp) -INSTALL ( +install( DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/" DESTINATION ${KOKKOS_HEADER_DIR} - FILES_MATCHING PATTERN "*.hpp" + FILES_MATCHING + PATTERN "*.hpp" ) #----------------------------------------------------------------------------- # We have to pass the sources in here for Tribits # These will get ignored for standalone CMake and a true interface library made -KOKKOS_ADD_INTERFACE_LIBRARY( - kokkosalgorithms - NOINSTALLHEADERS ${ALGO_HEADERS} - SOURCES ${ALGO_SOURCES} -) -KOKKOS_LIB_INCLUDE_DIRECTORIES(kokkosalgorithms - ${KOKKOS_TOP_BUILD_DIR} - ${CMAKE_CURRENT_BINARY_DIR} - ${CMAKE_CURRENT_SOURCE_DIR} +kokkos_add_interface_library(kokkosalgorithms NOINSTALLHEADERS ${ALGO_HEADERS} SOURCES ${ALGO_SOURCES}) +kokkos_lib_include_directories( + kokkosalgorithms ${KOKKOS_TOP_BUILD_DIR} ${CMAKE_CURRENT_BINARY_DIR} ${CMAKE_CURRENT_SOURCE_DIR} ) -KOKKOS_LINK_TPL(kokkoscontainers PUBLIC ROCTHRUST) -KOKKOS_LINK_TPL(kokkoscore PUBLIC ONEDPL) +kokkos_link_tpl(kokkoscontainers PUBLIC ROCTHRUST) +kokkos_link_tpl(kokkoscore PUBLIC ONEDPL) diff --git a/lib/kokkos/algorithms/src/Kokkos_Random.hpp b/lib/kokkos/algorithms/src/Kokkos_Random.hpp index 7df12b8518..b28ea4c2ca 100644 --- a/lib/kokkos/algorithms/src/Kokkos_Random.hpp +++ b/lib/kokkos/algorithms/src/Kokkos_Random.hpp @@ -615,7 +615,7 @@ template struct Random_UniqueIndex { using locks_view_type = View; KOKKOS_FUNCTION - static int get_state_idx(const locks_view_type) { + static int get_state_idx(const locks_view_type&) { KOKKOS_IF_ON_HOST( (return DeviceType::execution_space::impl_hardware_thread_id();)) @@ -665,17 +665,16 @@ struct Random_UniqueIndex< #ifdef KOKKOS_ENABLE_SYCL template -struct Random_UniqueIndex< - Kokkos::Device> { +struct Random_UniqueIndex> { using locks_view_type = - View>; + View>; KOKKOS_FUNCTION static int get_state_idx(const locks_view_type& locks_) { auto item = sycl::ext::oneapi::experimental::this_nd_item<3>(); std::size_t threadIdx[3] = {item.get_local_id(2), item.get_local_id(1), item.get_local_id(0)}; std::size_t blockIdx[3] = {item.get_group(2), item.get_group(1), - item.get_group(0)}; + item.get_group(0)}; std::size_t blockDim[3] = {item.get_local_range(2), item.get_local_range(1), item.get_local_range(0)}; std::size_t gridDim[3] = { @@ -1121,7 +1120,7 @@ class Random_XorShift1024_Pool { using execution_space = typename device_type::execution_space; using locks_type = View; using int_view_type = View; - using state_data_type = View; + using state_data_type = View; locks_type locks_ = {}; state_data_type state_ = {}; diff --git a/lib/kokkos/algorithms/src/sorting/Kokkos_BinOpsPublicAPI.hpp b/lib/kokkos/algorithms/src/sorting/Kokkos_BinOpsPublicAPI.hpp index 73e751f572..8e7de32a07 100644 --- a/lib/kokkos/algorithms/src/sorting/Kokkos_BinOpsPublicAPI.hpp +++ b/lib/kokkos/algorithms/src/sorting/Kokkos_BinOpsPublicAPI.hpp @@ -35,11 +35,11 @@ struct BinOp1D { #endif // Construct BinOp with number of bins, minimum value and maximum value - BinOp1D(int max_bins__, typename KeyViewType::const_value_type min, + BinOp1D(int max_bins, typename KeyViewType::const_value_type min, typename KeyViewType::const_value_type max) - : max_bins_(max_bins__ + 1), + : max_bins_(max_bins + 1), // Cast to double to avoid possible overflow when using integer - mul_(static_cast(max_bins__) / + mul_(static_cast(max_bins) / (static_cast(max) - static_cast(min))), min_(static_cast(min)) { // For integral types the number of bins may be larger than the range @@ -47,7 +47,7 @@ struct BinOp1D { // and then don't need to sort bins. if (std::is_integral::value && (static_cast(max) - static_cast(min)) <= - static_cast(max_bins__)) { + static_cast(max_bins)) { mul_ = 1.; } } @@ -82,16 +82,16 @@ struct BinOp3D { BinOp3D() = delete; #endif - BinOp3D(int max_bins__[], typename KeyViewType::const_value_type min[], + BinOp3D(int max_bins[], typename KeyViewType::const_value_type min[], typename KeyViewType::const_value_type max[]) { - max_bins_[0] = max_bins__[0]; - max_bins_[1] = max_bins__[1]; - max_bins_[2] = max_bins__[2]; - mul_[0] = static_cast(max_bins__[0]) / + max_bins_[0] = max_bins[0]; + max_bins_[1] = max_bins[1]; + max_bins_[2] = max_bins[2]; + mul_[0] = static_cast(max_bins[0]) / (static_cast(max[0]) - static_cast(min[0])); - mul_[1] = static_cast(max_bins__[1]) / + mul_[1] = static_cast(max_bins[1]) / (static_cast(max[1]) - static_cast(min[1])); - mul_[2] = static_cast(max_bins__[2]) / + mul_[2] = static_cast(max_bins[2]) / (static_cast(max[2]) - static_cast(min[2])); min_[0] = static_cast(min[0]); min_[1] = static_cast(min[1]); diff --git a/lib/kokkos/algorithms/src/sorting/Kokkos_BinSortPublicAPI.hpp b/lib/kokkos/algorithms/src/sorting/Kokkos_BinSortPublicAPI.hpp index c399279fe4..f417b6b13b 100644 --- a/lib/kokkos/algorithms/src/sorting/Kokkos_BinSortPublicAPI.hpp +++ b/lib/kokkos/algorithms/src/sorting/Kokkos_BinSortPublicAPI.hpp @@ -388,7 +388,8 @@ class BinSort { // reasonable experimentally. if (use_std_sort && bin_size > 10) { KOKKOS_IF_ON_HOST( - (std::sort(&sort_order(lower_bound), &sort_order(upper_bound), + (std::sort(sort_order.data() + lower_bound, + sort_order.data() + upper_bound, [this](int p, int q) { return bin_op(keys_rnd, p, q); });)) } else { for (int k = lower_bound + 1; k < upper_bound; ++k) { diff --git a/lib/kokkos/algorithms/src/sorting/Kokkos_SortPublicAPI.hpp b/lib/kokkos/algorithms/src/sorting/Kokkos_SortPublicAPI.hpp index 308e9e3a00..20026c77e4 100644 --- a/lib/kokkos/algorithms/src/sorting/Kokkos_SortPublicAPI.hpp +++ b/lib/kokkos/algorithms/src/sorting/Kokkos_SortPublicAPI.hpp @@ -53,9 +53,13 @@ void sort(const ExecutionSpace& exec, if constexpr (Impl::better_off_calling_std_sort_v) { exec.fence("Kokkos::sort without comparator use std::sort"); - auto first = ::Kokkos::Experimental::begin(view); - auto last = ::Kokkos::Experimental::end(view); - std::sort(first, last); + if (view.span_is_contiguous()) { + std::sort(view.data(), view.data() + view.size()); + } else { + auto first = ::Kokkos::Experimental::begin(view); + auto last = ::Kokkos::Experimental::end(view); + std::sort(first, last); + } } else { Impl::sort_device_view_without_comparator(exec, view); } @@ -107,9 +111,13 @@ void sort(const ExecutionSpace& exec, if constexpr (Impl::better_off_calling_std_sort_v) { exec.fence("Kokkos::sort with comparator use std::sort"); - auto first = ::Kokkos::Experimental::begin(view); - auto last = ::Kokkos::Experimental::end(view); - std::sort(first, last, comparator); + if (view.span_is_contiguous()) { + std::sort(view.data(), view.data() + view.size(), comparator); + } else { + auto first = ::Kokkos::Experimental::begin(view); + auto last = ::Kokkos::Experimental::end(view); + std::sort(first, last, comparator); + } } else { Impl::sort_device_view_with_comparator(exec, view, comparator); } diff --git a/lib/kokkos/algorithms/src/sorting/impl/Kokkos_SortByKeyImpl.hpp b/lib/kokkos/algorithms/src/sorting/impl/Kokkos_SortByKeyImpl.hpp index f11f807048..2a8f761d9b 100644 --- a/lib/kokkos/algorithms/src/sorting/impl/Kokkos_SortByKeyImpl.hpp +++ b/lib/kokkos/algorithms/src/sorting/impl/Kokkos_SortByKeyImpl.hpp @@ -30,6 +30,7 @@ #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wshadow" +#pragma GCC diagnostic ignored "-Wsuggest-override" #if defined(KOKKOS_COMPILER_CLANG) // Some versions of Clang fail to compile Thrust, failing with errors like @@ -76,13 +77,10 @@ namespace Kokkos::Impl { template constexpr inline bool is_admissible_to_kokkos_sort_by_key = - ::Kokkos::is_view::value&& T::rank() == 1 && - (std::is_same::value || - std::is_same::value || - std::is_same::value); + ::Kokkos::is_view::value && T::rank() == 1 && + (std::is_same_v || + std::is_same_v || + std::is_same_v); template KOKKOS_INLINE_FUNCTION constexpr void @@ -144,7 +142,7 @@ void sort_by_key_rocthrust( #if defined(KOKKOS_ENABLE_ONEDPL) template -inline constexpr bool sort_on_device_v = +inline constexpr bool sort_on_device_v = std::is_same_v || std::is_same_v; @@ -152,7 +150,7 @@ inline constexpr bool sort_on_device_v = template void sort_by_key_onedpl( - const Kokkos::Experimental::SYCL& exec, + const Kokkos::SYCL& exec, const Kokkos::View& keys, const Kokkos::View& values, MaybeComparator&&... maybeComparator) { @@ -176,7 +174,7 @@ template void applyPermutation(const ExecutionSpace& space, const PermutationView& permutation, const ViewType& view) { - static_assert(std::is_integral::value); + static_assert(std::is_integral_v); auto view_copy = Kokkos::create_mirror( Kokkos::view_alloc(space, typename ExecutionSpace::memory_space{}, @@ -335,7 +333,7 @@ void sort_by_key_device_view_without_comparator( template void sort_by_key_device_view_without_comparator( - const Kokkos::Experimental::SYCL& exec, + const Kokkos::SYCL& exec, const Kokkos::View& keys, const Kokkos::View& values) { #ifdef KOKKOS_ONEDPL_HAS_SORT_BY_KEY @@ -392,7 +390,7 @@ void sort_by_key_device_view_with_comparator( template void sort_by_key_device_view_with_comparator( - const Kokkos::Experimental::SYCL& exec, + const Kokkos::SYCL& exec, const Kokkos::View& keys, const Kokkos::View& values, const ComparatorType& comparator) { diff --git a/lib/kokkos/algorithms/src/sorting/impl/Kokkos_SortImpl.hpp b/lib/kokkos/algorithms/src/sorting/impl/Kokkos_SortImpl.hpp index 0894622891..734ce450f6 100644 --- a/lib/kokkos/algorithms/src/sorting/impl/Kokkos_SortImpl.hpp +++ b/lib/kokkos/algorithms/src/sorting/impl/Kokkos_SortImpl.hpp @@ -34,6 +34,7 @@ #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wshadow" +#pragma GCC diagnostic ignored "-Wsuggest-override" #if defined(KOKKOS_COMPILER_CLANG) // Some versions of Clang fail to compile Thrust, failing with errors like @@ -146,7 +147,7 @@ void sort_via_binsort(const ExecutionSpace& exec, bool sort_in_bins = true; // TODO: figure out better max_bins then this ... int64_t max_bins = view.extent(0) / 2; - if (std::is_integral::value) { + if (std::is_integral_v) { // Cast to double to avoid possible overflow when using integer auto const max_val = static_cast(result.max_val); auto const min_val = static_cast(result.min_val); @@ -157,7 +158,7 @@ void sort_via_binsort(const ExecutionSpace& exec, sort_in_bins = false; } } - if (std::is_floating_point::value) { + if (std::is_floating_point_v) { KOKKOS_ASSERT(std::isfinite(static_cast(result.max_val) - static_cast(result.min_val))); } @@ -211,11 +212,11 @@ void sort_rocthrust(const HIP& space, #if defined(KOKKOS_ENABLE_ONEDPL) template -void sort_onedpl(const Kokkos::Experimental::SYCL& space, +void sort_onedpl(const Kokkos::SYCL& space, const Kokkos::View& view, MaybeComparator&&... maybeComparator) { using ViewType = Kokkos::View; - static_assert(SpaceAccessibility::accessible, "SYCL execution space is not able to access the memory space " "of the View argument!"); @@ -268,19 +269,29 @@ void copy_to_host_run_stdsort_copy_back( KE::copy(exec, view, view_dc); // run sort on the mirror of view_dc - auto mv_h = create_mirror_view_and_copy(Kokkos::HostSpace(), view_dc); - auto first = KE::begin(mv_h); - auto last = KE::end(mv_h); - std::sort(first, last, std::forward(maybeComparator)...); + auto mv_h = create_mirror_view_and_copy(Kokkos::HostSpace(), view_dc); + if (view.span_is_contiguous()) { + std::sort(mv_h.data(), mv_h.data() + mv_h.size(), + std::forward(maybeComparator)...); + } else { + auto first = KE::begin(mv_h); + auto last = KE::end(mv_h); + std::sort(first, last, std::forward(maybeComparator)...); + } Kokkos::deep_copy(exec, view_dc, mv_h); // copy back to argument view KE::copy(exec, KE::cbegin(view_dc), KE::cend(view_dc), KE::begin(view)); } else { auto view_h = create_mirror_view_and_copy(Kokkos::HostSpace(), view); - auto first = KE::begin(view_h); - auto last = KE::end(view_h); - std::sort(first, last, std::forward(maybeComparator)...); + if (view.span_is_contiguous()) { + std::sort(view_h.data(), view_h.data() + view_h.size(), + std::forward(maybeComparator)...); + } else { + auto first = KE::begin(view_h); + auto last = KE::end(view_h); + std::sort(first, last, std::forward(maybeComparator)...); + } Kokkos::deep_copy(exec, view, view_h); } } @@ -310,7 +321,7 @@ void sort_device_view_without_comparator( #if defined(KOKKOS_ENABLE_ONEDPL) template void sort_device_view_without_comparator( - const Kokkos::Experimental::SYCL& exec, + const Kokkos::SYCL& exec, const Kokkos::View& view) { using ViewType = Kokkos::View; static_assert( @@ -365,8 +376,7 @@ void sort_device_view_with_comparator( #if defined(KOKKOS_ENABLE_ONEDPL) template void sort_device_view_with_comparator( - const Kokkos::Experimental::SYCL& exec, - const Kokkos::View& view, + const Kokkos::SYCL& exec, const Kokkos::View& view, const ComparatorType& comparator) { using ViewType = Kokkos::View; static_assert( @@ -397,12 +407,12 @@ sort_device_view_with_comparator( // and then copies data back. Potentially, this can later be changed // with a better solution like our own quicksort on device or similar. - using ViewType = Kokkos::View; - using MemSpace = typename ViewType::memory_space; // Note with HIP unified memory this code path is still the right thing to do // if we end up here when RocThrust is not enabled. // The create_mirror_view_and_copy will do the right thing (no copy). -#ifndef KOKKOS_ENABLE_IMPL_HIP_UNIFIED_MEMORY +#ifndef KOKKOS_IMPL_HIP_UNIFIED_MEMORY + using ViewType = Kokkos::View; + using MemSpace = typename ViewType::memory_space; static_assert(!SpaceAccessibility::accessible, "Impl::sort_device_view_with_comparator: should not be called " "on a view that is already accessible on the host"); diff --git a/lib/kokkos/algorithms/src/std_algorithms/Kokkos_Reduce.hpp b/lib/kokkos/algorithms/src/std_algorithms/Kokkos_Reduce.hpp index b84f00f8bb..ea7e55ca61 100644 --- a/lib/kokkos/algorithms/src/std_algorithms/Kokkos_Reduce.hpp +++ b/lib/kokkos/algorithms/src/std_algorithms/Kokkos_Reduce.hpp @@ -91,7 +91,7 @@ template = 0> ValueType reduce(const ExecutionSpace& ex, IteratorType first, IteratorType last, ValueType init_reduction_value) { - static_assert(std::is_move_constructible::value, + static_assert(std::is_move_constructible_v, "ValueType must be move constructible."); return Impl::reduce_default_functors_exespace_impl( @@ -105,7 +105,7 @@ template ::value, + static_assert(std::is_move_constructible_v, "ValueType must be move constructible."); return Impl::reduce_default_functors_exespace_impl(label, ex, first, last, @@ -119,7 +119,7 @@ template & view, ValueType init_reduction_value) { - static_assert(std::is_move_constructible::value, + static_assert(std::is_move_constructible_v, "ValueType must be move constructible."); namespace KE = ::Kokkos::Experimental; @@ -137,7 +137,7 @@ template & view, ValueType init_reduction_value) { - static_assert(std::is_move_constructible::value, + static_assert(std::is_move_constructible_v, "ValueType must be move constructible."); namespace KE = ::Kokkos::Experimental; @@ -157,7 +157,7 @@ template ::value, + static_assert(std::is_move_constructible_v, "ValueType must be move constructible."); return Impl::reduce_custom_functors_exespace_impl( @@ -172,7 +172,7 @@ template ::value, + static_assert(std::is_move_constructible_v, "ValueType must be move constructible."); return Impl::reduce_custom_functors_exespace_impl( @@ -186,7 +186,7 @@ template & view, ValueType init_reduction_value, BinaryOp joiner) { - static_assert(std::is_move_constructible::value, + static_assert(std::is_move_constructible_v, "ValueType must be move constructible."); namespace KE = ::Kokkos::Experimental; @@ -204,7 +204,7 @@ template & view, ValueType init_reduction_value, BinaryOp joiner) { - static_assert(std::is_move_constructible::value, + static_assert(std::is_move_constructible_v, "ValueType must be move constructible."); namespace KE = ::Kokkos::Experimental; @@ -258,7 +258,7 @@ template < KOKKOS_FUNCTION ValueType reduce(const TeamHandleType& teamHandle, IteratorType first, IteratorType last, ValueType init_reduction_value) { - static_assert(std::is_move_constructible::value, + static_assert(std::is_move_constructible_v, "ValueType must be move constructible."); return Impl::reduce_default_functors_team_impl(teamHandle, first, last, @@ -273,7 +273,7 @@ KOKKOS_FUNCTION ValueType reduce(const TeamHandleType& teamHandle, const ::Kokkos::View& view, ValueType init_reduction_value) { - static_assert(std::is_move_constructible::value, + static_assert(std::is_move_constructible_v, "ValueType must be move constructible."); namespace KE = ::Kokkos::Experimental; @@ -294,7 +294,7 @@ KOKKOS_FUNCTION ValueType reduce(const TeamHandleType& teamHandle, IteratorType first, IteratorType last, ValueType init_reduction_value, BinaryOp joiner) { - static_assert(std::is_move_constructible::value, + static_assert(std::is_move_constructible_v, "ValueType must be move constructible."); return Impl::reduce_custom_functors_team_impl(teamHandle, first, last, @@ -309,7 +309,7 @@ KOKKOS_FUNCTION ValueType reduce(const TeamHandleType& teamHandle, const ::Kokkos::View& view, ValueType init_reduction_value, BinaryOp joiner) { - static_assert(std::is_move_constructible::value, + static_assert(std::is_move_constructible_v, "ValueType must be move constructible."); namespace KE = ::Kokkos::Experimental; diff --git a/lib/kokkos/algorithms/src/std_algorithms/Kokkos_TransformReduce.hpp b/lib/kokkos/algorithms/src/std_algorithms/Kokkos_TransformReduce.hpp index 101f5113f6..89585ddbea 100644 --- a/lib/kokkos/algorithms/src/std_algorithms/Kokkos_TransformReduce.hpp +++ b/lib/kokkos/algorithms/src/std_algorithms/Kokkos_TransformReduce.hpp @@ -117,7 +117,7 @@ ValueType transform_reduce(const ExecutionSpace& ex, IteratorType1 first1, ValueType init_reduction_value, BinaryJoinerType joiner, BinaryTransform transformer) { - static_assert(std::is_move_constructible::value, + static_assert(std::is_move_constructible_v, "ValueType must be move constructible."); return Impl::transform_reduce_custom_functors_exespace_impl( @@ -136,7 +136,7 @@ ValueType transform_reduce(const std::string& label, const ExecutionSpace& ex, IteratorType2 first2, ValueType init_reduction_value, BinaryJoinerType joiner, BinaryTransform transformer) { - static_assert(std::is_move_constructible::value, + static_assert(std::is_move_constructible_v, "ValueType must be move constructible."); return Impl::transform_reduce_custom_functors_exespace_impl( @@ -157,7 +157,7 @@ ValueType transform_reduce( ValueType init_reduction_value, BinaryJoinerType joiner, BinaryTransform transformer) { namespace KE = ::Kokkos::Experimental; - static_assert(std::is_move_constructible::value, + static_assert(std::is_move_constructible_v, "ValueType must be move constructible."); Impl::static_assert_is_admissible_to_kokkos_std_algorithms(first_view); @@ -182,7 +182,7 @@ ValueType transform_reduce( ValueType init_reduction_value, BinaryJoinerType joiner, BinaryTransform transformer) { namespace KE = ::Kokkos::Experimental; - static_assert(std::is_move_constructible::value, + static_assert(std::is_move_constructible_v, "ValueType must be move constructible."); Impl::static_assert_is_admissible_to_kokkos_std_algorithms(first_view); @@ -208,7 +208,7 @@ ValueType transform_reduce(const ExecutionSpace& ex, IteratorType first1, IteratorType last1, ValueType init_reduction_value, BinaryJoinerType joiner, UnaryTransform transformer) { - static_assert(std::is_move_constructible::value, + static_assert(std::is_move_constructible_v, "ValueType must be move constructible."); return Impl::transform_reduce_custom_functors_exespace_impl( @@ -228,7 +228,7 @@ ValueType transform_reduce(const std::string& label, const ExecutionSpace& ex, ValueType init_reduction_value, BinaryJoinerType joiner, UnaryTransform transformer) { - static_assert(std::is_move_constructible::value, + static_assert(std::is_move_constructible_v, "ValueType must be move constructible."); return Impl::transform_reduce_custom_functors_exespace_impl( @@ -248,7 +248,7 @@ ValueType transform_reduce(const ExecutionSpace& ex, BinaryJoinerType joiner, UnaryTransform transformer) { namespace KE = ::Kokkos::Experimental; - static_assert(std::is_move_constructible::value, + static_assert(std::is_move_constructible_v, "ValueType must be move constructible."); Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view); @@ -270,7 +270,7 @@ ValueType transform_reduce(const std::string& label, const ExecutionSpace& ex, BinaryJoinerType joiner, UnaryTransform transformer) { namespace KE = ::Kokkos::Experimental; - static_assert(std::is_move_constructible::value, + static_assert(std::is_move_constructible_v, "ValueType must be move constructible."); Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view); @@ -345,7 +345,7 @@ KOKKOS_FUNCTION ValueType transform_reduce( const TeamHandleType& teamHandle, IteratorType1 first1, IteratorType1 last1, IteratorType2 first2, ValueType init_reduction_value, BinaryJoinerType joiner, BinaryTransform transformer) { - static_assert(std::is_move_constructible::value, + static_assert(std::is_move_constructible_v, "ValueType must be move constructible."); return Impl::transform_reduce_custom_functors_team_impl( @@ -366,7 +366,7 @@ transform_reduce(const TeamHandleType& teamHandle, ValueType init_reduction_value, BinaryJoinerType joiner, BinaryTransform transformer) { namespace KE = ::Kokkos::Experimental; - static_assert(std::is_move_constructible::value, + static_assert(std::is_move_constructible_v, "ValueType must be move constructible."); Impl::static_assert_is_admissible_to_kokkos_std_algorithms(first_view); @@ -393,7 +393,7 @@ KOKKOS_FUNCTION ValueType transform_reduce(const TeamHandleType& teamHandle, ValueType init_reduction_value, BinaryJoinerType joiner, UnaryTransform transformer) { - static_assert(std::is_move_constructible::value, + static_assert(std::is_move_constructible_v, "ValueType must be move constructible."); return Impl::transform_reduce_custom_functors_team_impl( @@ -412,7 +412,7 @@ transform_reduce(const TeamHandleType& teamHandle, ValueType init_reduction_value, BinaryJoinerType joiner, UnaryTransform transformer) { namespace KE = ::Kokkos::Experimental; - static_assert(std::is_move_constructible::value, + static_assert(std::is_move_constructible_v, "ValueType must be move constructible."); Impl::static_assert_is_admissible_to_kokkos_std_algorithms(view); diff --git a/lib/kokkos/algorithms/src/std_algorithms/impl/Kokkos_Constraints.hpp b/lib/kokkos/algorithms/src/std_algorithms/impl/Kokkos_Constraints.hpp index 54bb13e25b..da16141f5a 100644 --- a/lib/kokkos/algorithms/src/std_algorithms/impl/Kokkos_Constraints.hpp +++ b/lib/kokkos/algorithms/src/std_algorithms/impl/Kokkos_Constraints.hpp @@ -33,12 +33,12 @@ struct is_admissible_to_kokkos_std_algorithms : std::false_type {}; template struct is_admissible_to_kokkos_std_algorithms< T, std::enable_if_t<::Kokkos::is_view::value && T::rank() == 1 && - (std::is_same::value || - std::is_same::value || - std::is_same::value)>> + (std::is_same_v || + std::is_same_v || + std::is_same_v)>> : std::true_type {}; template @@ -102,8 +102,8 @@ struct are_random_access_iterators; template struct are_random_access_iterators { static constexpr bool value = - is_iterator_v && std::is_base_of::value; + is_iterator_v && std::is_base_of_v; }; template @@ -165,9 +165,8 @@ struct iterators_have_matching_difference_type { template struct iterators_have_matching_difference_type { - static constexpr bool value = - std::is_same::value; + static constexpr bool value = std::is_same_v; }; template diff --git a/lib/kokkos/algorithms/src/std_algorithms/impl/Kokkos_MoveBackward.hpp b/lib/kokkos/algorithms/src/std_algorithms/impl/Kokkos_MoveBackward.hpp index 9075562d46..dc910861d5 100644 --- a/lib/kokkos/algorithms/src/std_algorithms/impl/Kokkos_MoveBackward.hpp +++ b/lib/kokkos/algorithms/src/std_algorithms/impl/Kokkos_MoveBackward.hpp @@ -30,7 +30,7 @@ namespace Impl { template struct StdMoveBackwardFunctor { using index_type = typename IteratorType1::difference_type; - static_assert(std::is_signed::value, + static_assert(std::is_signed_v, "Kokkos: StdMoveBackwardFunctor requires signed index type"); IteratorType1 m_last; diff --git a/lib/kokkos/algorithms/src/std_algorithms/impl/Kokkos_RandomAccessIterator.hpp b/lib/kokkos/algorithms/src/std_algorithms/impl/Kokkos_RandomAccessIterator.hpp index 5bce89e98f..e8c638c94c 100644 --- a/lib/kokkos/algorithms/src/std_algorithms/impl/Kokkos_RandomAccessIterator.hpp +++ b/lib/kokkos/algorithms/src/std_algorithms/impl/Kokkos_RandomAccessIterator.hpp @@ -36,18 +36,18 @@ class RandomAccessIterator< ::Kokkos::View > { using iterator_type = RandomAccessIterator; using iterator_category = std::random_access_iterator_tag; - using value_type = typename view_type::value_type; + using value_type = typename view_type::non_const_value_type; using difference_type = ptrdiff_t; using pointer = typename view_type::pointer_type; using reference = typename view_type::reference_type; static_assert(view_type::rank == 1 && - (std::is_same::value || - std::is_same::value || - std::is_same::value), + (std::is_same_v || + std::is_same_v || + std::is_same_v), "RandomAccessIterator only supports 1D Views with LayoutLeft, " "LayoutRight, LayoutStride."); @@ -61,9 +61,9 @@ class RandomAccessIterator< ::Kokkos::View > { #ifndef KOKKOS_ENABLE_CXX17 // C++20 and beyond template - requires(std::is_constructible_v) KOKKOS_FUNCTION - explicit(!std::is_convertible_v) - RandomAccessIterator(const RandomAccessIterator& other) + requires(std::is_constructible_v) + KOKKOS_FUNCTION explicit(!std::is_convertible_v) + RandomAccessIterator(const RandomAccessIterator& other) : m_view(other.m_view), m_current_index(other.m_current_index) {} #else template < diff --git a/lib/kokkos/algorithms/src/std_algorithms/impl/Kokkos_Reverse.hpp b/lib/kokkos/algorithms/src/std_algorithms/impl/Kokkos_Reverse.hpp index b4046c7645..e6caa07288 100644 --- a/lib/kokkos/algorithms/src/std_algorithms/impl/Kokkos_Reverse.hpp +++ b/lib/kokkos/algorithms/src/std_algorithms/impl/Kokkos_Reverse.hpp @@ -30,7 +30,7 @@ namespace Impl { template struct StdReverseFunctor { using index_type = typename InputIterator::difference_type; - static_assert(std::is_signed::value, + static_assert(std::is_signed_v, "Kokkos: StdReverseFunctor requires signed index type"); InputIterator m_first; diff --git a/lib/kokkos/algorithms/src/std_algorithms/impl/Kokkos_ReverseCopy.hpp b/lib/kokkos/algorithms/src/std_algorithms/impl/Kokkos_ReverseCopy.hpp index dd20d90e39..7aa0e4fc44 100644 --- a/lib/kokkos/algorithms/src/std_algorithms/impl/Kokkos_ReverseCopy.hpp +++ b/lib/kokkos/algorithms/src/std_algorithms/impl/Kokkos_ReverseCopy.hpp @@ -30,7 +30,7 @@ namespace Impl { template struct StdReverseCopyFunctor { using index_type = typename InputIterator::difference_type; - static_assert(std::is_signed::value, + static_assert(std::is_signed_v, "Kokkos: StdReverseCopyFunctor requires signed index type"); InputIterator m_last; diff --git a/lib/kokkos/algorithms/unit_tests/CMakeLists.txt b/lib/kokkos/algorithms/unit_tests/CMakeLists.txt index db184bc8a9..31247af159 100644 --- a/lib/kokkos/algorithms/unit_tests/CMakeLists.txt +++ b/lib/kokkos/algorithms/unit_tests/CMakeLists.txt @@ -1,12 +1,10 @@ - #Leave these here for now - I don't need transitive deps anyway -KOKKOS_INCLUDE_DIRECTORIES(${CMAKE_CURRENT_BINARY_DIR}) -KOKKOS_INCLUDE_DIRECTORIES(REQUIRED_DURING_INSTALLATION_TESTING ${CMAKE_CURRENT_SOURCE_DIR}) -KOKKOS_INCLUDE_DIRECTORIES(${CMAKE_CURRENT_SOURCE_DIR}/../src ) -KOKKOS_INCLUDE_DIRECTORIES(${KOKKOS_SOURCE_DIR}/core/unit_test/category_files) +kokkos_include_directories(${CMAKE_CURRENT_BINARY_DIR}) +kokkos_include_directories(REQUIRED_DURING_INSTALLATION_TESTING ${CMAKE_CURRENT_SOURCE_DIR}) +kokkos_include_directories(${CMAKE_CURRENT_SOURCE_DIR}/../src) +kokkos_include_directories(${KOKKOS_SOURCE_DIR}/core/unit_test/category_files) - -SET(ALGORITHM UnitTestMain.cpp) +set(ALGORITHM UnitTestMain.cpp) foreach(Tag Threads;Serial;OpenMP;Cuda;HPX;HIP;SYCL;OpenMPTarget) string(TOUPPER ${Tag} DEVICE) @@ -23,21 +21,11 @@ foreach(Tag Threads;Serial;OpenMP;Cuda;HPX;HIP;SYCL;OpenMPTarget) # Generate a .cpp file for each one that runs it on the current backend (Tag), # and add this .cpp file to the sources for UnitTest_RandomAndSort. set(ALGO_SORT_SOURCES) - foreach(SOURCE_Input - TestSort - TestSortByKey - TestSortCustomComp - TestBinSortA - TestBinSortB - TestNestedSort - ) + foreach(SOURCE_Input TestSort TestSortByKey TestSortCustomComp TestBinSortA TestBinSortB TestNestedSort) set(file ${dir}/${SOURCE_Input}.cpp) # Write to a temporary intermediate file and call configure_file to avoid # updating timestamps triggering unnecessary rebuilds on subsequent cmake runs. - file(WRITE ${dir}/dummy.cpp - "#include \n" - "#include <${SOURCE_Input}.hpp>\n" - ) + file(WRITE ${dir}/dummy.cpp "#include \n" "#include <${SOURCE_Input}.hpp>\n") configure_file(${dir}/dummy.cpp ${file}) list(APPEND ALGO_SORT_SOURCES ${file}) endforeach() @@ -47,14 +35,9 @@ foreach(Tag Threads;Serial;OpenMP;Cuda;HPX;HIP;SYCL;OpenMPTarget) # ------------------------------------------ # do as above set(ALGO_RANDOM_SOURCES) - foreach(SOURCE_Input - TestRandom - ) + foreach(SOURCE_Input TestRandom) set(file ${dir}/${SOURCE_Input}.cpp) - file(WRITE ${dir}/dummy.cpp - "#include \n" - "#include <${SOURCE_Input}.hpp>\n" - ) + file(WRITE ${dir}/dummy.cpp "#include \n" "#include <${SOURCE_Input}.hpp>\n") configure_file(${dir}/dummy.cpp ${file}) list(APPEND ALGO_RANDOM_SOURCES ${file}) endforeach() @@ -65,11 +48,7 @@ endforeach() # std set A # ------------------------------------------ set(STDALGO_SOURCES_A) -foreach(Name - StdReducers - StdAlgorithmsConstraints - RandomAccessIterator - ) +foreach(Name StdReducers StdAlgorithmsConstraints RandomAccessIterator) list(APPEND STDALGO_SOURCES_A Test${Name}.cpp) endforeach() @@ -77,10 +56,7 @@ endforeach() # std set B # ------------------------------------------ set(STDALGO_SOURCES_B) -foreach(Name - StdAlgorithmsCommon - StdAlgorithmsMinMaxElementOps - ) +foreach(Name StdAlgorithmsCommon StdAlgorithmsMinMaxElementOps) list(APPEND STDALGO_SOURCES_B Test${Name}.cpp) endforeach() @@ -88,22 +64,23 @@ endforeach() # std set C # ------------------------------------------ set(STDALGO_SOURCES_C) -foreach(Name - StdAlgorithmsCommon - StdAlgorithmsLexicographicalCompare - StdAlgorithmsForEach - StdAlgorithmsFind - StdAlgorithmsFindFirstOf - StdAlgorithmsFindEnd - StdAlgorithmsCount - StdAlgorithmsEqual - StdAlgorithmsAllAnyNoneOf - StdAlgorithmsAdjacentFind - StdAlgorithmsSearch - StdAlgorithmsSearch_n - StdAlgorithmsMismatch - StdAlgorithmsMoveBackward - ) +foreach( + Name + StdAlgorithmsCommon + StdAlgorithmsLexicographicalCompare + StdAlgorithmsForEach + StdAlgorithmsFind + StdAlgorithmsFindFirstOf + StdAlgorithmsFindEnd + StdAlgorithmsCount + StdAlgorithmsEqual + StdAlgorithmsAllAnyNoneOf + StdAlgorithmsAdjacentFind + StdAlgorithmsSearch + StdAlgorithmsSearch_n + StdAlgorithmsMismatch + StdAlgorithmsMoveBackward +) list(APPEND STDALGO_SOURCES_C Test${Name}.cpp) endforeach() @@ -111,27 +88,28 @@ endforeach() # std set D # ------------------------------------------ set(STDALGO_SOURCES_D) -foreach(Name - StdAlgorithmsCommon - StdAlgorithmsModOps - StdAlgorithmsModSeqOps - StdAlgorithmsReplace - StdAlgorithmsReplaceIf - StdAlgorithmsReplaceCopy - StdAlgorithmsReplaceCopyIf - StdAlgorithmsCopyIf - StdAlgorithmsUnique - StdAlgorithmsUniqueCopy - StdAlgorithmsRemove - StdAlgorithmsRemoveIf - StdAlgorithmsRemoveCopy - StdAlgorithmsRemoveCopyIf - StdAlgorithmsRotate - StdAlgorithmsRotateCopy - StdAlgorithmsReverse - StdAlgorithmsShiftLeft - StdAlgorithmsShiftRight - ) +foreach( + Name + StdAlgorithmsCommon + StdAlgorithmsModOps + StdAlgorithmsModSeqOps + StdAlgorithmsReplace + StdAlgorithmsReplaceIf + StdAlgorithmsReplaceCopy + StdAlgorithmsReplaceCopyIf + StdAlgorithmsCopyIf + StdAlgorithmsUnique + StdAlgorithmsUniqueCopy + StdAlgorithmsRemove + StdAlgorithmsRemoveIf + StdAlgorithmsRemoveCopy + StdAlgorithmsRemoveCopyIf + StdAlgorithmsRotate + StdAlgorithmsRotateCopy + StdAlgorithmsReverse + StdAlgorithmsShiftLeft + StdAlgorithmsShiftRight +) list(APPEND STDALGO_SOURCES_D Test${Name}.cpp) endforeach() @@ -139,20 +117,21 @@ endforeach() # std set E # ------------------------------------------ set(STDALGO_SOURCES_E) -foreach(Name - StdAlgorithmsCommon - StdAlgorithmsIsSorted - StdAlgorithmsIsSortedUntil - StdAlgorithmsPartitioningOps - StdAlgorithmsPartitionCopy - StdAlgorithmsNumerics - StdAlgorithmsAdjacentDifference - StdAlgorithmsExclusiveScan - StdAlgorithmsInclusiveScan - StdAlgorithmsTransformUnaryOp - StdAlgorithmsTransformExclusiveScan - StdAlgorithmsTransformInclusiveScan - ) +foreach( + Name + StdAlgorithmsCommon + StdAlgorithmsIsSorted + StdAlgorithmsIsSortedUntil + StdAlgorithmsPartitioningOps + StdAlgorithmsPartitionCopy + StdAlgorithmsNumerics + StdAlgorithmsAdjacentDifference + StdAlgorithmsExclusiveScan + StdAlgorithmsInclusiveScan + StdAlgorithmsTransformUnaryOp + StdAlgorithmsTransformExclusiveScan + StdAlgorithmsTransformInclusiveScan +) list(APPEND STDALGO_SOURCES_E Test${Name}.cpp) endforeach() @@ -160,11 +139,7 @@ endforeach() # std team Q # ------------------------------------------ set(STDALGO_TEAM_SOURCES_Q) -foreach(Name - StdAlgorithmsCommon - StdAlgorithmsTeamInclusiveScan - StdAlgorithmsTeamTransformInclusiveScan - ) +foreach(Name StdAlgorithmsCommon StdAlgorithmsTeamInclusiveScan StdAlgorithmsTeamTransformInclusiveScan) list(APPEND STDALGO_TEAM_SOURCES_Q Test${Name}.cpp) endforeach() @@ -172,11 +147,7 @@ endforeach() # std team P # ------------------------------------------ set(STDALGO_TEAM_SOURCES_P) -foreach(Name - StdAlgorithmsCommon - StdAlgorithmsTeamExclusiveScan - StdAlgorithmsTeamTransformExclusiveScan - ) +foreach(Name StdAlgorithmsCommon StdAlgorithmsTeamExclusiveScan StdAlgorithmsTeamTransformExclusiveScan) list(APPEND STDALGO_TEAM_SOURCES_P Test${Name}.cpp) endforeach() @@ -184,14 +155,9 @@ endforeach() # std team M # ------------------------------------------ set(STDALGO_TEAM_SOURCES_M) -foreach(Name - StdAlgorithmsCommon - StdAlgorithmsTeamTransformUnaryOp - StdAlgorithmsTeamTransformBinaryOp - StdAlgorithmsTeamGenerate - StdAlgorithmsTeamGenerate_n - StdAlgorithmsTeamSwapRanges - ) +foreach(Name StdAlgorithmsCommon StdAlgorithmsTeamTransformUnaryOp StdAlgorithmsTeamTransformBinaryOp + StdAlgorithmsTeamGenerate StdAlgorithmsTeamGenerate_n StdAlgorithmsTeamSwapRanges +) list(APPEND STDALGO_TEAM_SOURCES_M Test${Name}.cpp) endforeach() @@ -199,14 +165,9 @@ endforeach() # std team L # ------------------------------------------ set(STDALGO_TEAM_SOURCES_L) -foreach(Name - StdAlgorithmsCommon - StdAlgorithmsTeamIsSorted - StdAlgorithmsTeamIsSortedUntil - StdAlgorithmsTeamIsPartitioned - StdAlgorithmsTeamPartitionCopy - StdAlgorithmsTeamPartitionPoint - ) +foreach(Name StdAlgorithmsCommon StdAlgorithmsTeamIsSorted StdAlgorithmsTeamIsSortedUntil + StdAlgorithmsTeamIsPartitioned StdAlgorithmsTeamPartitionCopy StdAlgorithmsTeamPartitionPoint +) list(APPEND STDALGO_TEAM_SOURCES_L Test${Name}.cpp) endforeach() @@ -214,13 +175,9 @@ endforeach() # std team I # ------------------------------------------ set(STDALGO_TEAM_SOURCES_I) -foreach(Name - StdAlgorithmsCommon - StdAlgorithmsTeamUnique - StdAlgorithmsTeamAdjacentDifference - StdAlgorithmsTeamReduce - StdAlgorithmsTeamTransformReduce - ) +foreach(Name StdAlgorithmsCommon StdAlgorithmsTeamUnique StdAlgorithmsTeamAdjacentDifference StdAlgorithmsTeamReduce + StdAlgorithmsTeamTransformReduce +) list(APPEND STDALGO_TEAM_SOURCES_I Test${Name}.cpp) endforeach() @@ -228,18 +185,19 @@ endforeach() # std team H # ------------------------------------------ set(STDALGO_TEAM_SOURCES_H) -foreach(Name - StdAlgorithmsCommon - StdAlgorithmsTeamCopy - StdAlgorithmsTeamCopy_n - StdAlgorithmsTeamCopyBackward - StdAlgorithmsTeamCopyIf - StdAlgorithmsTeamUniqueCopy - StdAlgorithmsTeamRemove - StdAlgorithmsTeamRemoveIf - StdAlgorithmsTeamRemoveCopy - StdAlgorithmsTeamRemoveCopyIf - ) +foreach( + Name + StdAlgorithmsCommon + StdAlgorithmsTeamCopy + StdAlgorithmsTeamCopy_n + StdAlgorithmsTeamCopyBackward + StdAlgorithmsTeamCopyIf + StdAlgorithmsTeamUniqueCopy + StdAlgorithmsTeamRemove + StdAlgorithmsTeamRemoveIf + StdAlgorithmsTeamRemoveCopy + StdAlgorithmsTeamRemoveCopyIf +) list(APPEND STDALGO_TEAM_SOURCES_H Test${Name}.cpp) endforeach() @@ -247,13 +205,9 @@ endforeach() # std team G # ------------------------------------------ set(STDALGO_TEAM_SOURCES_G) -foreach(Name - StdAlgorithmsCommon - StdAlgorithmsTeamMove - StdAlgorithmsTeamMoveBackward - StdAlgorithmsTeamShiftLeft - StdAlgorithmsTeamShiftRight - ) +foreach(Name StdAlgorithmsCommon StdAlgorithmsTeamMove StdAlgorithmsTeamMoveBackward StdAlgorithmsTeamShiftLeft + StdAlgorithmsTeamShiftRight +) list(APPEND STDALGO_TEAM_SOURCES_G Test${Name}.cpp) endforeach() @@ -261,13 +215,9 @@ endforeach() # std team F # ------------------------------------------ set(STDALGO_TEAM_SOURCES_F) -foreach(Name - StdAlgorithmsCommon - StdAlgorithmsTeamReverse - StdAlgorithmsTeamReverseCopy - StdAlgorithmsTeamRotate - StdAlgorithmsTeamRotateCopy - ) +foreach(Name StdAlgorithmsCommon StdAlgorithmsTeamReverse StdAlgorithmsTeamReverseCopy StdAlgorithmsTeamRotate + StdAlgorithmsTeamRotateCopy +) list(APPEND STDALGO_TEAM_SOURCES_F Test${Name}.cpp) endforeach() @@ -275,15 +225,16 @@ endforeach() # std team E # ------------------------------------------ set(STDALGO_TEAM_SOURCES_E) -foreach(Name - StdAlgorithmsCommon - StdAlgorithmsTeamFill - StdAlgorithmsTeamFill_n - StdAlgorithmsTeamReplace - StdAlgorithmsTeamReplaceIf - StdAlgorithmsTeamReplaceCopy - StdAlgorithmsTeamReplaceCopyIf - ) +foreach( + Name + StdAlgorithmsCommon + StdAlgorithmsTeamFill + StdAlgorithmsTeamFill_n + StdAlgorithmsTeamReplace + StdAlgorithmsTeamReplaceIf + StdAlgorithmsTeamReplaceCopy + StdAlgorithmsTeamReplaceCopyIf +) list(APPEND STDALGO_TEAM_SOURCES_E Test${Name}.cpp) endforeach() @@ -291,12 +242,7 @@ endforeach() # std team D # ------------------------------------------ set(STDALGO_TEAM_SOURCES_D) -foreach(Name - StdAlgorithmsCommon - StdAlgorithmsTeamMinElement - StdAlgorithmsTeamMaxElement - StdAlgorithmsTeamMinMaxElement - ) +foreach(Name StdAlgorithmsCommon StdAlgorithmsTeamMinElement StdAlgorithmsTeamMaxElement StdAlgorithmsTeamMinMaxElement) list(APPEND STDALGO_TEAM_SOURCES_D Test${Name}.cpp) endforeach() @@ -304,16 +250,17 @@ endforeach() # std team C # ------------------------------------------ set(STDALGO_TEAM_SOURCES_C) -foreach(Name - StdAlgorithmsCommon - StdAlgorithmsTeamFind - StdAlgorithmsTeamFindIf - StdAlgorithmsTeamFindIfNot - StdAlgorithmsTeamAllOf - StdAlgorithmsTeamAnyOf - StdAlgorithmsTeamNoneOf - StdAlgorithmsTeamSearchN - ) +foreach( + Name + StdAlgorithmsCommon + StdAlgorithmsTeamFind + StdAlgorithmsTeamFindIf + StdAlgorithmsTeamFindIfNot + StdAlgorithmsTeamAllOf + StdAlgorithmsTeamAnyOf + StdAlgorithmsTeamNoneOf + StdAlgorithmsTeamSearchN +) list(APPEND STDALGO_TEAM_SOURCES_C Test${Name}.cpp) endforeach() @@ -321,13 +268,9 @@ endforeach() # std team B # ------------------------------------------ set(STDALGO_TEAM_SOURCES_B) -foreach(Name - StdAlgorithmsCommon - StdAlgorithmsTeamEqual - StdAlgorithmsTeamSearch - StdAlgorithmsTeamFindEnd - StdAlgorithmsTeamFindFirstOf - ) +foreach(Name StdAlgorithmsCommon StdAlgorithmsTeamEqual StdAlgorithmsTeamSearch StdAlgorithmsTeamFindEnd + StdAlgorithmsTeamFindFirstOf +) list(APPEND STDALGO_TEAM_SOURCES_B Test${Name}.cpp) endforeach() @@ -335,34 +278,33 @@ endforeach() # std team A # ------------------------------------------ set(STDALGO_TEAM_SOURCES_A) -foreach(Name - StdAlgorithmsCommon - StdAlgorithmsTeamAdjacentFind - StdAlgorithmsTeamCount - StdAlgorithmsTeamCountIf - StdAlgorithmsTeamForEach - StdAlgorithmsTeamForEachN - StdAlgorithmsTeamLexicographicalCompare - StdAlgorithmsTeamMismatch - ) +foreach( + Name + StdAlgorithmsCommon + StdAlgorithmsTeamAdjacentFind + StdAlgorithmsTeamCount + StdAlgorithmsTeamCountIf + StdAlgorithmsTeamForEach + StdAlgorithmsTeamForEachN + StdAlgorithmsTeamLexicographicalCompare + StdAlgorithmsTeamMismatch +) list(APPEND STDALGO_TEAM_SOURCES_A Test${Name}.cpp) endforeach() # FIXME_OPENMPTARGET - remove sort test as it leads to ICE with clang/16 and above at compile time. -if(KOKKOS_ENABLE_OPENMPTARGET AND KOKKOS_CXX_COMPILER_ID STREQUAL "Clang" AND KOKKOS_CXX_COMPILER_VERSION VERSION_GREATER_EQUAL 16.0.0) - list(REMOVE_ITEM ALGO_SORT_SOURCES - TestSort.cpp - ) +if(KOKKOS_ENABLE_OPENMPTARGET AND KOKKOS_CXX_COMPILER_ID STREQUAL "Clang" AND KOKKOS_CXX_COMPILER_VERSION + VERSION_GREATER_EQUAL 16.0.0 +) + list(REMOVE_ITEM ALGO_SORT_SOURCES TestSort.cpp) endif() # FIXME_OPENMPTARGET remove tests for OpenMPTarget because in these cases # the impl needs to use either Kokkos or tailored reducers # which results in runtime memory errors. if(KOKKOS_ENABLE_OPENMPTARGET) - list(REMOVE_ITEM STDALGO_TEAM_SOURCES_L - TestStdAlgorithmsTeamIsPartitioned.cpp - TestStdAlgorithmsTeamPartitionPoint.cpp - TestStdAlgorithmsTeamPartitionCopy.cpp + list(REMOVE_ITEM STDALGO_TEAM_SOURCES_L TestStdAlgorithmsTeamIsPartitioned.cpp + TestStdAlgorithmsTeamPartitionPoint.cpp TestStdAlgorithmsTeamPartitionCopy.cpp ) endif() @@ -370,7 +312,9 @@ endif() # in these cases the impl needs to use either Kokkos or # tailored reducers which results in runtime memory errors. if(KOKKOS_ENABLE_OPENMPTARGET) - list(REMOVE_ITEM STDALGO_TEAM_SOURCES_C + list( + REMOVE_ITEM + STDALGO_TEAM_SOURCES_C TestStdAlgorithmsTeamFind.cpp TestStdAlgorithmsTeamFindIf.cpp TestStdAlgorithmsTeamFindIfNot.cpp @@ -386,35 +330,20 @@ endif() # FRIZZI: 04/26/2023: not sure if the compilation error is still applicable # but we conservatively leave this guard on if(NOT (KOKKOS_ENABLE_OPENMPTARGET AND KOKKOS_CXX_COMPILER_ID STREQUAL IntelLLVM)) - KOKKOS_ADD_EXECUTABLE_AND_TEST( - UnitTest_Sort - SOURCES - UnitTestMain.cpp - TestStdAlgorithmsCommon.cpp - ${ALGO_SORT_SOURCES} + kokkos_add_executable_and_test( + UnitTest_Sort SOURCES UnitTestMain.cpp TestStdAlgorithmsCommon.cpp ${ALGO_SORT_SOURCES} ) - KOKKOS_ADD_EXECUTABLE_AND_TEST( - UnitTest_Random - SOURCES - UnitTestMain.cpp - ${ALGO_RANDOM_SOURCES} - ) + kokkos_add_executable_and_test(UnitTest_Random SOURCES UnitTestMain.cpp ${ALGO_RANDOM_SOURCES}) endif() # FIXME_OPENMPTARGET: These tests cause internal compiler errors as of 09/01/22 # when compiling for Intel's Xe-HP GPUs. if(KOKKOS_ENABLE_OPENMPTARGET AND KOKKOS_CXX_COMPILER_ID STREQUAL IntelLLVM) - list(REMOVE_ITEM STDALGO_SOURCES_D - TestStdAlgorithmsCopyIf.cpp - TestStdAlgorithmsRemoveCopy.cpp - TestStdAlgorithmsUnique.cpp - TestStdAlgorithmsUniqueCopy.cpp - ) - list(REMOVE_ITEM STDALGO_SOURCES_E - TestStdAlgorithmsExclusiveScan.cpp - TestStdAlgorithmsInclusiveScan.cpp + list(REMOVE_ITEM STDALGO_SOURCES_D TestStdAlgorithmsCopyIf.cpp TestStdAlgorithmsRemoveCopy.cpp + TestStdAlgorithmsUnique.cpp TestStdAlgorithmsUniqueCopy.cpp ) + list(REMOVE_ITEM STDALGO_SOURCES_E TestStdAlgorithmsExclusiveScan.cpp TestStdAlgorithmsInclusiveScan.cpp) endif() # FIXME_OPENMPTARGET remove tests for OpenMPTarget @@ -422,48 +351,31 @@ endif() if(KOKKOS_ENABLE_OPENMPTARGET) # the following use either Kokkos or tailored reducers # which results in runtime memory errors. - list(REMOVE_ITEM STDALGO_TEAM_SOURCES_B - TestStdAlgorithmsTeamFindEnd.cpp - TestStdAlgorithmsTeamFindFirstOf.cpp - TestStdAlgorithmsTeamSearch.cpp + list(REMOVE_ITEM STDALGO_TEAM_SOURCES_B TestStdAlgorithmsTeamFindEnd.cpp TestStdAlgorithmsTeamFindFirstOf.cpp + TestStdAlgorithmsTeamSearch.cpp ) - list(REMOVE_ITEM STDALGO_TEAM_SOURCES_A - TestStdAlgorithmsTeamAdjacentFind.cpp - TestStdAlgorithmsTeamLexicographicalCompare.cpp - TestStdAlgorithmsTeamMismatch.cpp + list(REMOVE_ITEM STDALGO_TEAM_SOURCES_A TestStdAlgorithmsTeamAdjacentFind.cpp + TestStdAlgorithmsTeamLexicographicalCompare.cpp TestStdAlgorithmsTeamMismatch.cpp ) # this causes an illegal memory access if team_members_have_matching_result # is called - list(REMOVE_ITEM STDALGO_TEAM_SOURCES_M - TestStdAlgorithmsTeamTransformBinaryOp.cpp - ) + list(REMOVE_ITEM STDALGO_TEAM_SOURCES_M TestStdAlgorithmsTeamTransformBinaryOp.cpp) endif() foreach(ID A;B;C;D;E) - KOKKOS_ADD_EXECUTABLE_AND_TEST( - AlgorithmsUnitTest_StdSet_${ID} - SOURCES - UnitTestMain.cpp - ${STDALGO_SOURCES_${ID}} - ) + kokkos_add_executable_and_test(AlgorithmsUnitTest_StdSet_${ID} SOURCES UnitTestMain.cpp ${STDALGO_SOURCES_${ID}}) endforeach() foreach(ID A;B;C;D;E;F;G;H;I;L;M;P;Q) - KOKKOS_ADD_EXECUTABLE_AND_TEST( - AlgorithmsUnitTest_StdSet_Team_${ID} - SOURCES - UnitTestMain.cpp - ${STDALGO_TEAM_SOURCES_${ID}} - ) + kokkos_add_executable_and_test( + AlgorithmsUnitTest_StdSet_Team_${ID} SOURCES UnitTestMain.cpp ${STDALGO_TEAM_SOURCES_${ID}} + ) endforeach() # FIXME_OPENMPTARGET This test causes internal compiler errors as of 09/01/22 # when compiling for Intel's Xe-HP GPUs. if(NOT (KOKKOS_ENABLE_OPENMPTARGET AND KOKKOS_CXX_COMPILER_ID STREQUAL IntelLLVM)) - KOKKOS_ADD_EXECUTABLE( - AlgorithmsUnitTest_StdAlgoCompileOnly - SOURCES TestStdAlgorithmsCompileOnly.cpp - ) + kokkos_add_executable(AlgorithmsUnitTest_StdAlgoCompileOnly SOURCES TestStdAlgorithmsCompileOnly.cpp) endif() diff --git a/lib/kokkos/algorithms/unit_tests/TestBinSortA.hpp b/lib/kokkos/algorithms/unit_tests/TestBinSortA.hpp index dd3569e671..bb074f2480 100644 --- a/lib/kokkos/algorithms/unit_tests/TestBinSortA.hpp +++ b/lib/kokkos/algorithms/unit_tests/TestBinSortA.hpp @@ -31,13 +31,13 @@ struct bin3d_is_sorted_struct { using value_type = unsigned int; using execution_space = ExecutionSpace; - Kokkos::View keys; + Kokkos::View keys; int max_bins; Scalar min; Scalar max; - bin3d_is_sorted_struct(Kokkos::View keys_, + bin3d_is_sorted_struct(Kokkos::View keys_, int max_bins_, Scalar min_, Scalar max_) : keys(keys_), max_bins(max_bins_), min(min_), max(max_) {} KOKKOS_INLINE_FUNCTION @@ -65,9 +65,9 @@ struct sum3D { using value_type = double; using execution_space = ExecutionSpace; - Kokkos::View keys; + Kokkos::View keys; - sum3D(Kokkos::View keys_) : keys(keys_) {} + sum3D(Kokkos::View keys_) : keys(keys_) {} KOKKOS_INLINE_FUNCTION void operator()(int i, double& count) const { count += keys(i, 0); @@ -77,8 +77,8 @@ struct sum3D { }; template -void test_3D_sort_impl(unsigned int n) { - using KeyViewType = Kokkos::View; +void test_3D_sort_impl(size_t n) { + using KeyViewType = Kokkos::View; KeyViewType keys("Keys", n * n * n); @@ -207,7 +207,7 @@ void test_sort_integer_overflow() { // array with two extrema in reverse order to expose integer overflow bug in // bin calculation T a[2] = {Kokkos::Experimental::finite_max::value, - Kokkos::Experimental::finite_min::value}; + Kokkos::Experimental::finite_min::value}; auto vd = Kokkos::create_mirror_view_and_copy( ExecutionSpace(), Kokkos::View(a)); Kokkos::sort(vd); @@ -219,6 +219,10 @@ void test_sort_integer_overflow() { } // namespace BinSortSetA TEST(TEST_CATEGORY, BinSortGenericTests) { + // FIXME_OPENMPTARGET - causes runtime failure with CrayClang compiler +#if defined(KOKKOS_COMPILER_CRAY_LLVM) && defined(KOKKOS_ENABLE_OPENMPTARGET) + GTEST_SKIP() << "known to fail with OpenMPTarget+Cray LLVM"; +#endif using ExecutionSpace = TEST_EXECSPACE; using key_type = unsigned; constexpr int N = 171; @@ -246,11 +250,11 @@ TEST(TEST_CATEGORY, BinSortEmptyView) { // does not matter if we use int or something else Kokkos::View v("v", 0); - // test all exposed public sort methods - ASSERT_NO_THROW(Sorter.sort(ExecutionSpace(), v, 0, 0)); - ASSERT_NO_THROW(Sorter.sort(v, 0, 0)); - ASSERT_NO_THROW(Sorter.sort(ExecutionSpace(), v)); - ASSERT_NO_THROW(Sorter.sort(v)); + // test all exposed public sort methods are callable and do not throw + Sorter.sort(ExecutionSpace(), v, 0, 0); + Sorter.sort(v, 0, 0); + Sorter.sort(ExecutionSpace(), v); + Sorter.sort(v); } TEST(TEST_CATEGORY, BinSortEmptyKeysView) { @@ -263,7 +267,26 @@ TEST(TEST_CATEGORY, BinSortEmptyKeysView) { BinOp_t binOp(5, 0, 10); Kokkos::BinSort Sorter(ExecutionSpace{}, kv, binOp); - ASSERT_NO_THROW(Sorter.create_permute_vector(ExecutionSpace{})); + Sorter.create_permute_vector(ExecutionSpace{}); // does not throw +} + +// BinSort may delegate sorting within bins to std::sort when running on host +// and having a sufficiently large number of items within a single bin (10 by +// default). Test that this is done without undefined behavior when accessing +// the boundaries of the bin. Should be used in conjunction with a memory +// sanitizer or bounds check. +TEST(TEST_CATEGORY, BinSort_issue_7221) { + using ExecutionSpace = TEST_EXECSPACE; + + using KeyViewType = Kokkos::View; + KeyViewType kv("kv", 11); + + using BinOp_t = Kokkos::BinOp1D; + BinOp_t binOp(1, -10, 10); + Kokkos::BinSort Sorter(ExecutionSpace{}, kv, binOp, + /*sort_within_bins*/ true); + + Sorter.create_permute_vector(ExecutionSpace{}); // does not throw } } // namespace Test diff --git a/lib/kokkos/algorithms/unit_tests/TestBinSortB.hpp b/lib/kokkos/algorithms/unit_tests/TestBinSortB.hpp index a90224bf31..d11b53a9a6 100644 --- a/lib/kokkos/algorithms/unit_tests/TestBinSortB.hpp +++ b/lib/kokkos/algorithms/unit_tests/TestBinSortB.hpp @@ -185,6 +185,10 @@ void run_for_rank2() { } // namespace BinSortSetB TEST(TEST_CATEGORY, BinSortUnsignedKeyLayoutStrideValues) { + // FIXME_OPENMPTARGET - causes runtime failure with CrayClang compiler +#if defined(KOKKOS_COMPILER_CRAY_LLVM) && defined(KOKKOS_ENABLE_OPENMPTARGET) + GTEST_SKIP() << "known to fail with OpenMPTarget+Cray LLVM"; +#endif using ExeSpace = TEST_EXECSPACE; using key_type = unsigned; BinSortSetB::run_for_rank1(); diff --git a/lib/kokkos/algorithms/unit_tests/TestNestedSort.hpp b/lib/kokkos/algorithms/unit_tests/TestNestedSort.hpp index 1b7a3f48fc..cd57fd23ec 100644 --- a/lib/kokkos/algorithms/unit_tests/TestNestedSort.hpp +++ b/lib/kokkos/algorithms/unit_tests/TestNestedSort.hpp @@ -386,6 +386,11 @@ void test_nested_sort_by_key(unsigned int N, KeyType minKey, KeyType maxKey, } // namespace NestedSortImpl TEST(TEST_CATEGORY, NestedSort) { + // FIXME_OPENMPTARGET - causes runtime failure with CrayClang compiler +#if defined(KOKKOS_COMPILER_CRAY_LLVM) && defined(KOKKOS_ENABLE_OPENMPTARGET) + GTEST_SKIP() << "known to fail with OpenMPTarget+Cray LLVM"; +#endif + using ExecutionSpace = TEST_EXECSPACE; NestedSortImpl::test_nested_sort(171, 0U, UINT_MAX); NestedSortImpl::test_nested_sort(42, -1e6f, 1e6f); @@ -394,6 +399,11 @@ TEST(TEST_CATEGORY, NestedSort) { } TEST(TEST_CATEGORY, NestedSortByKey) { + // FIXME_OPENMPTARGET - causes runtime failure with CrayClang compiler +#if defined(KOKKOS_COMPILER_CRAY_LLVM) && defined(KOKKOS_ENABLE_OPENMPTARGET) + GTEST_SKIP() << "known to fail with OpenMPTarget+Cray LLVM"; +#endif + using ExecutionSpace = TEST_EXECSPACE; // Second/third template arguments are key and value respectively. diff --git a/lib/kokkos/algorithms/unit_tests/TestRandom.hpp b/lib/kokkos/algorithms/unit_tests/TestRandom.hpp index 472af1403b..6960b912d0 100644 --- a/lib/kokkos/algorithms/unit_tests/TestRandom.hpp +++ b/lib/kokkos/algorithms/unit_tests/TestRandom.hpp @@ -542,6 +542,11 @@ void test_duplicate_stream() { } // namespace AlgoRandomImpl TEST(TEST_CATEGORY, Random_XorShift64) { + // FIXME_OPENMPTARGET - causes runtime failure with CrayClang compiler +#if defined(KOKKOS_COMPILER_CRAY_LLVM) && defined(KOKKOS_ENABLE_OPENMPTARGET) + GTEST_SKIP() << "known to fail with OpenMPTarget+Cray LLVM"; +#endif + using ExecutionSpace = TEST_EXECSPACE; #if defined(KOKKOS_ENABLE_SYCL) || defined(KOKKOS_ENABLE_CUDA) || \ @@ -562,6 +567,10 @@ TEST(TEST_CATEGORY, Random_XorShift64) { TEST(TEST_CATEGORY, Random_XorShift1024_0) { using ExecutionSpace = TEST_EXECSPACE; + // FIXME_OPENMPTARGET - causes runtime failure with CrayClang compiler +#if defined(KOKKOS_COMPILER_CRAY_LLVM) && defined(KOKKOS_ENABLE_OPENMPTARGET) + GTEST_SKIP() << "known to fail with OpenMPTarget+Cray LLVM"; +#endif #if defined(KOKKOS_ENABLE_SYCL) || defined(KOKKOS_ENABLE_CUDA) || \ defined(KOKKOS_ENABLE_HIP) @@ -589,7 +598,7 @@ TEST(TEST_CATEGORY, Multi_streams) { #endif #if defined(KOKKOS_ENABLE_SYCL) && defined(KOKKOS_IMPL_ARCH_NVIDIA_GPU) - if constexpr (std::is_same_v) { + if constexpr (std::is_same_v) { GTEST_SKIP() << "Failing on NVIDIA GPUs"; // FIXME_SYCL } #endif diff --git a/lib/kokkos/algorithms/unit_tests/TestRandomAccessIterator.cpp b/lib/kokkos/algorithms/unit_tests/TestRandomAccessIterator.cpp index 7d484136b6..5ab348cb19 100644 --- a/lib/kokkos/algorithms/unit_tests/TestRandomAccessIterator.cpp +++ b/lib/kokkos/algorithms/unit_tests/TestRandomAccessIterator.cpp @@ -23,7 +23,7 @@ namespace stdalgos { struct random_access_iterator_test : std_algorithms_test { public: - virtual void SetUp() { + void SetUp() override { Kokkos::parallel_for(m_static_view.extent(0), AssignIndexFunctor(m_static_view)); @@ -264,6 +264,37 @@ TEST_F(random_access_iterator_test, traits_helpers) { static_assert(KE::Impl::are_iterators_v); static_assert(KE::Impl::are_random_access_iterators_v); static_assert(!KE::Impl::are_iterators_v); + + static_assert(std::is_same_v); + static_assert(std::is_same_v); + static_assert(std::is_same_v); + + static_assert( + std::is_same_v); + static_assert(std::is_same_v); + static_assert(std::is_same_v); + + static_assert( + std::is_same_v); + static_assert( + std::is_same_v); + static_assert( + std::is_same_v); + + static_assert(std::is_same_v); + static_assert(std::is_same_v); + static_assert(std::is_same_v); } } // namespace stdalgos diff --git a/lib/kokkos/algorithms/unit_tests/TestSort.hpp b/lib/kokkos/algorithms/unit_tests/TestSort.hpp index 968fb8950b..5ea88ae5d6 100644 --- a/lib/kokkos/algorithms/unit_tests/TestSort.hpp +++ b/lib/kokkos/algorithms/unit_tests/TestSort.hpp @@ -197,7 +197,7 @@ void test_sort_integer_overflow() { // array with two extrema in reverse order to expose integer overflow bug in // bin calculation T a[2] = {Kokkos::Experimental::finite_max::value, - Kokkos::Experimental::finite_min::value}; + Kokkos::Experimental::finite_min::value}; auto vd = Kokkos::create_mirror_view_and_copy( ExecutionSpace(), Kokkos::View(a)); Kokkos::sort(vd); @@ -209,6 +209,10 @@ void test_sort_integer_overflow() { } // namespace SortImpl TEST(TEST_CATEGORY, SortUnsignedValueType) { + // FIXME_OPENMPTARGET - causes runtime failure with CrayClang compiler +#if defined(KOKKOS_COMPILER_CRAY_LLVM) && defined(KOKKOS_ENABLE_OPENMPTARGET) + GTEST_SKIP() << "known to fail with OpenMPTarget+Cray LLVM"; +#endif using ExecutionSpace = TEST_EXECSPACE; using key_type = unsigned; constexpr int N = 171; @@ -224,14 +228,19 @@ TEST(TEST_CATEGORY, SortUnsignedValueType) { } TEST(TEST_CATEGORY, SortEmptyView) { + // FIXME_OPENMPTARGET - causes runtime failure with CrayClang compiler +#if defined(KOKKOS_COMPILER_CRAY_LLVM) && defined(KOKKOS_ENABLE_OPENMPTARGET) + GTEST_SKIP() << "known to fail with OpenMPTarget+Cray LLVM"; +#endif using ExecutionSpace = TEST_EXECSPACE; // does not matter if we use int or something else Kokkos::View v("v", 0); + // checking that it does not throw // TODO check the synchronous behavior of the calls below - ASSERT_NO_THROW(Kokkos::sort(ExecutionSpace(), v)); - ASSERT_NO_THROW(Kokkos::sort(v)); + Kokkos::sort(ExecutionSpace(), v); + Kokkos::sort(v); } } // namespace Test diff --git a/lib/kokkos/algorithms/unit_tests/TestSortByKey.hpp b/lib/kokkos/algorithms/unit_tests/TestSortByKey.hpp index 9e5bd4a574..44abe4e73a 100644 --- a/lib/kokkos/algorithms/unit_tests/TestSortByKey.hpp +++ b/lib/kokkos/algorithms/unit_tests/TestSortByKey.hpp @@ -83,8 +83,8 @@ TEST(TEST_CATEGORY, SortByKeyEmptyView) { Kokkos::View keys("keys", 0); Kokkos::View values("values", 0); - ASSERT_NO_THROW( - Kokkos::Experimental::sort_by_key(ExecutionSpace(), keys, values)); + // checking that it does not throw + Kokkos::Experimental::sort_by_key(ExecutionSpace(), keys, values); } // Test #7036 @@ -95,8 +95,8 @@ TEST(TEST_CATEGORY, SortByKeyEmptyViewHost) { Kokkos::View keys("keys", 0); Kokkos::View values("values", 0); - ASSERT_NO_THROW( - Kokkos::Experimental::sort_by_key(ExecutionSpace(), keys, values)); + // checking that it does not throw + Kokkos::Experimental::sort_by_key(ExecutionSpace(), keys, values); } TEST(TEST_CATEGORY, SortByKey) { @@ -183,12 +183,12 @@ TEST(TEST_CATEGORY, SortByKeyStaticExtents) { Kokkos::View keys("keys"); Kokkos::View values_static("values_static"); - ASSERT_NO_THROW( - Kokkos::Experimental::sort_by_key(space, keys, values_static)); + // checking that it does not throw + Kokkos::Experimental::sort_by_key(space, keys, values_static); Kokkos::View values_dynamic("values_dynamic", 10); - ASSERT_NO_THROW( - Kokkos::Experimental::sort_by_key(space, keys, values_dynamic)); + // checking that it does not throw + Kokkos::Experimental::sort_by_key(space, keys, values_dynamic); } template @@ -234,7 +234,9 @@ TEST(TEST_CATEGORY, SortByKeyWithStrides) { ASSERT_EQ(sort_fails, 0u); } -TEST(TEST_CATEGORY, SortByKeyKeysLargerThanValues) { +TEST(TEST_CATEGORY_DEATH, SortByKeyKeysLargerThanValues) { + ::testing::FLAGS_gtest_death_test_style = "threadsafe"; + using ExecutionSpace = TEST_EXECSPACE; // does not matter if we use int or something else diff --git a/lib/kokkos/algorithms/unit_tests/TestStdAlgorithmsAdjacentDifference.cpp b/lib/kokkos/algorithms/unit_tests/TestStdAlgorithmsAdjacentDifference.cpp index 75ad533f6e..208b46b15f 100644 --- a/lib/kokkos/algorithms/unit_tests/TestStdAlgorithmsAdjacentDifference.cpp +++ b/lib/kokkos/algorithms/unit_tests/TestStdAlgorithmsAdjacentDifference.cpp @@ -96,7 +96,7 @@ void fill_view(DestViewType dest_view, const std::string& name) { } else { - throw std::runtime_error("invalid choice"); + FAIL() << "invalid choice"; } Kokkos::deep_copy(aux_view, aux_v_h); diff --git a/lib/kokkos/algorithms/unit_tests/TestStdAlgorithmsAdjacentFind.cpp b/lib/kokkos/algorithms/unit_tests/TestStdAlgorithmsAdjacentFind.cpp index fa4ff48dbe..d8b80675c9 100644 --- a/lib/kokkos/algorithms/unit_tests/TestStdAlgorithmsAdjacentFind.cpp +++ b/lib/kokkos/algorithms/unit_tests/TestStdAlgorithmsAdjacentFind.cpp @@ -173,7 +173,7 @@ void fill_view(ViewType dest_view, const std::string& name) { } else { - throw std::runtime_error("invalid choice"); + FAIL() << "invalid choice"; } Kokkos::deep_copy(aux_view, v_h); @@ -243,7 +243,7 @@ void run_single_scenario(const InfoType& scenario_info, Args... args) { { auto res_it = KE::adjacent_find(exespace(), KE::cbegin(view), - KE::cend(view), args...); + KE::cend(view), args...); const auto my_diff = res_it - KE::cbegin(view); verify(my_diff, view, args...); } diff --git a/lib/kokkos/algorithms/unit_tests/TestStdAlgorithmsCommon.hpp b/lib/kokkos/algorithms/unit_tests/TestStdAlgorithmsCommon.hpp index 67052e2f9d..dadce2d474 100644 --- a/lib/kokkos/algorithms/unit_tests/TestStdAlgorithmsCommon.hpp +++ b/lib/kokkos/algorithms/unit_tests/TestStdAlgorithmsCommon.hpp @@ -534,10 +534,10 @@ void fill_views_inc(ViewType view, ViewHostType host_view) { } template -std::enable_if_t::value> +std::enable_if_t> verify_values(ValueType expected, const ViewType view) { - static_assert(std::is_same::value, + static_assert(std::is_same_v, "Non-matching value types of view and reference value"); auto view_h = Kokkos::create_mirror_view_and_copy(Kokkos::HostSpace(), view); for (std::size_t i = 0; i < view_h.extent(0); i++) { @@ -546,10 +546,10 @@ verify_values(ValueType expected, const ViewType view) { } template -std::enable_if_t::value> +std::enable_if_t> verify_values(ValueType expected, const ViewType view) { - static_assert(std::is_same::value, + static_assert(std::is_same_v, "Non-matching value types of view and reference value"); using non_strided_view_t = Kokkos::View; @@ -566,11 +566,11 @@ verify_values(ValueType expected, const ViewType view) { } template -std::enable_if_t::value> +std::enable_if_t> compare_views(ViewType1 expected, const ViewType2 actual) { - static_assert(std::is_same::value, + static_assert(std::is_same_v, "Non-matching value types of expected and actual view"); auto expected_h = Kokkos::create_mirror_view_and_copy(Kokkos::HostSpace(), expected); @@ -583,11 +583,11 @@ compare_views(ViewType1 expected, const ViewType2 actual) { } template -std::enable_if_t::value> +std::enable_if_t> compare_views(ViewType1 expected, const ViewType2 actual) { - static_assert(std::is_same::value, + static_assert(std::is_same_v, "Non-matching value types of expected and actual view"); using non_strided_view_t = Kokkos::View; diff --git a/lib/kokkos/algorithms/unit_tests/TestStdAlgorithmsConstraints.cpp b/lib/kokkos/algorithms/unit_tests/TestStdAlgorithmsConstraints.cpp index 2a4525a8c3..923ea970f9 100644 --- a/lib/kokkos/algorithms/unit_tests/TestStdAlgorithmsConstraints.cpp +++ b/lib/kokkos/algorithms/unit_tests/TestStdAlgorithmsConstraints.cpp @@ -81,7 +81,7 @@ TEST(std_algorithms, is_admissible_to_std_algorithms) { strided_view_3d_t>::value); } -TEST(std_algorithms, expect_no_overlap) { +TEST(std_algorithms_DeathTest, expect_no_overlap) { namespace KE = Kokkos::Experimental; using value_type = double; @@ -104,6 +104,8 @@ TEST(std_algorithms, expect_no_overlap) { // Overlapping because iterators are identical #if defined(KOKKOS_ENABLE_DEBUG) + ::testing::FLAGS_gtest_death_test_style = "threadsafe"; + auto first_s = KE::begin(static_view_1d); auto last_s = first_s + extent0; EXPECT_DEATH({ KE::Impl::expect_no_overlap(first_s, last_s, first_s); }, @@ -148,8 +150,7 @@ TEST(std_algorithms, expect_no_overlap) { auto last_st0 = first_st0 + strided_view_1d_0.extent(0); auto first_st1 = KE::begin(strided_view_1d_1); // [3, 15) // Does not overlap since offset (=3) is not divisible by stride (=2) - EXPECT_NO_THROW( - { KE::Impl::expect_no_overlap(first_st0, last_st0, first_st1); }); + KE::Impl::expect_no_overlap(first_st0, last_st0, first_st1); // Iterating over the same range without overlapping Kokkos::View static_view_2d{ @@ -160,9 +161,7 @@ TEST(std_algorithms, expect_no_overlap) { auto sub_last_s0 = sub_first_s0 + sub_static_view_1d_0.extent(0); auto sub_first_s1 = KE::begin(sub_static_view_1d_1); // 1, 3, 5, ... - EXPECT_NO_THROW({ - KE::Impl::expect_no_overlap(sub_first_s0, sub_last_s0, sub_first_s1); - }); + KE::Impl::expect_no_overlap(sub_first_s0, sub_last_s0, sub_first_s1); Kokkos::View dynamic_view_2d{ "std-algo-test-2d-contiguous-view-dynamic", 2, extent0}; @@ -172,9 +171,7 @@ TEST(std_algorithms, expect_no_overlap) { auto sub_last_d0 = sub_first_d0 + sub_dynamic_view_1d_0.extent(0); auto sub_first_d1 = KE::begin(sub_dynamic_view_1d_1); // 1, 3, 5, ... - EXPECT_NO_THROW({ - KE::Impl::expect_no_overlap(sub_first_d0, sub_last_d0, sub_first_d1); - }); + KE::Impl::expect_no_overlap(sub_first_d0, sub_last_d0, sub_first_d1); Kokkos::LayoutStride layout2d{2, 3, extent0, 2 * 3}; Kokkos::View strided_view_2d{ @@ -185,9 +182,7 @@ TEST(std_algorithms, expect_no_overlap) { auto sub_last_st0 = sub_first_st0 + sub_strided_view_1d_0.extent(0); auto sub_first_st1 = KE::begin(sub_strided_view_1d_1); // 1, 7, 13, ... - EXPECT_NO_THROW({ - KE::Impl::expect_no_overlap(sub_first_st0, sub_last_st0, sub_first_st1); - }); + KE::Impl::expect_no_overlap(sub_first_st0, sub_last_st0, sub_first_st1); } } // namespace stdalgos diff --git a/lib/kokkos/algorithms/unit_tests/TestStdAlgorithmsCopyIf.cpp b/lib/kokkos/algorithms/unit_tests/TestStdAlgorithmsCopyIf.cpp index 5778e37be0..7c9e8f84bf 100644 --- a/lib/kokkos/algorithms/unit_tests/TestStdAlgorithmsCopyIf.cpp +++ b/lib/kokkos/algorithms/unit_tests/TestStdAlgorithmsCopyIf.cpp @@ -107,7 +107,7 @@ std::size_t fill_view(ViewType dest_view, const std::string& name, } else { - throw std::runtime_error("invalid choice"); + Kokkos::abort("invalid choice"); } Kokkos::deep_copy(aux_view, v_h); @@ -202,7 +202,7 @@ void verify_data(const std::string& name, ViewTypeFrom view_from, } else { - throw std::runtime_error("invalid choice"); + FAIL() << "invalid choice"; } } @@ -224,7 +224,7 @@ void run_single_scenario(const InfoType& scenario_info) { auto n = fill_view(view_from, name, pred); auto view_dest = create_view(Tag{}, view_ext, "copy_if_dest"); auto rit = KE::copy_if(exespace(), KE::cbegin(view_from), - KE::cend(view_from), KE::begin(view_dest), pred); + KE::cend(view_from), KE::begin(view_dest), pred); verify_data(name, view_from, view_dest, pred); ASSERT_EQ(rit, (KE::begin(view_dest) + n)); } @@ -233,7 +233,7 @@ void run_single_scenario(const InfoType& scenario_info) { auto n = fill_view(view_from, name, pred); auto view_dest = create_view(Tag{}, view_ext, "copy_if_dest"); auto rit = KE::copy_if("label", exespace(), KE::cbegin(view_from), - KE::cend(view_from), KE::begin(view_dest), pred); + KE::cend(view_from), KE::begin(view_dest), pred); verify_data(name, view_from, view_dest, pred); ASSERT_EQ(rit, (KE::begin(view_dest) + n)); } diff --git a/lib/kokkos/algorithms/unit_tests/TestStdAlgorithmsExclusiveScan.cpp b/lib/kokkos/algorithms/unit_tests/TestStdAlgorithmsExclusiveScan.cpp index b364c53a88..a85e63fe34 100644 --- a/lib/kokkos/algorithms/unit_tests/TestStdAlgorithmsExclusiveScan.cpp +++ b/lib/kokkos/algorithms/unit_tests/TestStdAlgorithmsExclusiveScan.cpp @@ -110,7 +110,7 @@ void fill_view(ViewType dest_view, const std::string& name) { } else { - throw std::runtime_error("invalid choice"); + FAIL() << "invalid choice"; } Kokkos::deep_copy(aux_view, v_h); diff --git a/lib/kokkos/algorithms/unit_tests/TestStdAlgorithmsForEach.cpp b/lib/kokkos/algorithms/unit_tests/TestStdAlgorithmsForEach.cpp index 793b98a67f..b24730ff00 100644 --- a/lib/kokkos/algorithms/unit_tests/TestStdAlgorithmsForEach.cpp +++ b/lib/kokkos/algorithms/unit_tests/TestStdAlgorithmsForEach.cpp @@ -55,7 +55,6 @@ void test_for_each(const ViewType view) { std::for_each(KE::begin(expected), KE::end(expected), non_mod_functor); compare_views(expected, view); -#if defined(KOKKOS_ENABLE_CXX11_DISPATCH_LAMBDA) const auto mod_lambda = KOKKOS_LAMBDA(value_t & i) { ++i; }; // pass view, lambda takes non-const ref @@ -79,7 +78,6 @@ void test_for_each(const ViewType view) { KE::for_each(exespace(), KE::cbegin(view), KE::cend(view), non_mod_lambda); std::for_each(KE::cbegin(expected), KE::cend(expected), non_mod_lambda); compare_views(expected, view); -#endif } // std::for_each_n is C++17, so we cannot compare results directly diff --git a/lib/kokkos/algorithms/unit_tests/TestStdAlgorithmsHelperFunctors.hpp b/lib/kokkos/algorithms/unit_tests/TestStdAlgorithmsHelperFunctors.hpp index 8dbd6cd7e3..2b3361743e 100644 --- a/lib/kokkos/algorithms/unit_tests/TestStdAlgorithmsHelperFunctors.hpp +++ b/lib/kokkos/algorithms/unit_tests/TestStdAlgorithmsHelperFunctors.hpp @@ -104,7 +104,7 @@ struct AssignIndexFunctor { template struct IsEvenFunctor { - static_assert(std::is_integral::value, + static_assert(std::is_integral_v, "IsEvenFunctor uses operator%, so ValueType must be int"); KOKKOS_INLINE_FUNCTION diff --git a/lib/kokkos/algorithms/unit_tests/TestStdAlgorithmsInclusiveScan.cpp b/lib/kokkos/algorithms/unit_tests/TestStdAlgorithmsInclusiveScan.cpp index a08a737210..b4f40b4651 100644 --- a/lib/kokkos/algorithms/unit_tests/TestStdAlgorithmsInclusiveScan.cpp +++ b/lib/kokkos/algorithms/unit_tests/TestStdAlgorithmsInclusiveScan.cpp @@ -110,7 +110,7 @@ void fill_view(ViewType dest_view, const std::string& name) { } else { - throw std::runtime_error("invalid choice"); + FAIL() << "invalid choice"; } Kokkos::deep_copy(aux_view, v_h); diff --git a/lib/kokkos/algorithms/unit_tests/TestStdAlgorithmsIsSorted.cpp b/lib/kokkos/algorithms/unit_tests/TestStdAlgorithmsIsSorted.cpp index 75d4f0afeb..18928a3526 100644 --- a/lib/kokkos/algorithms/unit_tests/TestStdAlgorithmsIsSorted.cpp +++ b/lib/kokkos/algorithms/unit_tests/TestStdAlgorithmsIsSorted.cpp @@ -92,7 +92,7 @@ void fill_view(ViewType dest_view, const std::string& name) { } else { - throw std::runtime_error("invalid choice"); + FAIL() << "invalid choice"; } Kokkos::deep_copy(aux_view, v_h); @@ -122,7 +122,8 @@ bool compute_gold(const std::string& name) { } else if (name == "large-b") { return false; } else { - throw std::runtime_error("invalid choice"); + Kokkos::abort("invalid choice"); + return false; // unreachable } } @@ -154,7 +155,7 @@ void run_single_scenario(const InfoType& scenario_info) { resultsB[0] = KE::is_sorted(exespace(), KE::cbegin(view), KE::cend(view), comp); resultsB[1] = KE::is_sorted("label", exespace(), KE::cbegin(view), - KE::cend(view), comp); + KE::cend(view), comp); resultsB[2] = KE::is_sorted(exespace(), view, comp); resultsB[3] = KE::is_sorted("label", exespace(), view, comp); const auto allB = std::all_of(resultsB.cbegin(), resultsB.cend(), diff --git a/lib/kokkos/algorithms/unit_tests/TestStdAlgorithmsIsSortedUntil.cpp b/lib/kokkos/algorithms/unit_tests/TestStdAlgorithmsIsSortedUntil.cpp index 29ac7cc9bc..8327bfe13c 100644 --- a/lib/kokkos/algorithms/unit_tests/TestStdAlgorithmsIsSortedUntil.cpp +++ b/lib/kokkos/algorithms/unit_tests/TestStdAlgorithmsIsSortedUntil.cpp @@ -92,7 +92,7 @@ void fill_view(ViewType dest_view, const std::string& name) { } else { - throw std::runtime_error("invalid choice"); + FAIL() << "invalid choice"; } Kokkos::deep_copy(aux_view, v_h); @@ -123,7 +123,8 @@ auto compute_gold(ViewType view, const std::string& name) { } else if (name == "large-b") { return KE::begin(view) + 156; } else { - throw std::runtime_error("invalid choice"); + Kokkos::abort("invalid choice"); + return KE::end(view); // unreachable } } diff --git a/lib/kokkos/algorithms/unit_tests/TestStdAlgorithmsMismatch.cpp b/lib/kokkos/algorithms/unit_tests/TestStdAlgorithmsMismatch.cpp index f3b3e269c4..df5df756d2 100644 --- a/lib/kokkos/algorithms/unit_tests/TestStdAlgorithmsMismatch.cpp +++ b/lib/kokkos/algorithms/unit_tests/TestStdAlgorithmsMismatch.cpp @@ -86,7 +86,7 @@ void run_single_scenario(ViewType view1, ViewType view2, v2_h(ext2 / 2) = -5; } } else { - throw std::runtime_error("Kokkos: stdalgo: test: mismatch: Invalid string"); + FAIL() << "Kokkos: stdalgo: test: mismatch: Invalid string"; } Kokkos::deep_copy(aux_view1, v1_h); diff --git a/lib/kokkos/algorithms/unit_tests/TestStdAlgorithmsModOps.cpp b/lib/kokkos/algorithms/unit_tests/TestStdAlgorithmsModOps.cpp index 1b1a02f39c..6918185bc0 100644 --- a/lib/kokkos/algorithms/unit_tests/TestStdAlgorithmsModOps.cpp +++ b/lib/kokkos/algorithms/unit_tests/TestStdAlgorithmsModOps.cpp @@ -48,7 +48,7 @@ struct MyMovableType { TEST(std_algorithms_mod_ops_test, move) { MyMovableType a; using move_t = decltype(std::move(a)); - static_assert(std::is_rvalue_reference::value); + static_assert(std::is_rvalue_reference_v); // move constr MyMovableType b(std::move(a)); diff --git a/lib/kokkos/algorithms/unit_tests/TestStdAlgorithmsModSeqOps.cpp b/lib/kokkos/algorithms/unit_tests/TestStdAlgorithmsModSeqOps.cpp index f80f30797e..42a17d7377 100644 --- a/lib/kokkos/algorithms/unit_tests/TestStdAlgorithmsModSeqOps.cpp +++ b/lib/kokkos/algorithms/unit_tests/TestStdAlgorithmsModSeqOps.cpp @@ -23,7 +23,7 @@ namespace stdalgos { struct std_algorithms_mod_seq_ops_test : std_algorithms_test { public: - virtual void SetUp() { + void SetUp() override { Kokkos::parallel_for(m_static_view.extent(0), AssignIndexFunctor(m_static_view)); } diff --git a/lib/kokkos/algorithms/unit_tests/TestStdAlgorithmsMoveBackward.cpp b/lib/kokkos/algorithms/unit_tests/TestStdAlgorithmsMoveBackward.cpp index b201ab95c1..88e2a68ff1 100644 --- a/lib/kokkos/algorithms/unit_tests/TestStdAlgorithmsMoveBackward.cpp +++ b/lib/kokkos/algorithms/unit_tests/TestStdAlgorithmsMoveBackward.cpp @@ -56,7 +56,7 @@ void run_single_scenario(const InfoType& scenario_info, int apiId) { ASSERT_EQ(dist, 5); } else if (apiId == 1) { auto rit = KE::move_backward("mylabel", exespace(), KE::begin(v), - KE::end(v), KE::end(v2)); + KE::end(v), KE::end(v2)); const int dist = KE::distance(KE::begin(v2), rit); ASSERT_EQ(dist, 5); } else if (apiId == 2) { diff --git a/lib/kokkos/algorithms/unit_tests/TestStdAlgorithmsPartitionCopy.cpp b/lib/kokkos/algorithms/unit_tests/TestStdAlgorithmsPartitionCopy.cpp index a36c9db2b9..e47cacdd7d 100644 --- a/lib/kokkos/algorithms/unit_tests/TestStdAlgorithmsPartitionCopy.cpp +++ b/lib/kokkos/algorithms/unit_tests/TestStdAlgorithmsPartitionCopy.cpp @@ -95,7 +95,7 @@ void fill_view(ViewType dest_view, const std::string& name) { } else { - throw std::runtime_error("invalid choice"); + FAIL() << "invalid choice"; } Kokkos::deep_copy(aux_view, v_h); @@ -110,9 +110,9 @@ void verify_data(const std::string& name, ResultType my_result, ViewTypeDestFalse view_dest_false, PredType pred) { using value_type = typename ViewTypeFrom::value_type; static_assert( - std::is_same::value); + std::is_same_v); static_assert( - std::is_same::value); + std::is_same_v); const std::size_t ext = view_from.extent(0); diff --git a/lib/kokkos/algorithms/unit_tests/TestStdAlgorithmsRemove.cpp b/lib/kokkos/algorithms/unit_tests/TestStdAlgorithmsRemove.cpp index c35fc5c24b..f897e9b657 100644 --- a/lib/kokkos/algorithms/unit_tests/TestStdAlgorithmsRemove.cpp +++ b/lib/kokkos/algorithms/unit_tests/TestStdAlgorithmsRemove.cpp @@ -99,7 +99,7 @@ void fill_view(ViewType dest_view, const std::string& name) { } else { - throw std::runtime_error("invalid choice"); + FAIL() << "invalid choice"; } Kokkos::deep_copy(aux_view, v_h); @@ -147,7 +147,7 @@ void run_single_scenario(const InfoType& scenario_info) { // make host copy BEFORE running algo auto data_h = create_host_space_copy(view); auto rit = KE::remove(exespace(), KE::begin(view), KE::end(view), - (ValueType)match_value); + (ValueType)match_value); verify_data(data_h, view, rit); } diff --git a/lib/kokkos/algorithms/unit_tests/TestStdAlgorithmsRemoveCopy.cpp b/lib/kokkos/algorithms/unit_tests/TestStdAlgorithmsRemoveCopy.cpp index 3d7c52108b..3137880ea8 100644 --- a/lib/kokkos/algorithms/unit_tests/TestStdAlgorithmsRemoveCopy.cpp +++ b/lib/kokkos/algorithms/unit_tests/TestStdAlgorithmsRemoveCopy.cpp @@ -110,7 +110,7 @@ void fill_view(ViewType dest_view, const std::string& name) { } else { - throw std::runtime_error("invalid choice"); + FAIL() << "invalid choice"; } Kokkos::deep_copy(aux_view, v_h); diff --git a/lib/kokkos/algorithms/unit_tests/TestStdAlgorithmsRemoveCopyIf.cpp b/lib/kokkos/algorithms/unit_tests/TestStdAlgorithmsRemoveCopyIf.cpp index cb699aa923..d88ab5473d 100644 --- a/lib/kokkos/algorithms/unit_tests/TestStdAlgorithmsRemoveCopyIf.cpp +++ b/lib/kokkos/algorithms/unit_tests/TestStdAlgorithmsRemoveCopyIf.cpp @@ -93,7 +93,7 @@ void fill_view(ViewType dest_view, const std::string& name) { } else { - throw std::runtime_error("invalid choice"); + FAIL() << "invalid choice"; } Kokkos::deep_copy(aux_view, v_h); diff --git a/lib/kokkos/algorithms/unit_tests/TestStdAlgorithmsRemoveIf.cpp b/lib/kokkos/algorithms/unit_tests/TestStdAlgorithmsRemoveIf.cpp index f06f2234ee..e42788799e 100644 --- a/lib/kokkos/algorithms/unit_tests/TestStdAlgorithmsRemoveIf.cpp +++ b/lib/kokkos/algorithms/unit_tests/TestStdAlgorithmsRemoveIf.cpp @@ -93,7 +93,7 @@ void fill_view(ViewType dest_view, const std::string& name) { } else { - throw std::runtime_error("invalid choice"); + FAIL() << "invalid choice"; } Kokkos::deep_copy(aux_view, v_h); @@ -144,7 +144,7 @@ void run_single_scenario(const InfoType& scenario_info) { // make host copy BEFORE running algo auto data_h = create_host_space_copy(view); auto rit = KE::remove_if(exespace(), KE::begin(view), KE::end(view), - remove_if_even); + remove_if_even); verify_data(data_h, view, rit, remove_if_even); } @@ -154,7 +154,7 @@ void run_single_scenario(const InfoType& scenario_info) { // make host copy BEFORE running algo auto data_h = create_host_space_copy(view); auto rit = KE::remove_if("label", exespace(), KE::begin(view), - KE::end(view), remove_if_even); + KE::end(view), remove_if_even); verify_data(data_h, view, rit, remove_if_even); } diff --git a/lib/kokkos/algorithms/unit_tests/TestStdAlgorithmsReplace.cpp b/lib/kokkos/algorithms/unit_tests/TestStdAlgorithmsReplace.cpp index a22ab32d76..4596726cf3 100644 --- a/lib/kokkos/algorithms/unit_tests/TestStdAlgorithmsReplace.cpp +++ b/lib/kokkos/algorithms/unit_tests/TestStdAlgorithmsReplace.cpp @@ -84,7 +84,7 @@ void fill_view(ViewType dest_view, const std::string& name) { } else { - throw std::runtime_error("invalid choice"); + FAIL() << "invalid choice"; } Kokkos::deep_copy(aux_view, v_h); @@ -153,7 +153,7 @@ void verify_data(const std::string& name, ViewType1 test_view, } else { - throw std::runtime_error("invalid choice"); + FAIL() << "invalid choice"; } } diff --git a/lib/kokkos/algorithms/unit_tests/TestStdAlgorithmsReplaceCopy.cpp b/lib/kokkos/algorithms/unit_tests/TestStdAlgorithmsReplaceCopy.cpp index a964ec8e17..b18c859af5 100644 --- a/lib/kokkos/algorithms/unit_tests/TestStdAlgorithmsReplaceCopy.cpp +++ b/lib/kokkos/algorithms/unit_tests/TestStdAlgorithmsReplaceCopy.cpp @@ -84,7 +84,7 @@ void fill_view(ViewType dest_view, const std::string& name) { } else { - throw std::runtime_error("invalid choice"); + FAIL() << "invalid choice"; } Kokkos::deep_copy(aux_view, v_h); @@ -175,7 +175,7 @@ void verify_data(const std::string& name, ViewTypeFrom view_from, } else { - throw std::runtime_error("invalid choice"); + FAIL() << "invalid choice"; } } diff --git a/lib/kokkos/algorithms/unit_tests/TestStdAlgorithmsReplaceCopyIf.cpp b/lib/kokkos/algorithms/unit_tests/TestStdAlgorithmsReplaceCopyIf.cpp index ceeba88971..82f859bac1 100644 --- a/lib/kokkos/algorithms/unit_tests/TestStdAlgorithmsReplaceCopyIf.cpp +++ b/lib/kokkos/algorithms/unit_tests/TestStdAlgorithmsReplaceCopyIf.cpp @@ -84,7 +84,7 @@ void fill_view(ViewType dest_view, const std::string& name) { } else { - throw std::runtime_error("invalid choice"); + FAIL() << "invalid choice"; } Kokkos::deep_copy(aux_view, v_h); @@ -175,7 +175,7 @@ void verify_data(const std::string& name, ViewTypeFrom view_from, } else { - throw std::runtime_error("invalid choice"); + FAIL() << "invalid choice"; } } diff --git a/lib/kokkos/algorithms/unit_tests/TestStdAlgorithmsReplaceIf.cpp b/lib/kokkos/algorithms/unit_tests/TestStdAlgorithmsReplaceIf.cpp index 802c0093c5..5ae2ff4278 100644 --- a/lib/kokkos/algorithms/unit_tests/TestStdAlgorithmsReplaceIf.cpp +++ b/lib/kokkos/algorithms/unit_tests/TestStdAlgorithmsReplaceIf.cpp @@ -96,7 +96,7 @@ void fill_view(ViewType dest_view, const std::string& name) { } else { - throw std::runtime_error("invalid choice"); + FAIL() << "invalid choice"; } Kokkos::deep_copy(aux_view, v_h); diff --git a/lib/kokkos/algorithms/unit_tests/TestStdAlgorithmsReverse.cpp b/lib/kokkos/algorithms/unit_tests/TestStdAlgorithmsReverse.cpp index 6e6ca72783..3c934d6485 100644 --- a/lib/kokkos/algorithms/unit_tests/TestStdAlgorithmsReverse.cpp +++ b/lib/kokkos/algorithms/unit_tests/TestStdAlgorithmsReverse.cpp @@ -62,7 +62,7 @@ void fill_view(ViewType dest_view, const std::string& name) { } else { - throw std::runtime_error("invalid choice"); + FAIL() << "invalid choice"; } Kokkos::deep_copy(aux_view, v_h); diff --git a/lib/kokkos/algorithms/unit_tests/TestStdAlgorithmsRotate.cpp b/lib/kokkos/algorithms/unit_tests/TestStdAlgorithmsRotate.cpp index 5638cbee4a..bf5c2ee782 100644 --- a/lib/kokkos/algorithms/unit_tests/TestStdAlgorithmsRotate.cpp +++ b/lib/kokkos/algorithms/unit_tests/TestStdAlgorithmsRotate.cpp @@ -117,7 +117,7 @@ void fill_view(ViewType dest_view, const std::string& name) { } else { - throw std::runtime_error("invalid choice"); + FAIL() << "invalid choice"; } Kokkos::deep_copy(aux_view, v_h); diff --git a/lib/kokkos/algorithms/unit_tests/TestStdAlgorithmsRotateCopy.cpp b/lib/kokkos/algorithms/unit_tests/TestStdAlgorithmsRotateCopy.cpp index d0caca7cea..1a860c58ce 100644 --- a/lib/kokkos/algorithms/unit_tests/TestStdAlgorithmsRotateCopy.cpp +++ b/lib/kokkos/algorithms/unit_tests/TestStdAlgorithmsRotateCopy.cpp @@ -117,7 +117,7 @@ void fill_view(ViewType dest_view, const std::string& name) { } else { - throw std::runtime_error("invalid choice"); + FAIL() << "invalid choice"; } Kokkos::deep_copy(aux_view, v_h); @@ -175,7 +175,7 @@ void run_single_scenario(const InfoType& scenario_info, create_view(Tag{}, view_ext, "rotate_copy_dest"); auto n_it = KE::cbegin(view_from) + rotation_point; auto rit = KE::rotate_copy(exespace(), KE::cbegin(view_from), n_it, - KE::cend(view_from), KE::begin(view_dest)); + KE::cend(view_from), KE::begin(view_dest)); verify_data(view_from, view_dest, rotation_point); ASSERT_EQ(rit, (KE::begin(view_dest) + view_ext)); } diff --git a/lib/kokkos/algorithms/unit_tests/TestStdAlgorithmsSearch.cpp b/lib/kokkos/algorithms/unit_tests/TestStdAlgorithmsSearch.cpp index 021609c444..195f88a0b7 100644 --- a/lib/kokkos/algorithms/unit_tests/TestStdAlgorithmsSearch.cpp +++ b/lib/kokkos/algorithms/unit_tests/TestStdAlgorithmsSearch.cpp @@ -256,7 +256,7 @@ void run_single_scenario(const InfoType& scenario_info, std::size_t seq_ext, { auto myrit = KE::search(exespace(), KE::cbegin(view), KE::cend(view), - KE::cbegin(s_view), KE::cend(s_view), args...); + KE::cbegin(s_view), KE::cend(s_view), args...); const auto mydiff = myrit - KE::cbegin(view); const auto stddiff = stdrit - KE::cbegin(view_h); ASSERT_EQ(mydiff, stddiff); diff --git a/lib/kokkos/algorithms/unit_tests/TestStdAlgorithmsSearch_n.cpp b/lib/kokkos/algorithms/unit_tests/TestStdAlgorithmsSearch_n.cpp index 53ad8daa2e..79d88bec23 100644 --- a/lib/kokkos/algorithms/unit_tests/TestStdAlgorithmsSearch_n.cpp +++ b/lib/kokkos/algorithms/unit_tests/TestStdAlgorithmsSearch_n.cpp @@ -154,7 +154,7 @@ void fill_view(ViewType dest_view, ValueType value, std::size_t count, } else { - throw std::runtime_error("Kokkos: test: search_n: this should not happen"); + FAIL() << "Kokkos: test: search_n: this should not happen"; } Kokkos::deep_copy(aux_view, v_h); @@ -208,7 +208,7 @@ void run_single_scenario(const InfoType& scenario_info, std::size_t count, { auto myrit = KE::search_n("label", exespace(), KE::cbegin(view), - KE::cend(view), count, value, args...); + KE::cend(view), count, value, args...); const auto mydiff = myrit - KE::cbegin(view); ASSERT_EQ(mydiff, stddiff); } diff --git a/lib/kokkos/algorithms/unit_tests/TestStdAlgorithmsShiftLeft.cpp b/lib/kokkos/algorithms/unit_tests/TestStdAlgorithmsShiftLeft.cpp index 0b5fe9216e..12835d5a2f 100644 --- a/lib/kokkos/algorithms/unit_tests/TestStdAlgorithmsShiftLeft.cpp +++ b/lib/kokkos/algorithms/unit_tests/TestStdAlgorithmsShiftLeft.cpp @@ -150,7 +150,7 @@ void run_single_scenario(const InfoType& scenario_info, // create host copy BEFORE shift_left or view will be modified auto view_h = create_host_space_copy(view); auto rit = KE::shift_left("label", exespace(), KE::begin(view), - KE::end(view), shift_value); + KE::end(view), shift_value); verify_data(rit, view, view_h, shift_value); } diff --git a/lib/kokkos/algorithms/unit_tests/TestStdAlgorithmsShiftRight.cpp b/lib/kokkos/algorithms/unit_tests/TestStdAlgorithmsShiftRight.cpp index 8e4ae94375..3e350cf3b3 100644 --- a/lib/kokkos/algorithms/unit_tests/TestStdAlgorithmsShiftRight.cpp +++ b/lib/kokkos/algorithms/unit_tests/TestStdAlgorithmsShiftRight.cpp @@ -141,7 +141,7 @@ void run_single_scenario(const InfoType& scenario_info, // create host copy BEFORE shift_right or view will be modified auto view_h = create_host_space_copy(view); auto rit = KE::shift_right(exespace(), KE::begin(view), KE::end(view), - shift_value); + shift_value); verify_data(rit, view, view_h, shift_value); } @@ -152,7 +152,7 @@ void run_single_scenario(const InfoType& scenario_info, // create host copy BEFORE shift_right or view will be modified auto view_h = create_host_space_copy(view); auto rit = KE::shift_right("label", exespace(), KE::begin(view), - KE::end(view), shift_value); + KE::end(view), shift_value); verify_data(rit, view, view_h, shift_value); } diff --git a/lib/kokkos/algorithms/unit_tests/TestStdAlgorithmsTeamAdjacentDifference.cpp b/lib/kokkos/algorithms/unit_tests/TestStdAlgorithmsTeamAdjacentDifference.cpp index c388cadc9b..5a2c046939 100644 --- a/lib/kokkos/algorithms/unit_tests/TestStdAlgorithmsTeamAdjacentDifference.cpp +++ b/lib/kokkos/algorithms/unit_tests/TestStdAlgorithmsTeamAdjacentDifference.cpp @@ -62,8 +62,8 @@ struct TestFunctorA { switch (m_apiPick) { case 0: { auto it = KE::adjacent_difference(member, KE::cbegin(myRowViewFrom), - KE::cend(myRowViewFrom), - KE::begin(myRowViewDest)); + KE::cend(myRowViewFrom), + KE::begin(myRowViewDest)); resultDist = KE::distance(KE::begin(myRowViewDest), it); Kokkos::single(Kokkos::PerTeam(member), [=, *this]() { m_distancesView(myRowIndex) = resultDist; @@ -73,8 +73,8 @@ struct TestFunctorA { case 1: { auto it = KE::adjacent_difference(member, KE::cbegin(myRowViewFrom), - KE::cend(myRowViewFrom), - KE::begin(myRowViewDest), m_binaryOp); + KE::cend(myRowViewFrom), + KE::begin(myRowViewDest), m_binaryOp); resultDist = KE::distance(KE::begin(myRowViewDest), it); Kokkos::single(Kokkos::PerTeam(member), [=, *this]() { m_distancesView(myRowIndex) = resultDist; diff --git a/lib/kokkos/algorithms/unit_tests/TestStdAlgorithmsTeamCopy.cpp b/lib/kokkos/algorithms/unit_tests/TestStdAlgorithmsTeamCopy.cpp index e24ac37bf0..071ecd5a9a 100644 --- a/lib/kokkos/algorithms/unit_tests/TestStdAlgorithmsTeamCopy.cpp +++ b/lib/kokkos/algorithms/unit_tests/TestStdAlgorithmsTeamCopy.cpp @@ -50,7 +50,7 @@ struct TestFunctorA { if (m_apiPick == 0) { auto it = KE::copy(member, KE::begin(myRowViewFrom), - KE::end(myRowViewFrom), KE::begin(myRowViewDest)); + KE::end(myRowViewFrom), KE::begin(myRowViewDest)); resultDist = KE::distance(KE::begin(myRowViewDest), it); Kokkos::single(Kokkos::PerTeam(member), [=, *this]() { m_distancesView(myRowIndex) = resultDist; diff --git a/lib/kokkos/algorithms/unit_tests/TestStdAlgorithmsTeamCopyIf.cpp b/lib/kokkos/algorithms/unit_tests/TestStdAlgorithmsTeamCopyIf.cpp index 7c3c465dc8..3f83ac7404 100644 --- a/lib/kokkos/algorithms/unit_tests/TestStdAlgorithmsTeamCopyIf.cpp +++ b/lib/kokkos/algorithms/unit_tests/TestStdAlgorithmsTeamCopyIf.cpp @@ -144,7 +144,7 @@ void test_A(std::size_t numTeams, std::size_t numCols, int apiId) { auto rowFrom = Kokkos::subview(sourceViewBeforeOp_h, i, Kokkos::ALL()); auto rowDest = Kokkos::subview(stdDestView, i, Kokkos::ALL()); auto it = std::copy_if(KE::cbegin(rowFrom), KE::cend(rowFrom), - KE::begin(rowDest), predicate); + KE::begin(rowDest), predicate); const std::size_t stdDistance = KE::distance(KE::begin(rowDest), it); ASSERT_EQ(stdDistance, distancesView_h(i)); ASSERT_TRUE(intraTeamSentinelView_h(i)); diff --git a/lib/kokkos/algorithms/unit_tests/TestStdAlgorithmsTeamCopy_n.cpp b/lib/kokkos/algorithms/unit_tests/TestStdAlgorithmsTeamCopy_n.cpp index 7cbc788f8e..9b509af55b 100644 --- a/lib/kokkos/algorithms/unit_tests/TestStdAlgorithmsTeamCopy_n.cpp +++ b/lib/kokkos/algorithms/unit_tests/TestStdAlgorithmsTeamCopy_n.cpp @@ -53,7 +53,7 @@ struct TestFunctorA { if (m_apiPick == 0) { auto it = KE::copy_n(member, KE::begin(myRowViewFrom), m_copyCount, - KE::begin(myRowViewDest)); + KE::begin(myRowViewDest)); resultDist = KE::distance(KE::begin(myRowViewDest), it); Kokkos::single(Kokkos::PerTeam(member), [=, *this]() { m_distancesView(myRowIndex) = resultDist; diff --git a/lib/kokkos/algorithms/unit_tests/TestStdAlgorithmsTeamCount.cpp b/lib/kokkos/algorithms/unit_tests/TestStdAlgorithmsTeamCount.cpp index 922424afbd..38df5c30ce 100644 --- a/lib/kokkos/algorithms/unit_tests/TestStdAlgorithmsTeamCount.cpp +++ b/lib/kokkos/algorithms/unit_tests/TestStdAlgorithmsTeamCount.cpp @@ -111,7 +111,7 @@ void test_A(const bool searched_value_exist, std::size_t numTeams, using rand_pool = Kokkos::Random_XorShift64_Pool; - rand_pool pool(lowerBound * upperBound); + rand_pool pool(static_cast(lowerBound) * upperBound); if (searched_value_exist) { Kokkos::View randomIndices( diff --git a/lib/kokkos/algorithms/unit_tests/TestStdAlgorithmsTeamExclusiveScan.cpp b/lib/kokkos/algorithms/unit_tests/TestStdAlgorithmsTeamExclusiveScan.cpp index 7cb9851087..0c35c5e599 100644 --- a/lib/kokkos/algorithms/unit_tests/TestStdAlgorithmsTeamExclusiveScan.cpp +++ b/lib/kokkos/algorithms/unit_tests/TestStdAlgorithmsTeamExclusiveScan.cpp @@ -67,8 +67,8 @@ struct TestFunctorA { switch (m_apiPick) { case 0: { auto it = KE::exclusive_scan(member, KE::cbegin(rowViewSrc), - KE::cend(rowViewSrc), - KE::begin(rowViewDest), initVal); + KE::cend(rowViewSrc), + KE::begin(rowViewDest), initVal); resultDist = KE::distance(KE::begin(rowViewDest), it); Kokkos::single(Kokkos::PerTeam(member), [=, *this] { m_distancesView(rowIndex) = resultDist; }); diff --git a/lib/kokkos/algorithms/unit_tests/TestStdAlgorithmsTeamFind.cpp b/lib/kokkos/algorithms/unit_tests/TestStdAlgorithmsTeamFind.cpp index 430e4917e0..88c5e21f31 100644 --- a/lib/kokkos/algorithms/unit_tests/TestStdAlgorithmsTeamFind.cpp +++ b/lib/kokkos/algorithms/unit_tests/TestStdAlgorithmsTeamFind.cpp @@ -51,7 +51,7 @@ struct TestFunctorA { switch (m_apiPick) { case 0: { auto it = KE::find(member, KE::cbegin(myRowViewFrom), - KE::cend(myRowViewFrom), searchedValue); + KE::cend(myRowViewFrom), searchedValue); resultDist = KE::distance(KE::cbegin(myRowViewFrom), it); Kokkos::single(Kokkos::PerTeam(member), [=, *this]() { m_distancesView(myRowIndex) = resultDist; diff --git a/lib/kokkos/algorithms/unit_tests/TestStdAlgorithmsTeamFindEnd.cpp b/lib/kokkos/algorithms/unit_tests/TestStdAlgorithmsTeamFindEnd.cpp index 83eca33569..d350bc62cd 100644 --- a/lib/kokkos/algorithms/unit_tests/TestStdAlgorithmsTeamFindEnd.cpp +++ b/lib/kokkos/algorithms/unit_tests/TestStdAlgorithmsTeamFindEnd.cpp @@ -86,9 +86,9 @@ struct TestFunctorA { case 2: { auto it = KE::find_end(member, KE::cbegin(myRowViewFrom), - KE::cend(myRowViewFrom), - KE::cbegin(myRowSearchedSeqView), - KE::cend(myRowSearchedSeqView), m_binaryPred); + KE::cend(myRowViewFrom), + KE::cbegin(myRowSearchedSeqView), + KE::cend(myRowSearchedSeqView), m_binaryPred); resultDist = KE::distance(KE::cbegin(myRowViewFrom), it); Kokkos::single(Kokkos::PerTeam(member), [=, *this]() { m_distancesView(myRowIndex) = resultDist; @@ -99,7 +99,7 @@ struct TestFunctorA { case 3: { auto it = KE::find_end(member, myRowViewFrom, myRowSearchedSeqView, - m_binaryPred); + m_binaryPred); resultDist = KE::distance(KE::begin(myRowViewFrom), it); Kokkos::single(Kokkos::PerTeam(member), [=, *this]() { m_distancesView(myRowIndex) = resultDist; diff --git a/lib/kokkos/algorithms/unit_tests/TestStdAlgorithmsTeamFindIf.cpp b/lib/kokkos/algorithms/unit_tests/TestStdAlgorithmsTeamFindIf.cpp index ee4bbed7a3..70f2be77f6 100644 --- a/lib/kokkos/algorithms/unit_tests/TestStdAlgorithmsTeamFindIf.cpp +++ b/lib/kokkos/algorithms/unit_tests/TestStdAlgorithmsTeamFindIf.cpp @@ -70,7 +70,7 @@ struct TestFunctorA { switch (m_apiPick) { case 0: { auto it = KE::find_if(member, KE::cbegin(myRowViewFrom), - KE::cend(myRowViewFrom), unaryPred); + KE::cend(myRowViewFrom), unaryPred); resultDist = KE::distance(KE::cbegin(myRowViewFrom), it); Kokkos::single(Kokkos::PerTeam(member), [=, *this]() { m_distancesView(myRowIndex) = resultDist; diff --git a/lib/kokkos/algorithms/unit_tests/TestStdAlgorithmsTeamFindIfNot.cpp b/lib/kokkos/algorithms/unit_tests/TestStdAlgorithmsTeamFindIfNot.cpp index b9448c1a3e..873e8faf4c 100644 --- a/lib/kokkos/algorithms/unit_tests/TestStdAlgorithmsTeamFindIfNot.cpp +++ b/lib/kokkos/algorithms/unit_tests/TestStdAlgorithmsTeamFindIfNot.cpp @@ -70,7 +70,7 @@ struct TestFunctorA { switch (m_apiPick) { case 0: { auto it = KE::find_if_not(member, KE::cbegin(myRowViewFrom), - KE::cend(myRowViewFrom), unaryPred); + KE::cend(myRowViewFrom), unaryPred); resultDist = KE::distance(KE::cbegin(myRowViewFrom), it); Kokkos::single(Kokkos::PerTeam(member), [=, *this]() { m_distancesView(myRowIndex) = resultDist; diff --git a/lib/kokkos/algorithms/unit_tests/TestStdAlgorithmsTeamGenerate_n.cpp b/lib/kokkos/algorithms/unit_tests/TestStdAlgorithmsTeamGenerate_n.cpp index 4b66dd9131..265cdf4746 100644 --- a/lib/kokkos/algorithms/unit_tests/TestStdAlgorithmsTeamGenerate_n.cpp +++ b/lib/kokkos/algorithms/unit_tests/TestStdAlgorithmsTeamGenerate_n.cpp @@ -63,7 +63,7 @@ struct TestFunctorA { }); } else if (m_apiPick == 1) { auto it = KE::generate_n(member, myRowView, m_count, - GenerateFunctor()); + GenerateFunctor()); resultDist = KE::distance(KE::begin(myRowView), it); Kokkos::single(Kokkos::PerTeam(member), [=, *this]() { m_distancesView(myRowIndex) = resultDist; diff --git a/lib/kokkos/algorithms/unit_tests/TestStdAlgorithmsTeamIsSorted.cpp b/lib/kokkos/algorithms/unit_tests/TestStdAlgorithmsTeamIsSorted.cpp index 850e80dde1..f76a595b3f 100644 --- a/lib/kokkos/algorithms/unit_tests/TestStdAlgorithmsTeamIsSorted.cpp +++ b/lib/kokkos/algorithms/unit_tests/TestStdAlgorithmsTeamIsSorted.cpp @@ -62,7 +62,7 @@ struct TestFunctorA { } else if (m_apiPick == 3) { using value_type = typename ViewType::value_type; result = KE::is_sorted(member, myRowView, - CustomLessThanComparator{}); + CustomLessThanComparator{}); Kokkos::single(Kokkos::PerTeam(member), [=, *this]() { m_returnsView(myRowIndex) = result; }); } diff --git a/lib/kokkos/algorithms/unit_tests/TestStdAlgorithmsTeamIsSortedUntil.cpp b/lib/kokkos/algorithms/unit_tests/TestStdAlgorithmsTeamIsSortedUntil.cpp index e3b95527c7..5bc49e4600 100644 --- a/lib/kokkos/algorithms/unit_tests/TestStdAlgorithmsTeamIsSortedUntil.cpp +++ b/lib/kokkos/algorithms/unit_tests/TestStdAlgorithmsTeamIsSortedUntil.cpp @@ -61,7 +61,7 @@ struct TestFunctorA { if (m_apiPick == 0) { auto it = KE::is_sorted_until(member, KE::cbegin(myRowView), - KE::cend(myRowView)); + KE::cend(myRowView)); resultDist = KE::distance(KE::cbegin(myRowView), it); Kokkos::single(Kokkos::PerTeam(member), [=, *this]() { m_distancesView(myRowIndex) = resultDist; @@ -77,8 +77,8 @@ struct TestFunctorA { else if (m_apiPick == 2) { using value_type = typename ViewType::value_type; auto it = KE::is_sorted_until(member, KE::cbegin(myRowView), - KE::cend(myRowView), - CustomLessThanComparator{}); + KE::cend(myRowView), + CustomLessThanComparator{}); resultDist = KE::distance(KE::cbegin(myRowView), it); Kokkos::single(Kokkos::PerTeam(member), [=, *this]() { m_distancesView(myRowIndex) = resultDist; @@ -88,7 +88,7 @@ struct TestFunctorA { else if (m_apiPick == 3) { using value_type = typename ViewType::value_type; auto it = KE::is_sorted_until(member, myRowView, - CustomLessThanComparator{}); + CustomLessThanComparator{}); resultDist = KE::distance(KE::begin(myRowView), it); Kokkos::single(Kokkos::PerTeam(member), [=, *this]() { m_distancesView(myRowIndex) = resultDist; @@ -210,7 +210,7 @@ void test_A(std::size_t numTeams, std::size_t numCols, int apiId, stdDistance = KE::distance(KE::cbegin(myRow), it); } else { auto it = std::is_sorted_until(KE::cbegin(myRow), KE::cend(myRow), - CustomLessThanComparator{}); + CustomLessThanComparator{}); stdDistance = KE::distance(KE::cbegin(myRow), it); } ASSERT_EQ(stdDistance, distancesView_h(i)); diff --git a/lib/kokkos/algorithms/unit_tests/TestStdAlgorithmsTeamMaxElement.cpp b/lib/kokkos/algorithms/unit_tests/TestStdAlgorithmsTeamMaxElement.cpp index 283525dbd1..452a48df21 100644 --- a/lib/kokkos/algorithms/unit_tests/TestStdAlgorithmsTeamMaxElement.cpp +++ b/lib/kokkos/algorithms/unit_tests/TestStdAlgorithmsTeamMaxElement.cpp @@ -74,7 +74,7 @@ struct TestFunctorA { else if (m_apiPick == 3) { using value_type = typename ViewType::value_type; auto it = KE::max_element(member, myRowView, - CustomLessThanComparator{}); + CustomLessThanComparator{}); resultDist = KE::distance(KE::begin(myRowView), it); Kokkos::single(Kokkos::PerTeam(member), [=, *this]() { m_distancesView(myRowIndex) = resultDist; @@ -144,7 +144,7 @@ void test_A(std::size_t numTeams, std::size_t numCols, int apiId) { stdDistance = KE::distance(KE::cbegin(myRow), it); } else { auto it = std::max_element(KE::cbegin(myRow), KE::cend(myRow), - CustomLessThanComparator{}); + CustomLessThanComparator{}); stdDistance = KE::distance(KE::cbegin(myRow), it); } diff --git a/lib/kokkos/algorithms/unit_tests/TestStdAlgorithmsTeamMinElement.cpp b/lib/kokkos/algorithms/unit_tests/TestStdAlgorithmsTeamMinElement.cpp index 8579b48315..2c79370b92 100644 --- a/lib/kokkos/algorithms/unit_tests/TestStdAlgorithmsTeamMinElement.cpp +++ b/lib/kokkos/algorithms/unit_tests/TestStdAlgorithmsTeamMinElement.cpp @@ -74,7 +74,7 @@ struct TestFunctorA { else if (m_apiPick == 3) { using value_type = typename ViewType::value_type; auto it = KE::min_element(member, myRowView, - CustomLessThanComparator{}); + CustomLessThanComparator{}); resultDist = KE::distance(KE::begin(myRowView), it); Kokkos::single(Kokkos::PerTeam(member), [=, *this]() { m_distancesView(myRowIndex) = resultDist; @@ -144,7 +144,7 @@ void test_A(std::size_t numTeams, std::size_t numCols, int apiId) { stdDistance = KE::distance(KE::cbegin(myRow), it); } else { auto it = std::min_element(KE::cbegin(myRow), KE::cend(myRow), - CustomLessThanComparator{}); + CustomLessThanComparator{}); stdDistance = KE::distance(KE::cbegin(myRow), it); } ASSERT_EQ(stdDistance, distancesView_h(i)); diff --git a/lib/kokkos/algorithms/unit_tests/TestStdAlgorithmsTeamMinMaxElement.cpp b/lib/kokkos/algorithms/unit_tests/TestStdAlgorithmsTeamMinMaxElement.cpp index 51010fdff5..25a4487855 100644 --- a/lib/kokkos/algorithms/unit_tests/TestStdAlgorithmsTeamMinMaxElement.cpp +++ b/lib/kokkos/algorithms/unit_tests/TestStdAlgorithmsTeamMinMaxElement.cpp @@ -84,7 +84,7 @@ struct TestFunctorA { else if (m_apiPick == 3) { using value_type = typename ViewType::value_type; auto itPair = KE::minmax_element(member, myRowView, - CustomLessThanComparator{}); + CustomLessThanComparator{}); resultDist1 = KE::distance(KE::begin(myRowView), itPair.first); resultDist2 = KE::distance(KE::begin(myRowView), itPair.second); @@ -160,7 +160,7 @@ void test_A(std::size_t numTeams, std::size_t numCols, int apiId) { stdDistance[1] = KE::distance(KE::cbegin(myRow), itPair.second); } else { auto itPair = std::minmax_element(KE::cbegin(myRow), KE::cend(myRow), - CustomLessThanComparator{}); + CustomLessThanComparator{}); stdDistance[0] = KE::distance(KE::cbegin(myRow), itPair.first); stdDistance[1] = KE::distance(KE::cbegin(myRow), itPair.second); } diff --git a/lib/kokkos/algorithms/unit_tests/TestStdAlgorithmsTeamMove.cpp b/lib/kokkos/algorithms/unit_tests/TestStdAlgorithmsTeamMove.cpp index 1122d6d554..2c445dacf8 100644 --- a/lib/kokkos/algorithms/unit_tests/TestStdAlgorithmsTeamMove.cpp +++ b/lib/kokkos/algorithms/unit_tests/TestStdAlgorithmsTeamMove.cpp @@ -50,7 +50,7 @@ struct TestFunctorA { if (m_apiPick == 0) { auto it = KE::move(member, KE::begin(myRowViewFrom), - KE::end(myRowViewFrom), KE::begin(myRowViewDest)); + KE::end(myRowViewFrom), KE::begin(myRowViewDest)); resultDist = KE::distance(KE::begin(myRowViewDest), it); Kokkos::single(Kokkos::PerTeam(member), [=, *this]() { m_distancesView(myRowIndex) = resultDist; diff --git a/lib/kokkos/algorithms/unit_tests/TestStdAlgorithmsTeamRemove.cpp b/lib/kokkos/algorithms/unit_tests/TestStdAlgorithmsTeamRemove.cpp index fb9c70391b..2defa1dc6f 100644 --- a/lib/kokkos/algorithms/unit_tests/TestStdAlgorithmsTeamRemove.cpp +++ b/lib/kokkos/algorithms/unit_tests/TestStdAlgorithmsTeamRemove.cpp @@ -63,7 +63,7 @@ struct TestFunctorA { if (m_apiPick == 0) { auto it = KE::remove(member, KE::begin(myRowView), KE::end(myRowView), - m_targetValue); + m_targetValue); resultDist = KE::distance(KE::begin(myRowView), it); Kokkos::single(Kokkos::PerTeam(member), [=, *this]() { m_distancesView(myRowIndex) = resultDist; diff --git a/lib/kokkos/algorithms/unit_tests/TestStdAlgorithmsTeamRemoveCopy.cpp b/lib/kokkos/algorithms/unit_tests/TestStdAlgorithmsTeamRemoveCopy.cpp index 6bb0d24998..71a50e39e3 100644 --- a/lib/kokkos/algorithms/unit_tests/TestStdAlgorithmsTeamRemoveCopy.cpp +++ b/lib/kokkos/algorithms/unit_tests/TestStdAlgorithmsTeamRemoveCopy.cpp @@ -67,8 +67,8 @@ struct TestFunctorA { if (m_apiPick == 0) { auto it = KE::remove_copy(member, KE::cbegin(myRowViewFrom), - KE::cend(myRowViewFrom), - KE::begin(myRowViewDest), m_targetValue); + KE::cend(myRowViewFrom), + KE::begin(myRowViewDest), m_targetValue); resultDist = KE::distance(KE::begin(myRowViewDest), it); Kokkos::single(Kokkos::PerTeam(member), [=, *this]() { m_distancesView(myRowIndex) = resultDist; diff --git a/lib/kokkos/algorithms/unit_tests/TestStdAlgorithmsTeamRemoveCopyIf.cpp b/lib/kokkos/algorithms/unit_tests/TestStdAlgorithmsTeamRemoveCopyIf.cpp index cff9aa178a..d5b5304f63 100644 --- a/lib/kokkos/algorithms/unit_tests/TestStdAlgorithmsTeamRemoveCopyIf.cpp +++ b/lib/kokkos/algorithms/unit_tests/TestStdAlgorithmsTeamRemoveCopyIf.cpp @@ -65,8 +65,8 @@ struct TestFunctorA { GreaterThanValueFunctor predicate(m_threshold); if (m_apiPick == 0) { auto it = KE::remove_copy_if(member, KE::cbegin(myRowViewFrom), - KE::cend(myRowViewFrom), - KE::begin(myRowViewDest), predicate); + KE::cend(myRowViewFrom), + KE::begin(myRowViewDest), predicate); resultDist = KE::distance(KE::begin(myRowViewDest), it); Kokkos::single(Kokkos::PerTeam(member), [=, *this]() { m_distancesView(myRowIndex) = resultDist; diff --git a/lib/kokkos/algorithms/unit_tests/TestStdAlgorithmsTeamReplaceCopy.cpp b/lib/kokkos/algorithms/unit_tests/TestStdAlgorithmsTeamReplaceCopy.cpp index 70dbf10574..64f172e401 100644 --- a/lib/kokkos/algorithms/unit_tests/TestStdAlgorithmsTeamReplaceCopy.cpp +++ b/lib/kokkos/algorithms/unit_tests/TestStdAlgorithmsTeamReplaceCopy.cpp @@ -78,7 +78,7 @@ struct TestFunctorA { }); } else if (m_apiPick == 1) { auto it = KE::replace_copy(member, myRowViewFrom, myRowViewDest, - m_targetValue, m_newValue); + m_targetValue, m_newValue); resultDist = KE::distance(KE::begin(myRowViewDest), it); Kokkos::single(Kokkos::PerTeam(member), [=, *this]() { m_distancesView(myRowIndex) = resultDist; @@ -172,7 +172,7 @@ void test_A(std::size_t numTeams, std::size_t numCols, int apiId) { auto rowFrom = Kokkos::subview(sourceView_dc_h, i, Kokkos::ALL()); auto rowDest = Kokkos::subview(stdDestView, i, Kokkos::ALL()); auto it = std::replace_copy(KE::cbegin(rowFrom), KE::cend(rowFrom), - KE::begin(rowDest), targetVal, newVal); + KE::begin(rowDest), targetVal, newVal); const std::size_t stdDistance = KE::distance(KE::begin(rowDest), it); ASSERT_EQ(stdDistance, distancesView_h(i)); ASSERT_TRUE(intraTeamSentinelView_h(i)); diff --git a/lib/kokkos/algorithms/unit_tests/TestStdAlgorithmsTeamReplaceCopyIf.cpp b/lib/kokkos/algorithms/unit_tests/TestStdAlgorithmsTeamReplaceCopyIf.cpp index d0217aed7a..9c3699320d 100644 --- a/lib/kokkos/algorithms/unit_tests/TestStdAlgorithmsTeamReplaceCopyIf.cpp +++ b/lib/kokkos/algorithms/unit_tests/TestStdAlgorithmsTeamReplaceCopyIf.cpp @@ -76,7 +76,7 @@ struct TestFunctorA { }); } else if (m_apiPick == 1) { auto it = KE::replace_copy_if(member, myRowViewFrom, myRowViewDest, - predicate, m_newValue); + predicate, m_newValue); resultDist = KE::distance(KE::begin(myRowViewDest), it); Kokkos::single(Kokkos::PerTeam(member), [=, *this]() { m_distancesView(myRowIndex) = resultDist; @@ -151,7 +151,7 @@ void test_A(std::size_t numTeams, std::size_t numCols, int apiId) { Kokkos::subview(cloneOfSourceViewBeforeOp_h, i, Kokkos::ALL()); auto rowDest = Kokkos::subview(stdDestView, i, Kokkos::ALL()); auto it = std::replace_copy_if(KE::cbegin(rowFrom), KE::cend(rowFrom), - KE::begin(rowDest), predicate, newVal); + KE::begin(rowDest), predicate, newVal); const std::size_t stdDistance = KE::distance(KE::begin(rowDest), it); ASSERT_EQ(stdDistance, distancesView_h(i)); ASSERT_TRUE(intraTeamSentinelView_h(i)); diff --git a/lib/kokkos/algorithms/unit_tests/TestStdAlgorithmsTeamRotateCopy.cpp b/lib/kokkos/algorithms/unit_tests/TestStdAlgorithmsTeamRotateCopy.cpp index e865b998f6..51f600faba 100644 --- a/lib/kokkos/algorithms/unit_tests/TestStdAlgorithmsTeamRotateCopy.cpp +++ b/lib/kokkos/algorithms/unit_tests/TestStdAlgorithmsTeamRotateCopy.cpp @@ -136,7 +136,7 @@ void test_A(std::size_t numTeams, std::size_t numCols, std::size_t pivotShift, auto pivot = KE::cbegin(myRowFrom) + pivotShift; auto it = std::rotate_copy(KE::cbegin(myRowFrom), pivot, - KE::cend(myRowFrom), KE::begin(myRowDest)); + KE::cend(myRowFrom), KE::begin(myRowDest)); const std::size_t stdDistance = KE::distance(KE::begin(myRowDest), it); ASSERT_EQ(stdDistance, distancesView_h(i)); ASSERT_TRUE(intraTeamSentinelView_h(i)); diff --git a/lib/kokkos/algorithms/unit_tests/TestStdAlgorithmsTeamShiftRight.cpp b/lib/kokkos/algorithms/unit_tests/TestStdAlgorithmsTeamShiftRight.cpp index 00a80c5ef0..08ff8fbbca 100644 --- a/lib/kokkos/algorithms/unit_tests/TestStdAlgorithmsTeamShiftRight.cpp +++ b/lib/kokkos/algorithms/unit_tests/TestStdAlgorithmsTeamShiftRight.cpp @@ -47,7 +47,7 @@ struct TestFunctorA { if (m_apiPick == 0) { auto it = KE::shift_right(member, KE::begin(myRowView), - KE::end(myRowView), m_shift); + KE::end(myRowView), m_shift); resultDist = KE::distance(KE::begin(myRowView), it); Kokkos::single(Kokkos::PerTeam(member), [=, *this]() { m_distancesView(myRowIndex) = resultDist; diff --git a/lib/kokkos/algorithms/unit_tests/TestStdAlgorithmsTeamSwapRanges.cpp b/lib/kokkos/algorithms/unit_tests/TestStdAlgorithmsTeamSwapRanges.cpp index 5fc9612caa..60cb3f0837 100644 --- a/lib/kokkos/algorithms/unit_tests/TestStdAlgorithmsTeamSwapRanges.cpp +++ b/lib/kokkos/algorithms/unit_tests/TestStdAlgorithmsTeamSwapRanges.cpp @@ -49,7 +49,7 @@ struct TestFunctorA { if (m_apiPick == 0) { auto it = KE::swap_ranges(member, KE::begin(myRowView1), - KE::end(myRowView1), KE::begin(myRowView2)); + KE::end(myRowView1), KE::begin(myRowView2)); resultDist = KE::distance(KE::begin(myRowView2), it); Kokkos::single(Kokkos::PerTeam(member), [=, *this]() { m_distancesView(myRowIndex) = resultDist; diff --git a/lib/kokkos/algorithms/unit_tests/TestStdAlgorithmsTeamTransformInclusiveScan.cpp b/lib/kokkos/algorithms/unit_tests/TestStdAlgorithmsTeamTransformInclusiveScan.cpp index 0b0d798fd8..78a21c4430 100644 --- a/lib/kokkos/algorithms/unit_tests/TestStdAlgorithmsTeamTransformInclusiveScan.cpp +++ b/lib/kokkos/algorithms/unit_tests/TestStdAlgorithmsTeamTransformInclusiveScan.cpp @@ -91,7 +91,7 @@ struct TestFunctorA { case 1: { auto it = KE::transform_inclusive_scan(member, srcRow, destRow, - m_binaryOp, m_unaryOp); + m_binaryOp, m_unaryOp); resultDist = KE::distance(firstDest, it); Kokkos::single(Kokkos::PerTeam(member), [=, *this] { m_distancesView(rowIndex) = resultDist; }); @@ -111,7 +111,7 @@ struct TestFunctorA { case 3: { auto it = KE::transform_inclusive_scan(member, srcRow, destRow, - m_binaryOp, m_unaryOp, initVal); + m_binaryOp, m_unaryOp, initVal); resultDist = KE::distance(firstDest, it); Kokkos::single(Kokkos::PerTeam(member), [=, *this] { m_distancesView(rowIndex) = resultDist; }); diff --git a/lib/kokkos/algorithms/unit_tests/TestStdAlgorithmsTeamUnique.cpp b/lib/kokkos/algorithms/unit_tests/TestStdAlgorithmsTeamUnique.cpp index c46146e0a8..cef0f7c13d 100644 --- a/lib/kokkos/algorithms/unit_tests/TestStdAlgorithmsTeamUnique.cpp +++ b/lib/kokkos/algorithms/unit_tests/TestStdAlgorithmsTeamUnique.cpp @@ -58,7 +58,7 @@ struct TestFunctorA { } else if (m_apiPick == 2) { using value_type = typename ViewType::value_type; auto it = KE::unique(member, KE::begin(myRowView), KE::end(myRowView), - CustomEqualityComparator{}); + CustomEqualityComparator{}); resultDist = KE::distance(KE::begin(myRowView), it); Kokkos::single(Kokkos::PerTeam(member), [=, *this]() { m_distancesView(myRowIndex) = resultDist; @@ -138,7 +138,7 @@ void test_A(std::size_t numTeams, std::size_t numCols, int apiId) { stdDistance = KE::distance(KE::begin(myRow), it); } else { auto it = std::unique(KE::begin(myRow), KE::end(myRow), - CustomEqualityComparator{}); + CustomEqualityComparator{}); stdDistance = KE::distance(KE::begin(myRow), it); } ASSERT_EQ(stdDistance, distancesView_h(i)); diff --git a/lib/kokkos/algorithms/unit_tests/TestStdAlgorithmsTeamUniqueCopy.cpp b/lib/kokkos/algorithms/unit_tests/TestStdAlgorithmsTeamUniqueCopy.cpp index 0d3289e196..89ea8154c7 100644 --- a/lib/kokkos/algorithms/unit_tests/TestStdAlgorithmsTeamUniqueCopy.cpp +++ b/lib/kokkos/algorithms/unit_tests/TestStdAlgorithmsTeamUniqueCopy.cpp @@ -72,8 +72,8 @@ struct TestFunctorA { using comparator_t = CustomEqualityComparator; auto it = KE::unique_copy(member, KE::begin(myRowViewFrom), - KE::end(myRowViewFrom), - KE::begin(myRowViewDest), comparator_t()); + KE::end(myRowViewFrom), + KE::begin(myRowViewDest), comparator_t()); resultDist = KE::distance(KE::begin(myRowViewDest), it); Kokkos::single(Kokkos::PerTeam(member), [=, *this]() { m_distancesView(myRowIndex) = resultDist; @@ -159,12 +159,12 @@ void test_A(std::size_t numTeams, std::size_t numCols, int apiId) { std::size_t stdDistance = 0; if (apiId <= 1) { auto it = std::unique_copy(KE::cbegin(myRowFrom), KE::cend(myRowFrom), - KE::begin(myRowDest)); + KE::begin(myRowDest)); stdDistance = KE::distance(KE::begin(myRowDest), it); } else { auto it = std::unique_copy(KE::cbegin(myRowFrom), KE::cend(myRowFrom), - KE::begin(myRowDest), - CustomEqualityComparator{}); + KE::begin(myRowDest), + CustomEqualityComparator{}); stdDistance = KE::distance(KE::begin(myRowDest), it); } ASSERT_EQ(stdDistance, distancesView_h(i)); diff --git a/lib/kokkos/algorithms/unit_tests/TestStdAlgorithmsTransformExclusiveScan.cpp b/lib/kokkos/algorithms/unit_tests/TestStdAlgorithmsTransformExclusiveScan.cpp index fa2804256a..365ca21688 100644 --- a/lib/kokkos/algorithms/unit_tests/TestStdAlgorithmsTransformExclusiveScan.cpp +++ b/lib/kokkos/algorithms/unit_tests/TestStdAlgorithmsTransformExclusiveScan.cpp @@ -115,7 +115,7 @@ void fill_view(ViewType dest_view, const std::string& name) { } else { - throw std::runtime_error("invalid choice"); + FAIL() << "invalid choice"; } Kokkos::deep_copy(aux_view, v_h); @@ -161,7 +161,7 @@ void verify_data(ViewType1 data_view, // contains data create_mirror_view_and_copy(Kokkos::HostSpace(), test_view_dc); if (test_view_h.extent(0) > 0) { for (std::size_t i = 0; i < test_view_h.extent(0); ++i) { - if (std::is_same::value) { + if (std::is_same_v) { ASSERT_EQ(gold_h(i), test_view_h(i)); } else { const auto error = std::abs(gold_h(i) - test_view_h(i)); diff --git a/lib/kokkos/algorithms/unit_tests/TestStdAlgorithmsTransformInclusiveScan.cpp b/lib/kokkos/algorithms/unit_tests/TestStdAlgorithmsTransformInclusiveScan.cpp index fb81ae91b0..cc87262147 100644 --- a/lib/kokkos/algorithms/unit_tests/TestStdAlgorithmsTransformInclusiveScan.cpp +++ b/lib/kokkos/algorithms/unit_tests/TestStdAlgorithmsTransformInclusiveScan.cpp @@ -115,7 +115,7 @@ void fill_view(ViewType dest_view, const std::string& name) { } else { - throw std::runtime_error("invalid choice"); + FAIL() << "invalid choice"; } Kokkos::deep_copy(aux_view, v_h); @@ -173,7 +173,7 @@ void verify_data(ViewType1 data_view, // contains data create_mirror_view_and_copy(Kokkos::HostSpace(), test_view_dc); if (test_view_h.extent(0) > 0) { for (std::size_t i = 0; i < test_view_h.extent(0); ++i) { - if (std::is_same::value) { + if (std::is_same_v) { ASSERT_EQ(gold_h(i), test_view_h(i)); } else { const auto error = std::abs(gold_h(i) - test_view_h(i)); diff --git a/lib/kokkos/algorithms/unit_tests/TestStdAlgorithmsUnique.cpp b/lib/kokkos/algorithms/unit_tests/TestStdAlgorithmsUnique.cpp index 9c5ae0cf8a..6ee93e3d5f 100644 --- a/lib/kokkos/algorithms/unit_tests/TestStdAlgorithmsUnique.cpp +++ b/lib/kokkos/algorithms/unit_tests/TestStdAlgorithmsUnique.cpp @@ -138,7 +138,7 @@ void fill_view(ViewType dest_view, const std::string& name) { } else { - throw std::runtime_error("invalid choice"); + FAIL() << "invalid choice"; } Kokkos::deep_copy(aux_view, v_h); diff --git a/lib/kokkos/algorithms/unit_tests/TestStdAlgorithmsUniqueCopy.cpp b/lib/kokkos/algorithms/unit_tests/TestStdAlgorithmsUniqueCopy.cpp index 3cf43ad4db..e3e9696458 100644 --- a/lib/kokkos/algorithms/unit_tests/TestStdAlgorithmsUniqueCopy.cpp +++ b/lib/kokkos/algorithms/unit_tests/TestStdAlgorithmsUniqueCopy.cpp @@ -146,7 +146,7 @@ std::size_t fill_view(ViewType dest_view, const std::string& name) { } else { - throw std::runtime_error("invalid choice"); + Kokkos::abort("invalid choice"); } Kokkos::deep_copy(aux_view, v_h); @@ -235,7 +235,7 @@ void verify_data(const std::string& name, ViewTypeFrom view_from, } else { - throw std::runtime_error("invalid choice"); + FAIL() << "invalid choice"; } } diff --git a/lib/kokkos/algorithms/unit_tests/TestStdReducers.cpp b/lib/kokkos/algorithms/unit_tests/TestStdReducers.cpp index c05006a161..0044b93558 100644 --- a/lib/kokkos/algorithms/unit_tests/TestStdReducers.cpp +++ b/lib/kokkos/algorithms/unit_tests/TestStdReducers.cpp @@ -72,7 +72,7 @@ auto create_host_view_with_reduction_order_indices( result(8) = 7; result(9) = 5; } else { - throw std::runtime_error("test: Invalid enum"); + Kokkos::abort("test: Invalid enum"); } return result; @@ -80,7 +80,7 @@ auto create_host_view_with_reduction_order_indices( template auto run_min_or_max_test(ViewType view, StdReducersTestEnumOrder enValue) { - static_assert(std::is_same::value, + static_assert(std::is_same_v, "test is only enabled for HostSpace"); using view_value_type = typename ViewType::value_type; @@ -191,7 +191,7 @@ template void run_min_max_test(ViewType view, StdReducersTestEnumOrder enValue, const ValuesPair gold_values, const IndexPair gold_locs) { - static_assert(std::is_same::value, + static_assert(std::is_same_v, "test is only enabled for HostSpace"); using view_value_type = typename ViewType::value_type; diff --git a/lib/kokkos/appveyor.yml b/lib/kokkos/appveyor.yml deleted file mode 100644 index d0a5645ef7..0000000000 --- a/lib/kokkos/appveyor.yml +++ /dev/null @@ -1,10 +0,0 @@ -image: - - Visual Studio 2019 -clone_folder: c:\projects\source -build_script: -- cmd: >- - mkdir build && - cd build && - cmake c:\projects\source -DKokkos_ENABLE_IMPL_MDSPAN=OFF -DKokkos_ENABLE_TESTS=ON -DCMAKE_CXX_FLAGS="/W0 /EHsc" -DKokkos_ENABLE_DEPRECATED_CODE_4=ON -DKokkos_ENABLE_DEPRECATION_WARNINGS=OFF && - cmake --build . --target install && - ctest -C Debug --output-on-failure diff --git a/lib/kokkos/benchmarks/CMakeLists.txt b/lib/kokkos/benchmarks/CMakeLists.txt index 529ef393d9..968c8ae3bf 100644 --- a/lib/kokkos/benchmarks/CMakeLists.txt +++ b/lib/kokkos/benchmarks/CMakeLists.txt @@ -1,12 +1,12 @@ #FIXME_OPENMPTARGET - compiling in debug mode causes ICE. -KOKKOS_ADD_BENCHMARK_DIRECTORIES(atomic) -KOKKOS_ADD_BENCHMARK_DIRECTORIES(gather) -KOKKOS_ADD_BENCHMARK_DIRECTORIES(gups) -KOKKOS_ADD_BENCHMARK_DIRECTORIES(launch_latency) -KOKKOS_ADD_BENCHMARK_DIRECTORIES(stream) -KOKKOS_ADD_BENCHMARK_DIRECTORIES(view_copy_constructor) +kokkos_add_benchmark_directories(atomic) +kokkos_add_benchmark_directories(gather) +kokkos_add_benchmark_directories(gups) +kokkos_add_benchmark_directories(launch_latency) +kokkos_add_benchmark_directories(stream) +kokkos_add_benchmark_directories(view_copy_constructor) #FIXME_OPENMPTARGET - These two benchmarks cause ICE. Commenting them for now but a deeper analysis on the cause and a possible fix will follow. -IF(NOT Kokkos_ENABLE_OPENMPTARGET) - KOKKOS_ADD_BENCHMARK_DIRECTORIES(policy_performance) - KOKKOS_ADD_BENCHMARK_DIRECTORIES(bytes_and_flops) -ENDIF() +if(NOT Kokkos_ENABLE_OPENMPTARGET) + kokkos_add_benchmark_directories(policy_performance) + kokkos_add_benchmark_directories(bytes_and_flops) +endif() diff --git a/lib/kokkos/benchmarks/atomic/CMakeLists.txt b/lib/kokkos/benchmarks/atomic/CMakeLists.txt index 85f7412f49..7fda2bf6f6 100644 --- a/lib/kokkos/benchmarks/atomic/CMakeLists.txt +++ b/lib/kokkos/benchmarks/atomic/CMakeLists.txt @@ -1,4 +1 @@ -KOKKOS_ADD_EXECUTABLE( - atomic - SOURCES main.cpp -) +kokkos_add_executable(atomic SOURCES main.cpp) diff --git a/lib/kokkos/benchmarks/bytes_and_flops/CMakeLists.txt b/lib/kokkos/benchmarks/bytes_and_flops/CMakeLists.txt index 0ce44a6f1a..9c65d06ce2 100644 --- a/lib/kokkos/benchmarks/bytes_and_flops/CMakeLists.txt +++ b/lib/kokkos/benchmarks/bytes_and_flops/CMakeLists.txt @@ -1,4 +1,9 @@ -KOKKOS_ADD_EXECUTABLE( +kokkos_add_executable( bytes_and_flops - SOURCES bench_double.cpp bench_float.cpp bench_int32_t.cpp bench_int64_t.cpp main.cpp + SOURCES + bench_double.cpp + bench_float.cpp + bench_int32_t.cpp + bench_int64_t.cpp + main.cpp ) diff --git a/lib/kokkos/benchmarks/bytes_and_flops/bench_unroll_stride.hpp b/lib/kokkos/benchmarks/bytes_and_flops/bench_unroll_stride.hpp index 78cfd48eff..762cc988f1 100644 --- a/lib/kokkos/benchmarks/bytes_and_flops/bench_unroll_stride.hpp +++ b/lib/kokkos/benchmarks/bytes_and_flops/bench_unroll_stride.hpp @@ -17,9 +17,9 @@ template struct Run { static void run(int N, int K, int R, int F, int T, int S, int Ba, int I) { - Kokkos::View A("A", N, K); - Kokkos::View B("B", N, K); - Kokkos::View C("C", N, K); + Kokkos::View A("A", N, K); + Kokkos::View B("B", N, K); + Kokkos::View C("C", N, K); Kokkos::deep_copy(A, Scalar(1.5)); Kokkos::deep_copy(B, Scalar(2.5)); diff --git a/lib/kokkos/benchmarks/gather/CMakeLists.txt b/lib/kokkos/benchmarks/gather/CMakeLists.txt index 24c7062772..2de1ce85e6 100644 --- a/lib/kokkos/benchmarks/gather/CMakeLists.txt +++ b/lib/kokkos/benchmarks/gather/CMakeLists.txt @@ -1,4 +1 @@ -KOKKOS_ADD_EXECUTABLE( - gather - SOURCES main.cpp -) +kokkos_add_executable(gather SOURCES main.cpp) diff --git a/lib/kokkos/benchmarks/gups/CMakeLists.txt b/lib/kokkos/benchmarks/gups/CMakeLists.txt index 8de5b73cc6..dc70747029 100644 --- a/lib/kokkos/benchmarks/gups/CMakeLists.txt +++ b/lib/kokkos/benchmarks/gups/CMakeLists.txt @@ -1,4 +1 @@ -KOKKOS_ADD_EXECUTABLE( - gups - SOURCES gups.cpp -) +kokkos_add_executable(gups SOURCES gups.cpp) diff --git a/lib/kokkos/benchmarks/gups/gups.cpp b/lib/kokkos/benchmarks/gups/gups.cpp index 369052321d..e00f87968b 100644 --- a/lib/kokkos/benchmarks/gups/gups.cpp +++ b/lib/kokkos/benchmarks/gups/gups.cpp @@ -140,7 +140,7 @@ int run_benchmark(const Index indicesCount, const Index dataCount, break; } default: { - throw std::runtime_error("unexpected mode"); + Kokkos::abort("unexpected mode"); } } diff --git a/lib/kokkos/benchmarks/launch_latency/CMakeLists.txt b/lib/kokkos/benchmarks/launch_latency/CMakeLists.txt index bb14da749d..4775bf2261 100644 --- a/lib/kokkos/benchmarks/launch_latency/CMakeLists.txt +++ b/lib/kokkos/benchmarks/launch_latency/CMakeLists.txt @@ -1,4 +1 @@ -KOKKOS_ADD_EXECUTABLE( - launch_latency - SOURCES launch_latency.cpp -) +kokkos_add_executable(launch_latency SOURCES launch_latency.cpp) diff --git a/lib/kokkos/benchmarks/launch_latency/launch_latency.cpp b/lib/kokkos/benchmarks/launch_latency/launch_latency.cpp index 73b176ab8d..156c29af09 100644 --- a/lib/kokkos/benchmarks/launch_latency/launch_latency.cpp +++ b/lib/kokkos/benchmarks/launch_latency/launch_latency.cpp @@ -254,7 +254,7 @@ int main(int argc, char* argv[]) { else if (i == 3) K = atoi(arg.data()); else { - throw std::runtime_error("unexpected argument!"); + Kokkos::abort("unexpected argument!"); } } else if (arg == "--no-parallel-for") { opts.par_for = false; @@ -265,7 +265,7 @@ int main(int argc, char* argv[]) { } else { std::stringstream ss; ss << "unexpected argument \"" << arg << "\" at position " << i; - throw std::runtime_error(ss.str()); + Kokkos::abort(ss.str().c_str()); } } diff --git a/lib/kokkos/benchmarks/policy_performance/CMakeLists.txt b/lib/kokkos/benchmarks/policy_performance/CMakeLists.txt index 929b9c9702..4a939775c0 100644 --- a/lib/kokkos/benchmarks/policy_performance/CMakeLists.txt +++ b/lib/kokkos/benchmarks/policy_performance/CMakeLists.txt @@ -1,4 +1 @@ -KOKKOS_ADD_EXECUTABLE( - policy_performance - SOURCES main.cpp -) +kokkos_add_executable(policy_performance SOURCES main.cpp) diff --git a/lib/kokkos/benchmarks/stream/CMakeLists.txt b/lib/kokkos/benchmarks/stream/CMakeLists.txt index 0dded6e3a5..b096976c48 100644 --- a/lib/kokkos/benchmarks/stream/CMakeLists.txt +++ b/lib/kokkos/benchmarks/stream/CMakeLists.txt @@ -1,4 +1 @@ -KOKKOS_ADD_EXECUTABLE( - stream - SOURCES stream-kokkos.cpp -) +kokkos_add_executable(stream SOURCES stream-kokkos.cpp) diff --git a/lib/kokkos/benchmarks/view_copy_constructor/CMakeLists.txt b/lib/kokkos/benchmarks/view_copy_constructor/CMakeLists.txt index 50a331b2b3..f7bbc13b6e 100644 --- a/lib/kokkos/benchmarks/view_copy_constructor/CMakeLists.txt +++ b/lib/kokkos/benchmarks/view_copy_constructor/CMakeLists.txt @@ -1,4 +1 @@ -KOKKOS_ADD_EXECUTABLE( - view_copy_constructor - SOURCES view_copy_constructor.cpp -) +kokkos_add_executable(view_copy_constructor SOURCES view_copy_constructor.cpp) diff --git a/lib/kokkos/bin/kokkos_launch_compiler b/lib/kokkos/bin/kokkos_launch_compiler index d1f8896f91..ee3c29e96d 100755 --- a/lib/kokkos/bin/kokkos_launch_compiler +++ b/lib/kokkos/bin/kokkos_launch_compiler @@ -62,7 +62,7 @@ KOKKOS_COMPILER=${1} shift # store the expected C++ compiler -CXX_COMPILER=${1} +CXX_COMPILER=$(which "${1}") # remove the expected C++ compiler from the arguments shift @@ -84,7 +84,7 @@ shift # kokkos_launch_compiler ${KOKKOS_COMPILER} g++ g++ -c file.cpp -o file.o # results in this command being executed: # ${KOKKOS_COMPILER} -c file.cpp -o file.o -if [[ "${KOKKOS_DEPENDENCE}" -eq "0" || "${CXX_COMPILER}" != "${1}" ]]; then +if [[ "${KOKKOS_DEPENDENCE}" -eq "0" || "${CXX_COMPILER}" != $(which "${1}") ]]; then debug-message "$@" # the command does not depend on Kokkos so just execute the command w/o re-directing to ${KOKKOS_COMPILER} exec "$@" diff --git a/lib/kokkos/cmake/Dependencies.cmake b/lib/kokkos/cmake/Dependencies.cmake index fb1e73b579..2f70c2f038 100644 --- a/lib/kokkos/cmake/Dependencies.cmake +++ b/lib/kokkos/cmake/Dependencies.cmake @@ -1,5 +1,3 @@ -TRIBITS_PACKAGE_DEFINE_DEPENDENCIES( - LIB_OPTIONAL_TPLS Pthread CUDA HWLOC DLlib - ) +tribits_package_define_dependencies(LIB_OPTIONAL_TPLS Pthread CUDA HWLOC DLlib) -TRIBITS_TPL_TENTATIVELY_ENABLE(DLlib) +tribits_tpl_tentatively_enable(DLlib) diff --git a/lib/kokkos/cmake/KokkosCore_config.h.in b/lib/kokkos/cmake/KokkosCore_config.h.in index 08f128f2d1..44f81bb8ce 100644 --- a/lib/kokkos/cmake/KokkosCore_config.h.in +++ b/lib/kokkos/cmake/KokkosCore_config.h.in @@ -24,7 +24,6 @@ #cmakedefine KOKKOS_ENABLE_HIP #cmakedefine KOKKOS_ENABLE_HPX #cmakedefine KOKKOS_ENABLE_SYCL -#cmakedefine KOKKOS_IMPL_SYCL_DEVICE_GLOBAL_SUPPORTED /* General Settings */ #cmakedefine KOKKOS_ENABLE_CXX17 @@ -40,7 +39,10 @@ #cmakedefine KOKKOS_ENABLE_IMPL_CUDA_UNIFIED_MEMORY #cmakedefine KOKKOS_ENABLE_HIP_RELOCATABLE_DEVICE_CODE #cmakedefine KOKKOS_ENABLE_HIP_MULTIPLE_KERNEL_INSTANTIATIONS -#cmakedefine KOKKOS_ENABLE_IMPL_HIP_UNIFIED_MEMORY +#cmakedefine KOKKOS_ENABLE_IMPL_HIP_MALLOC_ASYNC +#cmakedefine KOKKOS_ENABLE_SYCL_RELOCATABLE_DEVICE_CODE +#cmakedefine KOKKOS_IMPL_SYCL_DEVICE_GLOBAL_SUPPORTED +#cmakedefine KOKKOS_ENABLE_OPENACC_FORCE_HOST_AS_DEVICE #cmakedefine KOKKOS_ENABLE_IMPL_HPX_ASYNC_DISPATCH #cmakedefine KOKKOS_ENABLE_DEBUG #cmakedefine KOKKOS_ENABLE_DEBUG_DUALVIEW_MODIFY_CHECK @@ -80,6 +82,7 @@ #cmakedefine KOKKOS_ARCH_POWER8 #cmakedefine KOKKOS_ARCH_POWER9 #cmakedefine KOKKOS_ARCH_RISCV_SG2042 +#cmakedefine KOKKOS_ARCH_RISCV_RVA22V #cmakedefine KOKKOS_ARCH_INTEL_GEN #cmakedefine KOKKOS_ARCH_INTEL_DG1 #cmakedefine KOKKOS_ARCH_INTEL_GEN9 @@ -118,10 +121,11 @@ #cmakedefine KOKKOS_ARCH_AMD_GFX90A #cmakedefine KOKKOS_ARCH_AMD_GFX940 #cmakedefine KOKKOS_ARCH_AMD_GFX942 +#cmakedefine KOKKOS_ARCH_AMD_GFX942_APU #cmakedefine KOKKOS_ARCH_AMD_GFX1030 #cmakedefine KOKKOS_ARCH_AMD_GFX1100 #cmakedefine KOKKOS_ARCH_AMD_GFX1103 -#cmakedefine KOKKOS_ARCH_AMD_GPU +#cmakedefine KOKKOS_ARCH_AMD_GPU "@KOKKOS_ARCH_AMD_GPU@" #cmakedefine KOKKOS_ARCH_VEGA // deprecated #cmakedefine KOKKOS_ARCH_VEGA906 // deprecated #cmakedefine KOKKOS_ARCH_VEGA908 // deprecated diff --git a/lib/kokkos/cmake/KokkosTrilinosConfig.cmake.in b/lib/kokkos/cmake/KokkosTrilinosConfig.cmake.in deleted file mode 100644 index 626ef5a8eb..0000000000 --- a/lib/kokkos/cmake/KokkosTrilinosConfig.cmake.in +++ /dev/null @@ -1,17 +0,0 @@ -IF (NOT TARGET Kokkos::kokkos) - # Compute the installation prefix relative to this file. - get_filename_component(KOKKOS_IMPORT_PREFIX "${CMAKE_CURRENT_LIST_FILE}" PATH) - get_filename_component(KOKKOS_IMPORT_PREFIX "${KOKKOS_IMPORT_PREFIX}" PATH) - get_filename_component(KOKKOS_IMPORT_PREFIX "${KOKKOS_IMPORT_PREFIX}" PATH) - get_filename_component(KOKKOS_IMPORT_PREFIX "${KOKKOS_IMPORT_PREFIX}" PATH) - if(KOKKOS_IMPORT_PREFIX STREQUAL "/") - set(KOKKOS_IMPORT_PREFIX "") - endif() - add_library(Kokkos::kokkos INTERFACE IMPORTED) - set_target_properties(Kokkos::kokkos PROPERTIES - INTERFACE_LINK_LIBRARIES "@Kokkos_LIBRARIES@;@KOKKOS_LINK_OPTIONS@" - INTERFACE_COMPILE_FEATURES "@KOKKOS_CXX_STANDARD_FEATURE@" - INTERFACE_COMPILE_OPTIONS "@KOKKOS_ALL_COMPILE_OPTIONS@" - INTERFACE_INCLUDE_DIRECTORIES "${KOKKOS_IMPORT_PREFIX}/include" - ) -ENDIF() diff --git a/lib/kokkos/cmake/Modules/CudaToolkit.cmake b/lib/kokkos/cmake/Modules/CudaToolkit.cmake index eda5541f7c..b8ac2048b5 100644 --- a/lib/kokkos/cmake/Modules/CudaToolkit.cmake +++ b/lib/kokkos/cmake/Modules/CudaToolkit.cmake @@ -483,38 +483,40 @@ endif() # Try language- or user-provided path first. if(CUDAToolkit_BIN_DIR) - find_program(CUDAToolkit_NVCC_EXECUTABLE + find_program( + CUDAToolkit_NVCC_EXECUTABLE NAMES nvcc nvcc.exe PATHS ${CUDAToolkit_BIN_DIR} NO_DEFAULT_PATH - ) + ) endif() # Search using CUDAToolkit_ROOT -find_program(CUDAToolkit_NVCC_EXECUTABLE +find_program( + CUDAToolkit_NVCC_EXECUTABLE NAMES nvcc nvcc.exe PATHS ENV CUDA_PATH PATH_SUFFIXES bin ) # If the user specified CUDAToolkit_ROOT but nvcc could not be found, this is an error. -if (NOT CUDAToolkit_NVCC_EXECUTABLE AND (DEFINED CUDAToolkit_ROOT OR DEFINED ENV{CUDAToolkit_ROOT})) +if(NOT CUDAToolkit_NVCC_EXECUTABLE AND (DEFINED CUDAToolkit_ROOT OR DEFINED ENV{CUDAToolkit_ROOT})) # Declare error messages now, print later depending on find_package args. set(fail_base "Could not find nvcc executable in path specified by") set(cuda_root_fail "${fail_base} CUDAToolkit_ROOT=${CUDAToolkit_ROOT}") set(env_cuda_root_fail "${fail_base} environment variable CUDAToolkit_ROOT=$ENV{CUDAToolkit_ROOT}") - if (CUDAToolkit_FIND_REQUIRED) - if (DEFINED CUDAToolkit_ROOT) + if(CUDAToolkit_FIND_REQUIRED) + if(DEFINED CUDAToolkit_ROOT) message(FATAL_ERROR ${cuda_root_fail}) - elseif (DEFINED ENV{CUDAToolkit_ROOT}) + elseif(DEFINED ENV{CUDAToolkit_ROOT}) message(FATAL_ERROR ${env_cuda_root_fail}) endif() else() - if (NOT CUDAToolkit_FIND_QUIETLY) - if (DEFINED CUDAToolkit_ROOT) + if(NOT CUDAToolkit_FIND_QUIETLY) + if(DEFINED CUDAToolkit_ROOT) message(STATUS ${cuda_root_fail}) - elseif (DEFINED ENV{CUDAToolkit_ROOT}) + elseif(DEFINED ENV{CUDAToolkit_ROOT}) message(STATUS ${env_cuda_root_fail}) endif() endif() @@ -535,9 +537,9 @@ endif() # We will also search the default symlink location /usr/local/cuda first since # if CUDAToolkit_ROOT is not specified, it is assumed that the symlinked # directory is the desired location. -if (NOT CUDAToolkit_NVCC_EXECUTABLE) - if (UNIX) - if (NOT APPLE) +if(NOT CUDAToolkit_NVCC_EXECUTABLE) + if(UNIX) + if(NOT APPLE) set(platform_base "/usr/local/cuda-") else() set(platform_base "/Developer/NVIDIA/CUDA-") @@ -550,10 +552,10 @@ if (NOT CUDAToolkit_NVCC_EXECUTABLE) file(GLOB possible_paths "${platform_base}*") # Iterate the glob results and create a descending list. set(possible_versions) - foreach (p ${possible_paths}) + foreach(p ${possible_paths}) # Extract version number from end of string string(REGEX MATCH "[0-9][0-9]?\\.[0-9]$" p_version ${p}) - if (IS_DIRECTORY ${p} AND p_version) + if(IS_DIRECTORY ${p} AND p_version) list(APPEND possible_versions ${p_version}) endif() endforeach() @@ -563,10 +565,10 @@ if (NOT CUDAToolkit_NVCC_EXECUTABLE) # every possible version of CUDA installed, this wouldn't create any # significant overhead. set(versions) - foreach (v ${possible_versions}) + foreach(v ${possible_versions}) list(LENGTH versions num_versions) # First version, nothing to compare with so just append. - if (num_versions EQUAL 0) + if(num_versions EQUAL 0) list(APPEND versions ${v}) else() # Loop through list. Insert at an index when comparison is @@ -574,9 +576,9 @@ if (NOT CUDAToolkit_NVCC_EXECUTABLE) # happen since this came from a glob list of directories. set(i 0) set(early_terminate FALSE) - while (i LESS num_versions) + while(i LESS num_versions) list(GET versions ${i} curr) - if (v VERSION_GREATER curr) + if(v VERSION_GREATER curr) list(INSERT versions ${i} ${v}) set(early_terminate TRUE) break() @@ -584,7 +586,7 @@ if (NOT CUDAToolkit_NVCC_EXECUTABLE) math(EXPR i "${i} + 1") endwhile() # If it did not get inserted, place it at the end. - if (NOT early_terminate) + if(NOT early_terminate) list(APPEND versions ${v}) endif() endif() @@ -592,17 +594,18 @@ if (NOT CUDAToolkit_NVCC_EXECUTABLE) # With a descending list of versions, populate possible paths to search. set(search_paths) - foreach (v ${versions}) + foreach(v ${versions}) list(APPEND search_paths "${platform_base}${v}") endforeach() # Force the global default /usr/local/cuda to the front on Unix. - if (UNIX) + if(UNIX) list(INSERT search_paths 0 "/usr/local/cuda") endif() # Now search for nvcc again using the platform default search paths. - find_program(CUDAToolkit_NVCC_EXECUTABLE + find_program( + CUDAToolkit_NVCC_EXECUTABLE NAMES nvcc nvcc.exe PATHS ${search_paths} PATH_SUFFIXES bin @@ -617,8 +620,8 @@ if (NOT CUDAToolkit_NVCC_EXECUTABLE) unset(early_terminate) unset(search_paths) - if (NOT CUDAToolkit_NVCC_EXECUTABLE) - if (CUDAToolkit_FIND_REQUIRED) + if(NOT CUDAToolkit_NVCC_EXECUTABLE) + if(CUDAToolkit_FIND_REQUIRED) message(FATAL_ERROR "Could not find nvcc, please set CUDAToolkit_ROOT.") elseif(NOT CUDAToolkit_FIND_QUIETLY) message(STATUS "Could not find nvcc, please set CUDAToolkit_ROOT.") @@ -636,8 +639,7 @@ if(NOT CUDAToolkit_BIN_DIR AND CUDAToolkit_NVCC_EXECUTABLE) unset(cuda_dir) endif() -if(CUDAToolkit_NVCC_EXECUTABLE AND - CUDAToolkit_NVCC_EXECUTABLE STREQUAL CMAKE_CUDA_COMPILER) +if(CUDAToolkit_NVCC_EXECUTABLE AND CUDAToolkit_NVCC_EXECUTABLE STREQUAL CMAKE_CUDA_COMPILER) # Need to set these based off the already computed CMAKE_CUDA_COMPILER_VERSION value # This if statement will always match, but is used to provide variables for MATCH 1,2,3... if(CMAKE_CUDA_COMPILER_VERSION MATCHES [=[([0-9]+)\.([0-9]+)\.([0-9]+)]=]) @@ -648,39 +650,38 @@ if(CUDAToolkit_NVCC_EXECUTABLE AND endif() else() # Compute the version by invoking nvcc - execute_process (COMMAND ${CUDAToolkit_NVCC_EXECUTABLE} "--version" OUTPUT_VARIABLE NVCC_OUT) + execute_process(COMMAND ${CUDAToolkit_NVCC_EXECUTABLE} "--version" OUTPUT_VARIABLE NVCC_OUT) if(NVCC_OUT MATCHES [=[ V([0-9]+)\.([0-9]+)\.([0-9]+)]=]) set(CUDAToolkit_VERSION_MAJOR "${CMAKE_MATCH_1}") set(CUDAToolkit_VERSION_MINOR "${CMAKE_MATCH_2}") set(CUDAToolkit_VERSION_PATCH "${CMAKE_MATCH_3}") - set(CUDAToolkit_VERSION "${CMAKE_MATCH_1}.${CMAKE_MATCH_2}.${CMAKE_MATCH_3}") + set(CUDAToolkit_VERSION "${CMAKE_MATCH_1}.${CMAKE_MATCH_2}.${CMAKE_MATCH_3}") endif() unset(NVCC_OUT) endif() - get_filename_component(CUDAToolkit_ROOT_DIR ${CUDAToolkit_BIN_DIR} DIRECTORY ABSOLUTE) # Handle cross compilation if(CMAKE_CROSSCOMPILING) if(CMAKE_SYSTEM_PROCESSOR STREQUAL "armv7-a") # Support for NVPACK - set (CUDAToolkit_TARGET_NAME "armv7-linux-androideabi") + set(CUDAToolkit_TARGET_NAME "armv7-linux-androideabi") elseif(CMAKE_SYSTEM_PROCESSOR MATCHES "arm") # Support for arm cross compilation set(CUDAToolkit_TARGET_NAME "armv7-linux-gnueabihf") elseif(CMAKE_SYSTEM_PROCESSOR MATCHES "aarch64") # Support for aarch64 cross compilation - if (ANDROID_ARCH_NAME STREQUAL "arm64") + if(ANDROID_ARCH_NAME STREQUAL "arm64") set(CUDAToolkit_TARGET_NAME "aarch64-linux-androideabi") else() set(CUDAToolkit_TARGET_NAME "aarch64-linux") - endif (ANDROID_ARCH_NAME STREQUAL "arm64") + endif(ANDROID_ARCH_NAME STREQUAL "arm64") elseif(CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64") - set(CUDAToolkit_TARGET_NAME "x86_64-linux") + set(CUDAToolkit_TARGET_NAME "x86_64-linux") endif() - if (EXISTS "${CUDAToolkit_ROOT_DIR}/targets/${CUDAToolkit_TARGET_NAME}") + if(EXISTS "${CUDAToolkit_ROOT_DIR}/targets/${CUDAToolkit_TARGET_NAME}") set(CUDAToolkit_TARGET_DIR "${CUDAToolkit_ROOT_DIR}/targets/${CUDAToolkit_TARGET_NAME}") # add known CUDA target root path to the set of directories we search for programs, libraries and headers list(PREPEND CMAKE_FIND_ROOT_PATH "${CUDAToolkit_TARGET_DIR}") @@ -702,25 +703,16 @@ else() set(_CUDAToolkit_Pop_Prefix True) endif() - # Find the include/ directory -find_path(CUDAToolkit_INCLUDE_DIR - NAMES cuda_runtime.h -) +find_path(CUDAToolkit_INCLUDE_DIR NAMES cuda_runtime.h) # And find the CUDA Runtime Library libcudart -find_library(CUDA_CUDART - NAMES cudart - PATH_SUFFIXES lib64 lib/x64 -) -if (NOT CUDA_CUDART) - find_library(CUDA_CUDART - NAMES cudart - PATH_SUFFIXES lib64/stubs lib/x64/stubs - ) +find_library(CUDA_CUDART NAMES cudart PATH_SUFFIXES lib64 lib/x64) +if(NOT CUDA_CUDART) + find_library(CUDA_CUDART NAMES cudart PATH_SUFFIXES lib64/stubs lib/x64/stubs) endif() -if (NOT CUDA_CUDART AND NOT CUDAToolkit_FIND_QUIETLY) +if(NOT CUDA_CUDART AND NOT CUDAToolkit_FIND_QUIETLY) message(STATUS "Unable to find cudart library.") endif() @@ -733,24 +725,17 @@ endif() #----------------------------------------------------------------------------- # Perform version comparison and validate all required variables are set. include(FindPackageHandleStandardArgs) -find_package_handle_standard_args(CUDAToolkit - REQUIRED_VARS - CUDAToolkit_INCLUDE_DIR - CUDA_CUDART - CUDAToolkit_NVCC_EXECUTABLE - VERSION_VAR - CUDAToolkit_VERSION +find_package_handle_standard_args( + CUDAToolkit REQUIRED_VARS CUDAToolkit_INCLUDE_DIR CUDA_CUDART CUDAToolkit_NVCC_EXECUTABLE + VERSION_VAR CUDAToolkit_VERSION ) -mark_as_advanced(CUDA_CUDART - CUDAToolkit_INCLUDE_DIR - CUDAToolkit_NVCC_EXECUTABLE - ) +mark_as_advanced(CUDA_CUDART CUDAToolkit_INCLUDE_DIR CUDAToolkit_NVCC_EXECUTABLE) #----------------------------------------------------------------------------- # Construct result variables if(CUDAToolkit_FOUND) - set(CUDAToolkit_INCLUDE_DIRS ${CUDAToolkit_INCLUDE_DIR}) - get_filename_component(CUDAToolkit_LIBRARY_DIR ${CUDA_CUDART} DIRECTORY ABSOLUTE) + set(CUDAToolkit_INCLUDE_DIRS ${CUDAToolkit_INCLUDE_DIR}) + get_filename_component(CUDAToolkit_LIBRARY_DIR ${CUDA_CUDART} DIRECTORY ABSOLUTE) endif() #----------------------------------------------------------------------------- @@ -762,27 +747,26 @@ if(CUDAToolkit_FOUND) set(search_names ${lib_name} ${arg_ALT}) - find_library(CUDA_${lib_name}_LIBRARY + find_library( + CUDA_${lib_name}_LIBRARY NAMES ${search_names} - HINTS ${CUDAToolkit_LIBRARY_DIR} - ENV CUDA_PATH - PATH_SUFFIXES nvidia/current lib64 lib/x64 lib - ${arg_EXTRA_PATH_SUFFIXES} + HINTS ${CUDAToolkit_LIBRARY_DIR} ENV CUDA_PATH + PATH_SUFFIXES nvidia/current lib64 lib/x64 lib ${arg_EXTRA_PATH_SUFFIXES} ) # Don't try any stub directories intil we have exhausted all other # search locations. if(NOT CUDA_${lib_name}_LIBRARY) - find_library(CUDA_${lib_name}_LIBRARY + find_library( + CUDA_${lib_name}_LIBRARY NAMES ${search_names} - HINTS ${CUDAToolkit_LIBRARY_DIR} - ENV CUDA_PATH + HINTS ${CUDAToolkit_LIBRARY_DIR} ENV CUDA_PATH PATH_SUFFIXES lib64/stubs lib/x64/stubs lib/stubs stubs ) endif() mark_as_advanced(CUDA_${lib_name}_LIBRARY) - if (NOT TARGET CUDA::${lib_name} AND CUDA_${lib_name}_LIBRARY) + if(NOT TARGET CUDA::${lib_name} AND CUDA_${lib_name}_LIBRARY) add_library(CUDA::${lib_name} IMPORTED INTERFACE) target_include_directories(CUDA::${lib_name} SYSTEM INTERFACE "${CUDAToolkit_INCLUDE_DIRS}") target_link_libraries(CUDA::${lib_name} INTERFACE "${CUDA_${lib_name}_LIBRARY}") @@ -800,16 +784,15 @@ if(CUDAToolkit_FOUND) target_link_directories(CUDA::toolkit INTERFACE "${CUDAToolkit_LIBRARY_DIR}") endif() - _CUDAToolkit_find_and_add_import_lib(cuda_driver ALT cuda) + _cudatoolkit_find_and_add_import_lib(cuda_driver ALT cuda) - _CUDAToolkit_find_and_add_import_lib(cudart) - _CUDAToolkit_find_and_add_import_lib(cudart_static) + _cudatoolkit_find_and_add_import_lib(cudart) + _cudatoolkit_find_and_add_import_lib(cudart_static) # setup dependencies that are required for cudart_static when building # on linux. These are generally only required when using the CUDA toolkit # when CUDA language is disabled - if(NOT TARGET CUDA::cudart_static_deps - AND TARGET CUDA::cudart_static) + if(NOT TARGET CUDA::cudart_static_deps AND TARGET CUDA::cudart_static) add_library(CUDA::cudart_static_deps IMPORTED INTERFACE) target_link_libraries(CUDA::cudart_static INTERFACE CUDA::cudart_static_deps) @@ -831,55 +814,64 @@ if(CUDAToolkit_FOUND) endif() endif() - _CUDAToolkit_find_and_add_import_lib(culibos) # it's a static library - foreach (cuda_lib cublas cufft curand cusparse nppc nvjpeg) - _CUDAToolkit_find_and_add_import_lib(${cuda_lib}) - _CUDAToolkit_find_and_add_import_lib(${cuda_lib}_static DEPS culibos) + _cudatoolkit_find_and_add_import_lib(culibos) # it's a static library + foreach(cuda_lib cublas cufft curand cusparse nppc nvjpeg) + _cudatoolkit_find_and_add_import_lib(${cuda_lib}) + _cudatoolkit_find_and_add_import_lib(${cuda_lib}_static DEPS culibos) endforeach() # cuFFTW depends on cuFFT - _CUDAToolkit_find_and_add_import_lib(cufftw DEPS cufft) - _CUDAToolkit_find_and_add_import_lib(cufftw DEPS cufft_static) + _cudatoolkit_find_and_add_import_lib(cufftw DEPS cufft) + _cudatoolkit_find_and_add_import_lib(cufftw DEPS cufft_static) # cuSOLVER depends on cuBLAS, and cuSPARSE - _CUDAToolkit_find_and_add_import_lib(cusolver DEPS cublas cusparse) - _CUDAToolkit_find_and_add_import_lib(cusolver_static DEPS cublas_static cusparse_static culibos) + _cudatoolkit_find_and_add_import_lib(cusolver DEPS cublas cusparse) + _cudatoolkit_find_and_add_import_lib(cusolver_static DEPS cublas_static cusparse_static culibos) # nvGRAPH depends on cuRAND, and cuSOLVER. - _CUDAToolkit_find_and_add_import_lib(nvgraph DEPS curand cusolver) - _CUDAToolkit_find_and_add_import_lib(nvgraph_static DEPS curand_static cusolver_static) + _cudatoolkit_find_and_add_import_lib(nvgraph DEPS curand cusolver) + _cudatoolkit_find_and_add_import_lib(nvgraph_static DEPS curand_static cusolver_static) # Process the majority of the NPP libraries. - foreach (cuda_lib nppial nppicc nppidei nppif nppig nppim nppist nppitc npps nppicom nppisu) - _CUDAToolkit_find_and_add_import_lib(${cuda_lib} DEPS nppc) - _CUDAToolkit_find_and_add_import_lib(${cuda_lib}_static DEPS nppc_static) + foreach( + cuda_lib + nppial + nppicc + nppidei + nppif + nppig + nppim + nppist + nppitc + npps + nppicom + nppisu + ) + _cudatoolkit_find_and_add_import_lib(${cuda_lib} DEPS nppc) + _cudatoolkit_find_and_add_import_lib(${cuda_lib}_static DEPS nppc_static) endforeach() - _CUDAToolkit_find_and_add_import_lib(cupti - EXTRA_PATH_SUFFIXES ../extras/CUPTI/lib64/ - ../extras/CUPTI/lib/) - _CUDAToolkit_find_and_add_import_lib(cupti_static - EXTRA_PATH_SUFFIXES ../extras/CUPTI/lib64/ - ../extras/CUPTI/lib/) + _cudatoolkit_find_and_add_import_lib(cupti EXTRA_PATH_SUFFIXES ../extras/CUPTI/lib64/ ../extras/CUPTI/lib/) + _cudatoolkit_find_and_add_import_lib(cupti_static EXTRA_PATH_SUFFIXES ../extras/CUPTI/lib64/ ../extras/CUPTI/lib/) - _CUDAToolkit_find_and_add_import_lib(nvrtc DEPS cuda_driver) + _cudatoolkit_find_and_add_import_lib(nvrtc DEPS cuda_driver) - _CUDAToolkit_find_and_add_import_lib(nvml ALT nvidia-ml nvml) + _cudatoolkit_find_and_add_import_lib(nvml ALT nvidia-ml nvml) if(WIN32) # nvtools can be installed outside the CUDA toolkit directory # so prefer the NVTOOLSEXT_PATH windows only environment variable # In addition on windows the most common name is nvToolsExt64_1 - find_library(CUDA_nvToolsExt_LIBRARY + find_library( + CUDA_nvToolsExt_LIBRARY NAMES nvToolsExt64_1 nvToolsExt64 nvToolsExt - PATHS ENV NVTOOLSEXT_PATH - ENV CUDA_PATH + PATHS ENV NVTOOLSEXT_PATH ENV CUDA_PATH PATH_SUFFIXES lib/x64 lib ) endif() - _CUDAToolkit_find_and_add_import_lib(nvToolsExt ALT nvToolsExt64) + _cudatoolkit_find_and_add_import_lib(nvToolsExt ALT nvToolsExt64) - _CUDAToolkit_find_and_add_import_lib(OpenCL) + _cudatoolkit_find_and_add_import_lib(OpenCL) endif() if(_CUDAToolkit_Pop_ROOT_PATH) diff --git a/lib/kokkos/cmake/Modules/FindTPLCUDA.cmake b/lib/kokkos/cmake/Modules/FindTPLCUDA.cmake index 445f4e93a5..3a6a826197 100644 --- a/lib/kokkos/cmake/Modules/FindTPLCUDA.cmake +++ b/lib/kokkos/cmake/Modules/FindTPLCUDA.cmake @@ -1,44 +1,40 @@ -IF (NOT CUDAToolkit_ROOT) - IF (NOT CUDA_ROOT) - SET(CUDA_ROOT $ENV{CUDA_ROOT}) - ENDIF() - IF(CUDA_ROOT) - SET(CUDAToolkit_ROOT ${CUDA_ROOT}) - ENDIF() -ENDIF() +if(NOT CUDAToolkit_ROOT) + if(NOT CUDA_ROOT) + set(CUDA_ROOT $ENV{CUDA_ROOT}) + endif() + if(CUDA_ROOT) + set(CUDAToolkit_ROOT ${CUDA_ROOT}) + endif() +endif() -IF(KOKKOS_CXX_HOST_COMPILER_ID STREQUAL NVHPC AND CMAKE_VERSION VERSION_LESS "3.20.1") - MESSAGE(FATAL_ERROR "Using NVHPC as host compiler requires at least CMake 3.20.1") -ENDIF() +if(KOKKOS_CXX_HOST_COMPILER_ID STREQUAL NVHPC AND CMAKE_VERSION VERSION_LESS "3.20.1") + message(FATAL_ERROR "Using NVHPC as host compiler requires at least CMake 3.20.1") +endif() -IF(CMAKE_VERSION VERSION_GREATER_EQUAL "3.17.0") +if(CMAKE_VERSION VERSION_GREATER_EQUAL "3.17.0") find_package(CUDAToolkit REQUIRED) - KOKKOS_CREATE_IMPORTED_TPL(CUDA INTERFACE - LINK_LIBRARIES CUDA::cuda_driver CUDA::cudart - ) - KOKKOS_EXPORT_CMAKE_TPL(CUDAToolkit REQUIRED) -ELSE() + kokkos_create_imported_tpl(CUDA INTERFACE LINK_LIBRARIES CUDA::cuda_driver CUDA::cudart) + kokkos_export_cmake_tpl(CUDAToolkit REQUIRED) +else() include(${CMAKE_CURRENT_LIST_DIR}/CudaToolkit.cmake) - IF (TARGET CUDA::cudart) - SET(FOUND_CUDART TRUE) - KOKKOS_EXPORT_IMPORTED_TPL(CUDA::cudart) - ELSE() - SET(FOUND_CUDART FALSE) - ENDIF() + if(TARGET CUDA::cudart) + set(FOUND_CUDART TRUE) + kokkos_export_imported_tpl(CUDA::cudart) + else() + set(FOUND_CUDART FALSE) + endif() - IF (TARGET CUDA::cuda_driver) - SET(FOUND_CUDA_DRIVER TRUE) - KOKKOS_EXPORT_IMPORTED_TPL(CUDA::cuda_driver) - ELSE() - SET(FOUND_CUDA_DRIVER FALSE) - ENDIF() + if(TARGET CUDA::cuda_driver) + set(FOUND_CUDA_DRIVER TRUE) + kokkos_export_imported_tpl(CUDA::cuda_driver) + else() + set(FOUND_CUDA_DRIVER FALSE) + endif() include(FindPackageHandleStandardArgs) - FIND_PACKAGE_HANDLE_STANDARD_ARGS(TPLCUDA ${DEFAULT_MSG} FOUND_CUDART FOUND_CUDA_DRIVER) - IF (FOUND_CUDA_DRIVER AND FOUND_CUDART) - KOKKOS_CREATE_IMPORTED_TPL(CUDA INTERFACE - LINK_LIBRARIES CUDA::cuda_driver CUDA::cudart - ) - ENDIF() -ENDIF() + find_package_handle_standard_args(TPLCUDA ${DEFAULT_MSG} FOUND_CUDART FOUND_CUDA_DRIVER) + if(FOUND_CUDA_DRIVER AND FOUND_CUDART) + kokkos_create_imported_tpl(CUDA INTERFACE LINK_LIBRARIES CUDA::cuda_driver CUDA::cudart) + endif() +endif() diff --git a/lib/kokkos/cmake/Modules/FindTPLHPX.cmake b/lib/kokkos/cmake/Modules/FindTPLHPX.cmake index d7b54fb9c9..e3c199b7c5 100644 --- a/lib/kokkos/cmake/Modules/FindTPLHPX.cmake +++ b/lib/kokkos/cmake/Modules/FindTPLHPX.cmake @@ -1,15 +1,10 @@ - -FIND_PACKAGE(HPX REQUIRED 1.8.0) +find_package(HPX REQUIRED 1.8.0) #as of right now, HPX doesn't export correctly #so let's convert it to an interface target -KOKKOS_CREATE_IMPORTED_TPL(HPX INTERFACE - LINK_LIBRARIES ${HPX_LIBRARIES} - INCLUDES ${HPX_INCLUDE_DIRS} -) +kokkos_create_imported_tpl(HPX INTERFACE LINK_LIBRARIES ${HPX_LIBRARIES} INCLUDES ${HPX_INCLUDE_DIRS}) #this is a bit funky since this is a CMake target #but HPX doesn't export itself correctly -KOKKOS_EXPORT_CMAKE_TPL(HPX) +kokkos_export_cmake_tpl(HPX) #I would prefer all of this gets replaced with #KOKKOS_IMPORT_CMAKE_TPL(HPX) - diff --git a/lib/kokkos/cmake/Modules/FindTPLHWLOC.cmake b/lib/kokkos/cmake/Modules/FindTPLHWLOC.cmake index cf763b7e5b..77ce8c71f7 100644 --- a/lib/kokkos/cmake/Modules/FindTPLHWLOC.cmake +++ b/lib/kokkos/cmake/Modules/FindTPLHWLOC.cmake @@ -1 +1 @@ -KOKKOS_FIND_IMPORTED(HWLOC HEADER hwloc.h LIBRARY hwloc) +kokkos_find_imported(HWLOC HEADER hwloc.h LIBRARY hwloc) diff --git a/lib/kokkos/cmake/Modules/FindTPLLIBDL.cmake b/lib/kokkos/cmake/Modules/FindTPLLIBDL.cmake index 8adcdcdbb8..85ae0b8224 100644 --- a/lib/kokkos/cmake/Modules/FindTPLLIBDL.cmake +++ b/lib/kokkos/cmake/Modules/FindTPLLIBDL.cmake @@ -1 +1 @@ -KOKKOS_FIND_IMPORTED(LIBDL HEADER dlfcn.h INTERFACE LIBRARIES ${CMAKE_DL_LIBS}) +kokkos_find_imported(LIBDL HEADER dlfcn.h INTERFACE LIBRARIES ${CMAKE_DL_LIBS}) diff --git a/lib/kokkos/cmake/Modules/FindTPLLIBQUADMATH.cmake b/lib/kokkos/cmake/Modules/FindTPLLIBQUADMATH.cmake index 70e0d6c454..ce428b0aee 100644 --- a/lib/kokkos/cmake/Modules/FindTPLLIBQUADMATH.cmake +++ b/lib/kokkos/cmake/Modules/FindTPLLIBQUADMATH.cmake @@ -2,17 +2,19 @@ # (which would not be contained in CMake's search paths anyway). # Hence, try if the compiler supports libquadmath natively first before doing # the standard package search. -SET(CMAKE_REQUIRED_LIBRARIES "quadmath") -INCLUDE(CheckCXXSourceCompiles) -CHECK_CXX_SOURCE_COMPILES(" +set(CMAKE_REQUIRED_LIBRARIES "quadmath") +include(CheckCXXSourceCompiles) +check_cxx_source_compiles( + " #include int main(void){ __float128 foo = ::sqrtq(123.456); return foo; }" - KOKKOS_QUADMATH_COMPILER_SUPPORT) -IF (KOKKOS_QUADMATH_COMPILER_SUPPORT) - KOKKOS_CREATE_IMPORTED_TPL(LIBQUADMATH INTERFACE LINK_LIBRARIES quadmath) -ELSE() - KOKKOS_FIND_IMPORTED(LIBQUADMATH HEADER quadmath.h LIBRARY quadmath) -ENDIF() + KOKKOS_QUADMATH_COMPILER_SUPPORT +) +if(KOKKOS_QUADMATH_COMPILER_SUPPORT) + kokkos_create_imported_tpl(LIBQUADMATH INTERFACE LINK_LIBRARIES quadmath) +else() + kokkos_find_imported(LIBQUADMATH HEADER quadmath.h LIBRARY quadmath) +endif() diff --git a/lib/kokkos/cmake/Modules/FindTPLONEDPL.cmake b/lib/kokkos/cmake/Modules/FindTPLONEDPL.cmake index 603510c315..68de942a69 100644 --- a/lib/kokkos/cmake/Modules/FindTPLONEDPL.cmake +++ b/lib/kokkos/cmake/Modules/FindTPLONEDPL.cmake @@ -1,9 +1,10 @@ -INCLUDE(CheckIncludeFileCXX) -CHECK_INCLUDE_FILE_CXX(oneapi/dpl/execution KOKKOS_COMPILER_HAS_ONEDPL_EXECUTION_HEADER) -CHECK_INCLUDE_FILE_CXX(oneapi/dpl/algorithm KOKKOS_COMPILER_HAS_ONEDPL_ALGORITHM_HEADER) +include(CheckIncludeFileCXX) +check_include_file_cxx(oneapi/dpl/execution KOKKOS_COMPILER_HAS_ONEDPL_EXECUTION_HEADER) +check_include_file_cxx(oneapi/dpl/algorithm KOKKOS_COMPILER_HAS_ONEDPL_ALGORITHM_HEADER) -INCLUDE(CheckCXXSourceCompiles) -CHECK_CXX_SOURCE_COMPILES(" +include(CheckCXXSourceCompiles) +check_cxx_source_compiles( + " #include int main() @@ -13,37 +14,40 @@ CHECK_CXX_SOURCE_COMPILES(" #endif return 0; }" - KOKKOS_NO_TBB_CONFLICT) + KOKKOS_NO_TBB_CONFLICT +) -IF (KOKKOS_COMPILER_HAS_ONEDPL_EXECUTION_HEADER AND KOKKOS_COMPILER_HAS_ONEDPL_ALGORITHM_HEADER) - IF(KOKKOS_NO_TBB_CONFLICT) - KOKKOS_CREATE_IMPORTED_TPL( - ONEDPL INTERFACE - ) - ELSE() - KOKKOS_CREATE_IMPORTED_TPL( - ONEDPL INTERFACE +if(KOKKOS_COMPILER_HAS_ONEDPL_EXECUTION_HEADER AND KOKKOS_COMPILER_HAS_ONEDPL_ALGORITHM_HEADER) + if(KOKKOS_NO_TBB_CONFLICT) + kokkos_create_imported_tpl(ONEDPL INTERFACE) + else() + kokkos_create_imported_tpl( + ONEDPL + INTERFACE # https://stackoverflow.com/questions/67923287/how-to-resolve-no-member-named-task-in-namespace-tbb-error-when-using-oned/ - COMPILE_DEFINITIONS PSTL_USE_PARALLEL_POLICIES=0 _GLIBCXX_USE_TBB_PAR_BACKEND=0 + COMPILE_DEFINITIONS + PSTL_USE_PARALLEL_POLICIES=0 + _GLIBCXX_USE_TBB_PAR_BACKEND=0 ) - ENDIF() -ELSE() - FIND_PACKAGE(oneDPL REQUIRED) + endif() +else() + find_package(oneDPL REQUIRED) - IF(KOKKOS_NO_TBB_CONFLICT) - KOKKOS_CREATE_IMPORTED_TPL( - ONEDPL INTERFACE - LINK_LIBRARIES oneDPL - ) - ELSE() - KOKKOS_CREATE_IMPORTED_TPL( - ONEDPL INTERFACE - LINK_LIBRARIES oneDPL + if(KOKKOS_NO_TBB_CONFLICT) + kokkos_create_imported_tpl(ONEDPL INTERFACE LINK_LIBRARIES oneDPL) + else() + kokkos_create_imported_tpl( + ONEDPL + INTERFACE + LINK_LIBRARIES + oneDPL # https://stackoverflow.com/questions/67923287/how-to-resolve-no-member-named-task-in-namespace-tbb-error-when-using-oned/ - COMPILE_DEFINITIONS PSTL_USE_PARALLEL_POLICIES=0 _GLIBCXX_USE_TBB_PAR_BACKEND=0 + COMPILE_DEFINITIONS + PSTL_USE_PARALLEL_POLICIES=0 + _GLIBCXX_USE_TBB_PAR_BACKEND=0 ) - ENDIF() + endif() # Export oneDPL as a Kokkos dependency - KOKKOS_EXPORT_CMAKE_TPL(oneDPL) -ENDIF() + kokkos_export_cmake_tpl(oneDPL) +endif() diff --git a/lib/kokkos/cmake/Modules/FindTPLROCM.cmake b/lib/kokkos/cmake/Modules/FindTPLROCM.cmake index f796737f5b..9673af0b9d 100644 --- a/lib/kokkos/cmake/Modules/FindTPLROCM.cmake +++ b/lib/kokkos/cmake/Modules/FindTPLROCM.cmake @@ -1,7 +1,7 @@ include(FindPackageHandleStandardArgs) -FIND_LIBRARY(AMD_HIP_LIBRARY amdhip64 PATHS ENV ROCM_PATH PATH_SUFFIXES lib) -FIND_LIBRARY(HSA_RUNTIME_LIBRARY hsa-runtime64 PATHS ENV ROCM_PATH PATH_SUFFIXES lib) +find_library(AMD_HIP_LIBRARY amdhip64 PATHS ENV ROCM_PATH PATH_SUFFIXES lib) +find_library(HSA_RUNTIME_LIBRARY hsa-runtime64 PATHS ENV ROCM_PATH PATH_SUFFIXES lib) # FIXME_HIP Starting with ROCm 5.5 it is not necessary to link againt clang_rt. # We keep the code as is for now because it is hard to find the version of ROCM @@ -16,18 +16,24 @@ execute_process( COMMAND ${CMAKE_CXX_COMPILER} -print-libgcc-file-name --rtlib=compiler-rt OUTPUT_VARIABLE CLANG_RT_LIBRARY OUTPUT_STRIP_TRAILING_WHITESPACE - RESULT_VARIABLE CLANG_RT_CHECK) + RESULT_VARIABLE CLANG_RT_CHECK +) -if( NOT "${CLANG_RT_CHECK}" STREQUAL "0" ) +if(NOT "${CLANG_RT_CHECK}" STREQUAL "0") # if the above failed, we delete CLANG_RT_LIBRARY to make the args check # below fail unset(CLANG_RT_LIBRARY) endif() - find_package_handle_standard_args(TPLROCM DEFAULT_MSG AMD_HIP_LIBRARY HSA_RUNTIME_LIBRARY CLANG_RT_LIBRARY) -kokkos_create_imported_tpl(ROCM INTERFACE - LINK_LIBRARIES ${HSA_RUNTIME_LIBRARY} ${AMD_HIP_LIBRARY} ${CLANG_RT_LIBRARY} - COMPILE_DEFINITIONS __HIP_ROCclr__ +kokkos_create_imported_tpl( + ROCM + INTERFACE + LINK_LIBRARIES + ${HSA_RUNTIME_LIBRARY} + ${AMD_HIP_LIBRARY} + ${CLANG_RT_LIBRARY} + COMPILE_DEFINITIONS + __HIP_ROCclr__ ) diff --git a/lib/kokkos/cmake/Modules/FindTPLROCTHRUST.cmake b/lib/kokkos/cmake/Modules/FindTPLROCTHRUST.cmake index dae7dc3c95..b4b905795d 100644 --- a/lib/kokkos/cmake/Modules/FindTPLROCTHRUST.cmake +++ b/lib/kokkos/cmake/Modules/FindTPLROCTHRUST.cmake @@ -6,10 +6,10 @@ # behavior of ROCm 5.7 and later for earlier version of ROCm we set # AMDGPU_TARGETS and GPU_TARGETS to empty and set the values in the cache. If # the values are not cached, FIND_PACKAGE(rocthrust) will overwrite them. -SET(AMDGPU_TARGETS "" CACHE STRING "AMD GPU targets to compile for") -SET(GPU_TARGETS "" CACHE STRING "GPU targets to compile for") -FIND_PACKAGE(rocthrust REQUIRED) -KOKKOS_CREATE_IMPORTED_TPL(ROCTHRUST INTERFACE LINK_LIBRARIES roc::rocthrust) +set(AMDGPU_TARGETS "" CACHE STRING "AMD GPU targets to compile for") +set(GPU_TARGETS "" CACHE STRING "GPU targets to compile for") +find_package(rocthrust REQUIRED) +kokkos_create_imported_tpl(ROCTHRUST INTERFACE LINK_LIBRARIES roc::rocthrust) # Export ROCTHRUST as a Kokkos dependency -KOKKOS_EXPORT_CMAKE_TPL(rocthrust) +kokkos_export_cmake_tpl(rocthrust) diff --git a/lib/kokkos/cmake/Modules/FindTPLTHREADS.cmake b/lib/kokkos/cmake/Modules/FindTPLTHREADS.cmake index ff0db5123f..280b8641da 100644 --- a/lib/kokkos/cmake/Modules/FindTPLTHREADS.cmake +++ b/lib/kokkos/cmake/Modules/FindTPLTHREADS.cmake @@ -1,15 +1,14 @@ -INCLUDE(FindPackageHandleStandardArgs) -FIND_PACKAGE(Threads) +include(FindPackageHandleStandardArgs) +find_package(Threads) -IF (TARGET Threads::Threads) - SET(FOUND_THREADS TRUE) -ELSE() - SET(FOUND_THREADS FALSE) -ENDIF() +if(TARGET Threads::Threads) + set(FOUND_THREADS TRUE) +else() + set(FOUND_THREADS FALSE) +endif() -FIND_PACKAGE_HANDLE_STANDARD_ARGS(TPLTHREADS DEFAULT_MSG FOUND_THREADS) +find_package_handle_standard_args(TPLTHREADS DEFAULT_MSG FOUND_THREADS) #Only create the TPL if we succeed -IF (FOUND_THREADS) - KOKKOS_CREATE_IMPORTED_TPL(THREADS INTERFACE LINK_OPTIONS - ${CMAKE_THREAD_LIBS_INIT}) -ENDIF() +if(FOUND_THREADS) + kokkos_create_imported_tpl(THREADS INTERFACE LINK_OPTIONS ${CMAKE_THREAD_LIBS_INIT}) +endif() diff --git a/lib/kokkos/cmake/README.md b/lib/kokkos/cmake/README.md index 385bbfcd5d..0548e89a90 100644 --- a/lib/kokkos/cmake/README.md +++ b/lib/kokkos/cmake/README.md @@ -310,20 +310,6 @@ When Kokkos is loaded by a downstream project, this TPL must be loaded. Calling this function simply appends text recording the location where the TPL was found and adding a `find_dependency(...)` call that will reload the CMake target. -### The Great TriBITS Compromise - -TriBITS was a masterpiece of CMake version 2 before the modern CMake idioms of building and using. -TriBITS greatly limited verbosity of CMake files, handled complicated dependency trees between packages, and handled automatically setting up include and linker paths for dependent libraries. - -Kokkos is now used by numerous projects that don't (and won't) depend on TriBITS for their build systems. -Kokkos has to work outside of TriBITS and provide a standard CMake 3+ build system. -At the same time, Kokkos is used by numerous projects that depend on TriBITS and don't (and won't) switch to a standard CMake 3+ build system. - -Instead of calling functions `TRIBITS_X(...)`, the CMake calls wrapper functions `KOKKOS_X(...)`. -If TriBITS is available (as in Trilinos), `KOKKOS_X` will just be a thin wrapper around `TRIBITS_X`. -If TriBITS is not available, Kokkos maps `KOKKOS_X` calls to native CMake that complies with CMake 3 idioms. -For the time being, this seems the most sensible way to handle the competing requirements of a standalone modern CMake and TriBITS build system. - ##### [LICENSE](https://github.com/kokkos/kokkos/blob/devel/LICENSE) [![License](https://img.shields.io/badge/License-BSD%203--Clause-blue.svg)](https://opensource.org/licenses/BSD-3-Clause) diff --git a/lib/kokkos/cmake/build_env_info.cmake b/lib/kokkos/cmake/build_env_info.cmake index 0eeb637245..76afbb74b6 100644 --- a/lib/kokkos/cmake/build_env_info.cmake +++ b/lib/kokkos/cmake/build_env_info.cmake @@ -2,121 +2,118 @@ find_package(Git QUIET) -SET(CURRENT_LIST_DIR ${CMAKE_CURRENT_LIST_DIR}) -SET(pre_configure_dir ${CMAKE_CURRENT_LIST_DIR}) -SET(post_configure_dir ${CMAKE_BINARY_DIR}/generated) +set(CURRENT_LIST_DIR ${CMAKE_CURRENT_LIST_DIR}) +set(pre_configure_dir ${CMAKE_CURRENT_LIST_DIR}) +set(post_configure_dir ${CMAKE_CURRENT_BINARY_DIR}/generated) -SET(pre_configure_file ${pre_configure_dir}/Kokkos_Version_Info.cpp.in) -SET(post_configure_file ${post_configure_dir}/Kokkos_Version_Info.cpp) +set(pre_configure_file ${pre_configure_dir}/Kokkos_Version_Info.cpp.in) +set(post_configure_file ${post_configure_dir}/Kokkos_Version_Info.cpp) -FUNCTION(check_git_write git_hash git_clean_status) - FILE( - WRITE - ${CMAKE_BINARY_DIR}/git-state.txt - "${git_hash}-${git_clean_status}") -ENDFUNCTION() +function(check_git_write git_hash git_clean_status) + file(WRITE ${CMAKE_BINARY_DIR}/git-state.txt "${git_hash}-${git_clean_status}") +endfunction() -FUNCTION(check_git_read git_hash) - IF(EXISTS ${CMAKE_BINARY_DIR}/git-state.txt) - FILE(STRINGS ${CMAKE_BINARY_DIR}/git-state.txt CONTENT) - LIST(GET CONTENT 0 var) +function(check_git_read git_hash) + if(EXISTS ${CMAKE_BINARY_DIR}/git-state.txt) + file(STRINGS ${CMAKE_BINARY_DIR}/git-state.txt CONTENT) + list(GET CONTENT 0 var) message(DEBUG "Cached Git hash: ${var}") - SET(${git_hash} ${var} PARENT_SCOPE) + set(${git_hash} ${var} PARENT_SCOPE) else() - SET(${git_hash} "INVALID" PARENT_SCOPE) - ENDIF() -ENDFUNCTION() + set(${git_hash} "INVALID" PARENT_SCOPE) + endif() +endfunction() -FUNCTION(check_git_version) - IF(NOT EXISTS ${post_configure_dir}/Kokkos_Version_Info.hpp) - FILE( - COPY ${pre_configure_dir}/Kokkos_Version_Info.hpp - DESTINATION ${post_configure_dir}) - ENDIF() +function(check_git_version) + if(NOT EXISTS ${post_configure_dir}/Kokkos_Version_Info.hpp) + file(COPY ${pre_configure_dir}/Kokkos_Version_Info.hpp DESTINATION ${post_configure_dir}) + endif() - IF(NOT Git_FOUND OR NOT EXISTS ${KOKKOS_SOURCE_DIR}/.git) + if(NOT Git_FOUND OR NOT EXISTS ${KOKKOS_SOURCE_DIR}/.git) configure_file(${pre_configure_file} ${post_configure_file} @ONLY) return() - ENDIF() + endif() # Get the current working branch execute_process( COMMAND ${GIT_EXECUTABLE} rev-parse --abbrev-ref HEAD WORKING_DIRECTORY ${KOKKOS_SOURCE_DIR} OUTPUT_VARIABLE GIT_BRANCH - OUTPUT_STRIP_TRAILING_WHITESPACE) + OUTPUT_STRIP_TRAILING_WHITESPACE + ) # Get the latest commit description execute_process( COMMAND ${GIT_EXECUTABLE} show -s --format=%s WORKING_DIRECTORY ${KOKKOS_SOURCE_DIR} OUTPUT_VARIABLE GIT_COMMIT_DESCRIPTION - OUTPUT_STRIP_TRAILING_WHITESPACE) + OUTPUT_STRIP_TRAILING_WHITESPACE + ) # Get the latest commit date execute_process( COMMAND ${GIT_EXECUTABLE} log -1 --format=%cI WORKING_DIRECTORY ${KOKKOS_SOURCE_DIR} OUTPUT_VARIABLE GIT_COMMIT_DATE - OUTPUT_STRIP_TRAILING_WHITESPACE) + OUTPUT_STRIP_TRAILING_WHITESPACE + ) # Check if repo is dirty / clean execute_process( COMMAND ${GIT_EXECUTABLE} diff-index --quiet HEAD -- WORKING_DIRECTORY ${KOKKOS_SOURCE_DIR} RESULT_VARIABLE IS_DIRTY - OUTPUT_STRIP_TRAILING_WHITESPACE) + OUTPUT_STRIP_TRAILING_WHITESPACE + ) - IF(IS_DIRTY EQUAL 0) - SET(GIT_CLEAN_STATUS "CLEAN") + if(IS_DIRTY EQUAL 0) + set(GIT_CLEAN_STATUS "CLEAN") else() - SET(GIT_CLEAN_STATUS "DIRTY") - ENDIF() + set(GIT_CLEAN_STATUS "DIRTY") + endif() # Get the latest abbreviated commit hash of the working branch execute_process( COMMAND ${GIT_EXECUTABLE} log -1 --format=%h WORKING_DIRECTORY ${KOKKOS_SOURCE_DIR} OUTPUT_VARIABLE GIT_COMMIT_HASH - OUTPUT_STRIP_TRAILING_WHITESPACE) + OUTPUT_STRIP_TRAILING_WHITESPACE + ) check_git_read(GIT_HASH_CACHE) - IF(NOT EXISTS ${post_configure_dir}) + if(NOT EXISTS ${post_configure_dir}) file(MAKE_DIRECTORY ${post_configure_dir}) - ENDIF() + endif() # Only update the git_version.cpp if the hash has changed. This will # prevent us from rebuilding the project more than we need to. - IF(NOT "${GIT_COMMIT_HASH}-${GIT_CLEAN_STATUS}" STREQUAL ${GIT_HASH_CACHE} - OR NOT EXISTS ${post_configure_file}) + if(NOT "${GIT_COMMIT_HASH}-${GIT_CLEAN_STATUS}" STREQUAL ${GIT_HASH_CACHE} OR NOT EXISTS ${post_configure_file}) # Set the GIT_HASH_CACHE variable so the next build won't have # to regenerate the source file. check_git_write(${GIT_COMMIT_HASH} ${GIT_CLEAN_STATUS}) configure_file(${pre_configure_file} ${post_configure_file} @ONLY) message(STATUS "Configured git information in ${post_configure_file}") - ENDIF() -ENDFUNCTION() + endif() +endfunction() -FUNCTION(check_git_setup) +function(check_git_setup) add_custom_target( - AlwaysCheckGit COMMAND ${CMAKE_COMMAND} - -DRUN_CHECK_GIT_VERSION=1 - -DKOKKOS_SOURCE_DIR=${Kokkos_SOURCE_DIR} - -P ${CURRENT_LIST_DIR}/build_env_info.cmake - BYPRODUCTS ${post_configure_file}) + AlwaysCheckGit COMMAND ${CMAKE_COMMAND} -DRUN_CHECK_GIT_VERSION=1 -DKOKKOS_SOURCE_DIR=${Kokkos_SOURCE_DIR} -P + ${CURRENT_LIST_DIR}/build_env_info.cmake BYPRODUCTS ${post_configure_file} + ) - add_library(impl_git_version ${CMAKE_BINARY_DIR}/generated/Kokkos_Version_Info.cpp) + add_library(impl_git_version ${CMAKE_CURRENT_BINARY_DIR}/generated/Kokkos_Version_Info.cpp) target_include_directories(impl_git_version PUBLIC ${CMAKE_BINARY_DIR}/generated) target_compile_features(impl_git_version PRIVATE cxx_raw_string_literals) add_dependencies(impl_git_version AlwaysCheckGit) check_git_version() -ENDFUNCTION() +endfunction() # This is used to run this function from an external cmake process. -IF(RUN_CHECK_GIT_VERSION) +if(RUN_CHECK_GIT_VERSION) check_git_version() -ENDIF() +endif() diff --git a/lib/kokkos/core/unit_test/sycl/TestSYCL_Task.cpp b/lib/kokkos/cmake/compile_tests/amd_apu.cc similarity index 57% rename from lib/kokkos/core/unit_test/sycl/TestSYCL_Task.cpp rename to lib/kokkos/cmake/compile_tests/amd_apu.cc index 3c599b95a6..a9c1edbd57 100644 --- a/lib/kokkos/core/unit_test/sycl/TestSYCL_Task.cpp +++ b/lib/kokkos/cmake/compile_tests/amd_apu.cc @@ -14,5 +14,25 @@ // //@HEADER -#include -#include +#include +#include + +int main() { + hipDeviceProp_t hipProp; + hipError_t error = hipGetDeviceProperties(&hipProp, 0); + + if (error != hipSuccess) { + std::cout << hipGetErrorString(error) << '\n'; + return error; + } + + if (hipProp.integrated == 1) { + // We detected an APU + std::cout << "ON"; + } else { + // We detected a discrete GPU + std::cout << "OFF"; + } + + return 0; +} diff --git a/lib/kokkos/cmake/cray.cmake b/lib/kokkos/cmake/cray.cmake index 08912f5130..4ce5352bda 100644 --- a/lib/kokkos/cmake/cray.cmake +++ b/lib/kokkos/cmake/cray.cmake @@ -1,9 +1,6 @@ - - function(kokkos_set_cray_flags full_standard int_standard) - STRING(TOLOWER ${full_standard} FULL_LC_STANDARD) - STRING(TOLOWER ${int_standard} INT_LC_STANDARD) - SET(KOKKOS_CXX_STANDARD_FLAG "-hstd=c++${FULL_LC_STANDARD}", PARENT_SCOPE) - SET(KOKKOS_CXX_INTERMDIATE_STANDARD_FLAG "-hstd=c++${INT_LC_STANDARD}" PARENT_SCOPE) + string(TOLOWER ${full_standard} FULL_LC_STANDARD) + string(TOLOWER ${int_standard} INT_LC_STANDARD) + set(KOKKOS_CXX_STANDARD_FLAG "-hstd=c++${FULL_LC_STANDARD}", PARENT_SCOPE) + set(KOKKOS_CXX_INTERMDIATE_STANDARD_FLAG "-hstd=c++${INT_LC_STANDARD}" PARENT_SCOPE) endfunction() - diff --git a/lib/kokkos/cmake/deps/CUDA.cmake b/lib/kokkos/cmake/deps/CUDA.cmake index 5b6afd6151..49eaf883a4 100644 --- a/lib/kokkos/cmake/deps/CUDA.cmake +++ b/lib/kokkos/cmake/deps/CUDA.cmake @@ -17,24 +17,24 @@ # Check for CUDA support -SET(_CUDA_FAILURE OFF) +set(_CUDA_FAILURE OFF) # Have CMake find CUDA -IF(NOT _CUDA_FAILURE) - FIND_PACKAGE(CUDA 3.2) - IF (NOT CUDA_FOUND) - SET(_CUDA_FAILURE ON) - ENDIF() -ENDIF() +if(NOT _CUDA_FAILURE) + find_package(CUDA 3.2) + if(NOT CUDA_FOUND) + set(_CUDA_FAILURE ON) + endif() +endif() -IF(NOT _CUDA_FAILURE) +if(NOT _CUDA_FAILURE) # if we haven't met failure macro(PACKAGE_ADD_CUDA_LIBRARY cuda_target) - TRIBITS_ADD_LIBRARY(${cuda_target} ${ARGN} CUDALIBRARY) + tribits_add_library(${cuda_target} ${ARGN} CUDALIBRARY) endmacro() - GLOBAL_SET(TPL_CUDA_LIBRARY_DIRS) - GLOBAL_SET(TPL_CUDA_INCLUDE_DIRS ${CUDA_TOOLKIT_INCLUDE}) - GLOBAL_SET(TPL_CUDA_LIBRARIES ${CUDA_CUDART_LIBRARY} ${CUDA_cublas_LIBRARY} ${CUDA_cufft_LIBRARY}) -ELSE() - SET(TPL_ENABLE_CUDA OFF) -ENDIF() + global_set(TPL_CUDA_LIBRARY_DIRS) + global_set(TPL_CUDA_INCLUDE_DIRS ${CUDA_TOOLKIT_INCLUDE}) + global_set(TPL_CUDA_LIBRARIES ${CUDA_CUDART_LIBRARY} ${CUDA_cublas_LIBRARY} ${CUDA_cufft_LIBRARY}) +else() + set(TPL_ENABLE_CUDA OFF) +endif() diff --git a/lib/kokkos/cmake/deps/HWLOC.cmake b/lib/kokkos/cmake/deps/HWLOC.cmake index 77d5a9b83a..52d8368d04 100644 --- a/lib/kokkos/cmake/deps/HWLOC.cmake +++ b/lib/kokkos/cmake/deps/HWLOC.cmake @@ -15,7 +15,6 @@ # ************************************************************************ # @HEADER - #----------------------------------------------------------------------------- # Hardware locality detection and control library. # @@ -26,7 +25,4 @@ # Version: 1.3 # -KOKKOS_TPL_FIND_INCLUDE_DIRS_AND_LIBRARIES( HWLOC - REQUIRED_HEADERS hwloc.h - REQUIRED_LIBS_NAMES "hwloc" - ) +kokkos_tpl_find_include_dirs_and_libraries(HWLOC REQUIRED_HEADERS hwloc.h REQUIRED_LIBS_NAMES "hwloc") diff --git a/lib/kokkos/cmake/deps/Pthread.cmake b/lib/kokkos/cmake/deps/Pthread.cmake index e879bff374..b811f85084 100644 --- a/lib/kokkos/cmake/deps/Pthread.cmake +++ b/lib/kokkos/cmake/deps/Pthread.cmake @@ -15,31 +15,27 @@ # ************************************************************************ # @HEADER +set(USE_THREADS FALSE) -SET(USE_THREADS FALSE) - -IF(NOT TPL_Pthread_INCLUDE_DIRS AND NOT TPL_Pthread_LIBRARY_DIRS AND NOT TPL_Pthread_LIBRARIES) +if(NOT TPL_Pthread_INCLUDE_DIRS AND NOT TPL_Pthread_LIBRARY_DIRS AND NOT TPL_Pthread_LIBRARIES) # Use CMake's Thread finder since it is a bit smarter in determining # whether pthreads is already built into the compiler and doesn't need # a library to link. - FIND_PACKAGE(Threads) + find_package(Threads) #If Threads found a copy of pthreads make sure it is one of the cases the tribits #tpl system cannot handle. - IF(Threads_FOUND AND CMAKE_USE_PTHREADS_INIT) - IF(CMAKE_THREAD_LIBS_INIT STREQUAL "" OR CMAKE_THREAD_LIBS_INIT STREQUAL "-pthread") - SET(USE_THREADS TRUE) - ENDIF() - ENDIF() -ENDIF() + if(Threads_FOUND AND CMAKE_USE_PTHREADS_INIT) + if(CMAKE_THREAD_LIBS_INIT STREQUAL "" OR CMAKE_THREAD_LIBS_INIT STREQUAL "-pthread") + set(USE_THREADS TRUE) + endif() + endif() +endif() -IF(USE_THREADS) - SET(TPL_Pthread_INCLUDE_DIRS "") - SET(TPL_Pthread_LIBRARIES "${CMAKE_THREAD_LIBS_INIT}") - SET(TPL_Pthread_LIBRARY_DIRS "") - KOKKOS_CREATE_IMPORTED_TPL_LIBRARY(Pthread) -ELSE() - KOKKOS_TPL_FIND_INCLUDE_DIRS_AND_LIBRARIES( Pthread - REQUIRED_HEADERS pthread.h - REQUIRED_LIBS_NAMES pthread - ) -ENDIF() +if(USE_THREADS) + set(TPL_Pthread_INCLUDE_DIRS "") + set(TPL_Pthread_LIBRARIES "${CMAKE_THREAD_LIBS_INIT}") + set(TPL_Pthread_LIBRARY_DIRS "") + kokkos_create_imported_tpl_library(Pthread) +else() + kokkos_tpl_find_include_dirs_and_libraries(Pthread REQUIRED_HEADERS pthread.h REQUIRED_LIBS_NAMES pthread) +endif() diff --git a/lib/kokkos/cmake/deps/quadmath.cmake b/lib/kokkos/cmake/deps/quadmath.cmake index 6aef08e881..9006d0cb9e 100644 --- a/lib/kokkos/cmake/deps/quadmath.cmake +++ b/lib/kokkos/cmake/deps/quadmath.cmake @@ -15,7 +15,4 @@ # ************************************************************************ # @HEADER -KOKKOS_TPL_FIND_INCLUDE_DIRS_AND_LIBRARIES( quadmath - REQUIRED_HEADERS quadmath.h - REQUIRED_LIBS_NAMES quadmath -) +kokkos_tpl_find_include_dirs_and_libraries(quadmath REQUIRED_HEADERS quadmath.h REQUIRED_LIBS_NAMES quadmath) diff --git a/lib/kokkos/cmake/fake_tribits.cmake b/lib/kokkos/cmake/fake_tribits.cmake index a18d2ac518..d3fe1e6e2f 100644 --- a/lib/kokkos/cmake/fake_tribits.cmake +++ b/lib/kokkos/cmake/fake_tribits.cmake @@ -1,288 +1,213 @@ #These are tribits wrappers used by all projects in the Kokkos ecosystem -INCLUDE(CMakeParseArguments) -INCLUDE(CTest) +include(CMakeParseArguments) +include(CTest) -FUNCTION(ASSERT_DEFINED VARS) - FOREACH(VAR ${VARS}) - IF(NOT DEFINED ${VAR}) - MESSAGE(SEND_ERROR "Error, the variable ${VAR} is not defined!") - ENDIF() - ENDFOREACH() -ENDFUNCTION() - -IF(NOT KOKKOS_HAS_TRILINOS) -MACRO(APPEND_GLOB VAR) - FILE(GLOB LOCAL_TMP_VAR ${ARGN}) - LIST(APPEND ${VAR} ${LOCAL_TMP_VAR}) -ENDMACRO() - -MACRO(GLOBAL_SET VARNAME) - SET(${VARNAME} ${ARGN} CACHE INTERNAL "" FORCE) -ENDMACRO() - -MACRO(PREPEND_GLOBAL_SET VARNAME) - ASSERT_DEFINED(${VARNAME}) - GLOBAL_SET(${VARNAME} ${ARGN} ${${VARNAME}}) -ENDMACRO() -ENDIF() - -MACRO(ADD_INTERFACE_LIBRARY LIB_NAME) - FILE(WRITE ${CMAKE_CURRENT_BINARY_DIR}/dummy.cpp "") - ADD_LIBRARY(${LIB_NAME} STATIC ${CMAKE_CURRENT_BINARY_DIR}/dummy.cpp) - SET_TARGET_PROPERTIES(${LIB_NAME} PROPERTIES INTERFACE TRUE) -ENDMACRO() - -FUNCTION(KOKKOS_ADD_TEST) - if (KOKKOS_HAS_TRILINOS) - CMAKE_PARSE_ARGUMENTS(TEST - "SKIP_TRIBITS" - "EXE;NAME;TOOL" - "ARGS" - ${ARGN}) - - IF(TEST_SKIP_TRIBITS) - MESSAGE(STATUS "Skipping test ${TEST_NAME} in TriBits") - RETURN() - ENDIF() - - IF(TEST_EXE) - SET(EXE_ROOT ${TEST_EXE}) - ELSE() - SET(EXE_ROOT ${TEST_NAME}) - ENDIF() - - TRIBITS_ADD_TEST( - ${EXE_ROOT} - NAME ${TEST_NAME} - COMM serial mpi - NUM_MPI_PROCS 1 - ARGS ${TEST_ARGS} - ${TEST_UNPARSED_ARGUMENTS} - ADDED_TESTS_NAMES_OUT ALL_TESTS_ADDED - ) - - # We will get prepended package name here - SET(TEST_NAME ${PACKAGE_NAME}_${TEST_NAME}) - SET(EXE ${PACKAGE_NAME}_${EXE_ROOT}) - - # The function TRIBITS_ADD_TEST() has a CATEGORIES argument that defaults - # to BASIC. If a project elects to only enable tests marked as PERFORMANCE, - # the test won't actually be added and attempting to set a property on it below - # will yield an error. - if(TARGET ${EXE}) - if(TEST_TOOL) - add_dependencies(${EXE} ${TEST_TOOL}) #make sure the exe has to build the tool - foreach(TEST_ADDED ${ALL_TESTS_ADDED}) - set_property(TEST ${TEST_ADDED} APPEND PROPERTY ENVIRONMENT "KOKKOS_TOOLS_LIBS=$") - endforeach() - endif() +function(ASSERT_DEFINED VARS) + foreach(VAR ${VARS}) + if(NOT DEFINED ${VAR}) + message(SEND_ERROR "Error, the variable ${VAR} is not defined!") endif() - else() - CMAKE_PARSE_ARGUMENTS(TEST - "WILL_FAIL;SKIP_TRIBITS" - "FAIL_REGULAR_EXPRESSION;PASS_REGULAR_EXPRESSION;EXE;NAME;TOOL" - "CATEGORIES;ARGS" - ${ARGN}) - # To match Tribits, we should always be receiving - # the root names of exes/libs - IF(TEST_EXE) - SET(EXE_ROOT ${TEST_EXE}) - ELSE() - SET(EXE_ROOT ${TEST_NAME}) - ENDIF() - # Prepend package name to the test name - # These should be the full target name - SET(TEST_NAME ${PACKAGE_NAME}_${TEST_NAME}) - SET(EXE ${PACKAGE_NAME}_${EXE_ROOT}) - IF(WIN32) - ADD_TEST(NAME ${TEST_NAME} WORKING_DIRECTORY ${LIBRARY_OUTPUT_PATH} - COMMAND ${EXE}${CMAKE_EXECUTABLE_SUFFIX} ${TEST_ARGS}) - ELSE() - ADD_TEST(NAME ${TEST_NAME} COMMAND ${EXE} ${TEST_ARGS}) - ENDIF() - IF(TEST_WILL_FAIL) - SET_TESTS_PROPERTIES(${TEST_NAME} PROPERTIES WILL_FAIL ${TEST_WILL_FAIL}) - ENDIF() - IF(TEST_FAIL_REGULAR_EXPRESSION) - SET_TESTS_PROPERTIES(${TEST_NAME} PROPERTIES FAIL_REGULAR_EXPRESSION ${TEST_FAIL_REGULAR_EXPRESSION}) - ENDIF() - IF(TEST_PASS_REGULAR_EXPRESSION) - SET_TESTS_PROPERTIES(${TEST_NAME} PROPERTIES PASS_REGULAR_EXPRESSION ${TEST_PASS_REGULAR_EXPRESSION}) - ENDIF() - IF(TEST_TOOL) - ADD_DEPENDENCIES(${EXE} ${TEST_TOOL}) #make sure the exe has to build the tool - SET_PROPERTY(TEST ${TEST_NAME} APPEND_STRING PROPERTY ENVIRONMENT "KOKKOS_PROFILE_LIBRARY=$") - ENDIF() - VERIFY_EMPTY(KOKKOS_ADD_TEST ${TEST_UNPARSED_ARGUMENTS}) - ENDIF() -ENDFUNCTION() + endforeach() +endfunction() -MACRO(KOKKOS_CREATE_IMPORTED_TPL_LIBRARY TPL_NAME) - ADD_INTERFACE_LIBRARY(TPL_LIB_${TPL_NAME}) - TARGET_LINK_LIBRARIES(TPL_LIB_${TPL_NAME} LINK_PUBLIC ${TPL_${TPL_NAME}_LIBRARIES}) - TARGET_INCLUDE_DIRECTORIES(TPL_LIB_${TPL_NAME} INTERFACE ${TPL_${TPL_NAME}_INCLUDE_DIRS}) -ENDMACRO() +macro(APPEND_GLOB VAR) + file(GLOB LOCAL_TMP_VAR ${ARGN}) + list(APPEND ${VAR} ${LOCAL_TMP_VAR}) +endmacro() -FUNCTION(KOKKOS_TPL_FIND_INCLUDE_DIRS_AND_LIBRARIES TPL_NAME) - if (KOKKOS_HAS_TRILINOS) - TRIBITS_TPL_FIND_INCLUDE_DIRS_AND_LIBRARIES(${TPL_NAME} ${ARGN}) - else() - CMAKE_PARSE_ARGUMENTS(PARSE - "" - "" - "REQUIRED_HEADERS;REQUIRED_LIBS_NAMES" - ${ARGN}) +macro(GLOBAL_SET VARNAME) + set(${VARNAME} ${ARGN} CACHE INTERNAL "" FORCE) +endmacro() - SET(_${TPL_NAME}_ENABLE_SUCCESS TRUE) - IF (PARSE_REQUIRED_LIBS_NAMES) - FIND_LIBRARY(TPL_${TPL_NAME}_LIBRARIES NAMES ${PARSE_REQUIRED_LIBS_NAMES}) - IF(NOT TPL_${TPL_NAME}_LIBRARIES) - SET(_${TPL_NAME}_ENABLE_SUCCESS FALSE) - ENDIF() - ENDIF() - IF (PARSE_REQUIRED_HEADERS) - FIND_PATH(TPL_${TPL_NAME}_INCLUDE_DIRS NAMES ${PARSE_REQUIRED_HEADERS}) - IF(NOT TPL_${TPL_NAME}_INCLUDE_DIRS) - SET(_${TPL_NAME}_ENABLE_SUCCESS FALSE) - ENDIF() - ENDIF() - IF (_${TPL_NAME}_ENABLE_SUCCESS) - KOKKOS_CREATE_IMPORTED_TPL_LIBRARY(${TPL_NAME}) - ENDIF() - VERIFY_EMPTY(KOKKOS_CREATE_IMPORTED_TPL_LIBRARY ${PARSE_UNPARSED_ARGUMENTS}) - endif() -ENDFUNCTION() +macro(PREPEND_GLOBAL_SET VARNAME) + assert_defined(${VARNAME}) + global_set(${VARNAME} ${ARGN} ${${VARNAME}}) +endmacro() -MACRO(KOKKOS_TARGET_COMPILE_OPTIONS TARGET) -if(KOKKOS_HAS_TRILINOS) - TARGET_COMPILE_OPTIONS(${TARGET} ${ARGN}) -else() - TARGET_COMPILE_OPTIONS(${TARGET} ${ARGN}) -endif() -ENDMACRO() +macro(ADD_INTERFACE_LIBRARY LIB_NAME) + file(WRITE ${CMAKE_CURRENT_BINARY_DIR}/dummy.cpp "") + add_library(${LIB_NAME} STATIC ${CMAKE_CURRENT_BINARY_DIR}/dummy.cpp) + set_target_properties(${LIB_NAME} PROPERTIES INTERFACE TRUE) +endmacro() -FUNCTION(KOKKOS_LIB_TYPE LIB RET) -GET_TARGET_PROPERTY(PROP ${LIB} TYPE) -IF (${PROP} STREQUAL "INTERFACE_LIBRARY") - SET(${RET} "INTERFACE" PARENT_SCOPE) -ELSE() - SET(${RET} "PUBLIC" PARENT_SCOPE) -ENDIF() -ENDFUNCTION() - -FUNCTION(KOKKOS_TARGET_INCLUDE_DIRECTORIES TARGET) -IF(KOKKOS_HAS_TRILINOS) - KOKKOS_LIB_TYPE(${TARGET} INCTYPE) - #don't trust tribits to do this correctly - but need to add package name - TARGET_INCLUDE_DIRECTORIES(${TARGET} ${INCTYPE} ${ARGN}) -ELSEIF(TARGET ${TARGET}) - #the target actually exists - this means we are doing separate libs - #or this a test library - KOKKOS_LIB_TYPE(${TARGET} INCTYPE) - TARGET_INCLUDE_DIRECTORIES(${TARGET} ${INCTYPE} ${ARGN}) -ELSE() - GET_PROPERTY(LIBS GLOBAL PROPERTY KOKKOS_LIBRARIES_NAMES) - IF (${TARGET} IN_LIST LIBS) - SET_PROPERTY(GLOBAL APPEND PROPERTY KOKKOS_LIBRARY_INCLUDES ${ARGN}) - ELSE() - MESSAGE(FATAL_ERROR "Trying to set include directories on unknown target ${TARGET}") - ENDIF() -ENDIF() -ENDFUNCTION() - -FUNCTION(KOKKOS_LINK_INTERNAL_LIBRARY TARGET DEPLIB) -IF(KOKKOS_HAS_TRILINOS) - #do nothing -ELSE() - SET(options INTERFACE) - SET(oneValueArgs) - SET(multiValueArgs) - CMAKE_PARSE_ARGUMENTS(PARSE - "INTERFACE" - "" - "" - ${ARGN}) - SET(LINK_TYPE) - IF(PARSE_INTERFACE) - SET(LINK_TYPE INTERFACE) - ELSE() - SET(LINK_TYPE PUBLIC) - ENDIF() - TARGET_LINK_LIBRARIES(${TARGET} ${LINK_TYPE} ${DEPLIB}) - VERIFY_EMPTY(KOKKOS_LINK_INTERNAL_LIBRARY ${PARSE_UNPARSED_ARGUMENTS}) -ENDIF() -ENDFUNCTION() - -FUNCTION(KOKKOS_ADD_TEST_LIBRARY NAME) -IF (KOKKOS_HAS_TRILINOS) - TRIBITS_ADD_LIBRARY(${NAME} ${ARGN} TESTONLY) -ELSE() - SET(oneValueArgs) - SET(multiValueArgs HEADERS SOURCES) - - CMAKE_PARSE_ARGUMENTS(PARSE - "STATIC;SHARED" - "" - "HEADERS;SOURCES;DEPLIBS" - ${ARGN}) - - SET(LIB_TYPE) - IF (PARSE_STATIC) - SET(LIB_TYPE STATIC) - ELSEIF (PARSE_SHARED) - SET(LIB_TYPE SHARED) - ENDIF() - - IF(PARSE_HEADERS) - LIST(REMOVE_DUPLICATES PARSE_HEADERS) - ENDIF() - IF(PARSE_SOURCES) - LIST(REMOVE_DUPLICATES PARSE_SOURCES) - ENDIF() - ADD_LIBRARY(${NAME} ${LIB_TYPE} ${PARSE_SOURCES}) - IF (PARSE_DEPLIBS) - TARGET_LINK_LIBRARIES(${NAME} PRIVATE ${PARSE_DEPLIBS}) - ENDIF() -ENDIF() -ENDFUNCTION() - - -FUNCTION(KOKKOS_INCLUDE_DIRECTORIES) -IF(KOKKOS_HAS_TRILINOS) - TRIBITS_INCLUDE_DIRECTORIES(${ARGN}) -ELSE() - CMAKE_PARSE_ARGUMENTS( - INC - "REQUIRED_DURING_INSTALLATION_TESTING" - "" - "" +function(KOKKOS_ADD_TEST) + cmake_parse_arguments( + TEST "WILL_FAIL;SKIP_TRIBITS" "FAIL_REGULAR_EXPRESSION;PASS_REGULAR_EXPRESSION;EXE;NAME;TOOL" "CATEGORIES;ARGS" ${ARGN} ) - INCLUDE_DIRECTORIES(${INC_UNPARSED_ARGUMENTS}) -ENDIF() -ENDFUNCTION() - - -MACRO(PRINTALL match) -get_cmake_property(_variableNames VARIABLES) -list (SORT _variableNames) -foreach (_variableName ${_variableNames}) - if("${_variableName}" MATCHES "${match}") - message(STATUS "${_variableName}=${${_variableName}}") + # To match Tribits, we should always be receiving + # the root names of exes/libs + if(TEST_EXE) + set(EXE_ROOT ${TEST_EXE}) + else() + set(EXE_ROOT ${TEST_NAME}) endif() -endforeach() -ENDMACRO() + # Prepend package name to the test name + # These should be the full target name + set(TEST_NAME ${PACKAGE_NAME}_${TEST_NAME}) -MACRO(SET_GLOBAL_REPLACE SUBSTR VARNAME) - STRING(REPLACE ${SUBSTR} ${${VARNAME}} TEMP) - GLOBAL_SET(${VARNAME} ${TEMP}) -ENDMACRO() + # For compatibility with Trilinos testing, we support: + # * `-D _DISABLE=ON` + # * `-D _EXTRA_ARGS=";;;..."` + # * `-D _SET_RUN_SERIAL=ON` + if(${TEST_NAME}_DISABLE) + return() + endif() -FUNCTION(GLOBAL_APPEND VARNAME) + set(EXE ${PACKAGE_NAME}_${EXE_ROOT}) + if(WIN32) + add_test(NAME ${TEST_NAME} WORKING_DIRECTORY ${LIBRARY_OUTPUT_PATH} COMMAND ${EXE}${CMAKE_EXECUTABLE_SUFFIX} + ${TEST_ARGS} ${${TEST_NAME}_EXTRA_ARGS} + ) + else() + add_test(NAME ${TEST_NAME} COMMAND ${EXE} ${TEST_ARGS} ${${TEST_NAME}_EXTRA_ARGS}) + endif() + # Trilinos testing benefits from labeling the tests as "Kokkos" tests + set_tests_properties(${TEST_NAME} PROPERTIES LABELS Kokkos) + if(${TEST_NAME}_SET_RUN_SERIAL) + set_tests_properties(${TEST_NAME} PROPERTIES RUN_SERIAL ON) + endif() + # TriBITS doesn't actually currently support `-D _ENVIRONMENT` + # but we decided to add it anyway + if(${TEST_NAME}_ENVIRONMENT) + set_tests_properties(${TEST_NAME} PROPERTIES ENVIRONMENT "${${TEST_NAME}_ENVIRONMENT}") + endif() + if(TEST_WILL_FAIL) + set_tests_properties(${TEST_NAME} PROPERTIES WILL_FAIL ${TEST_WILL_FAIL}) + endif() + if(TEST_FAIL_REGULAR_EXPRESSION) + set_tests_properties(${TEST_NAME} PROPERTIES FAIL_REGULAR_EXPRESSION ${TEST_FAIL_REGULAR_EXPRESSION}) + endif() + if(TEST_PASS_REGULAR_EXPRESSION) + set_tests_properties(${TEST_NAME} PROPERTIES PASS_REGULAR_EXPRESSION ${TEST_PASS_REGULAR_EXPRESSION}) + endif() + if(TEST_TOOL) + add_dependencies(${EXE} ${TEST_TOOL}) #make sure the exe has to build the tool + set_property( + TEST ${TEST_NAME} APPEND_STRING PROPERTY ENVIRONMENT "KOKKOS_PROFILE_LIBRARY=$" + ) + endif() + verify_empty(KOKKOS_ADD_TEST ${TEST_UNPARSED_ARGUMENTS}) +endfunction() + +macro(KOKKOS_CREATE_IMPORTED_TPL_LIBRARY TPL_NAME) + add_interface_library(TPL_LIB_${TPL_NAME}) + target_link_libraries(TPL_LIB_${TPL_NAME} LINK_PUBLIC ${TPL_${TPL_NAME}_LIBRARIES}) + target_include_directories(TPL_LIB_${TPL_NAME} INTERFACE ${TPL_${TPL_NAME}_INCLUDE_DIRS}) +endmacro() + +function(KOKKOS_TPL_FIND_INCLUDE_DIRS_AND_LIBRARIES TPL_NAME) + cmake_parse_arguments(PARSE "" "" "REQUIRED_HEADERS;REQUIRED_LIBS_NAMES" ${ARGN}) + + set(_${TPL_NAME}_ENABLE_SUCCESS TRUE) + if(PARSE_REQUIRED_LIBS_NAMES) + find_library(TPL_${TPL_NAME}_LIBRARIES NAMES ${PARSE_REQUIRED_LIBS_NAMES}) + if(NOT TPL_${TPL_NAME}_LIBRARIES) + set(_${TPL_NAME}_ENABLE_SUCCESS FALSE) + endif() + endif() + if(PARSE_REQUIRED_HEADERS) + find_path(TPL_${TPL_NAME}_INCLUDE_DIRS NAMES ${PARSE_REQUIRED_HEADERS}) + if(NOT TPL_${TPL_NAME}_INCLUDE_DIRS) + set(_${TPL_NAME}_ENABLE_SUCCESS FALSE) + endif() + endif() + if(_${TPL_NAME}_ENABLE_SUCCESS) + kokkos_create_imported_tpl_library(${TPL_NAME}) + endif() + verify_empty(KOKKOS_CREATE_IMPORTED_TPL_LIBRARY ${PARSE_UNPARSED_ARGUMENTS}) +endfunction() + +function(KOKKOS_LIB_TYPE LIB RET) + get_target_property(PROP ${LIB} TYPE) + if(${PROP} STREQUAL "INTERFACE_LIBRARY") + set(${RET} "INTERFACE" PARENT_SCOPE) + else() + set(${RET} "PUBLIC" PARENT_SCOPE) + endif() +endfunction() + +function(KOKKOS_TARGET_INCLUDE_DIRECTORIES TARGET) + if(TARGET ${TARGET}) + #the target actually exists - this means we are doing separate libs + #or this a test library + kokkos_lib_type(${TARGET} INCTYPE) + target_include_directories(${TARGET} ${INCTYPE} ${ARGN}) + else() + get_property(LIBS GLOBAL PROPERTY KOKKOS_LIBRARIES_NAMES) + if(${TARGET} IN_LIST LIBS) + set_property(GLOBAL APPEND PROPERTY KOKKOS_LIBRARY_INCLUDES ${ARGN}) + else() + message(FATAL_ERROR "Trying to set include directories on unknown target ${TARGET}") + endif() + endif() +endfunction() + +function(KOKKOS_LINK_INTERNAL_LIBRARY TARGET DEPLIB) + set(options INTERFACE) + set(oneValueArgs) + set(multiValueArgs) + cmake_parse_arguments(PARSE "INTERFACE" "" "" ${ARGN}) + set(LINK_TYPE) + if(PARSE_INTERFACE) + set(LINK_TYPE INTERFACE) + else() + set(LINK_TYPE PUBLIC) + endif() + target_link_libraries(${TARGET} ${LINK_TYPE} ${DEPLIB}) + verify_empty(KOKKOS_LINK_INTERNAL_LIBRARY ${PARSE_UNPARSED_ARGUMENTS}) +endfunction() + +function(KOKKOS_ADD_TEST_LIBRARY NAME) + set(oneValueArgs) + set(multiValueArgs HEADERS SOURCES) + + cmake_parse_arguments(PARSE "STATIC;SHARED" "" "HEADERS;SOURCES;DEPLIBS" ${ARGN}) + + set(LIB_TYPE) + if(PARSE_STATIC) + set(LIB_TYPE STATIC) + elseif(PARSE_SHARED) + set(LIB_TYPE SHARED) + endif() + + if(PARSE_HEADERS) + list(REMOVE_DUPLICATES PARSE_HEADERS) + endif() + if(PARSE_SOURCES) + list(REMOVE_DUPLICATES PARSE_SOURCES) + endif() + add_library(${NAME} ${LIB_TYPE} ${PARSE_SOURCES}) + if(PARSE_DEPLIBS) + target_link_libraries(${NAME} PRIVATE ${PARSE_DEPLIBS}) + endif() +endfunction() + +function(KOKKOS_INCLUDE_DIRECTORIES) + cmake_parse_arguments(INC "REQUIRED_DURING_INSTALLATION_TESTING" "" "" ${ARGN}) + include_directories(${INC_UNPARSED_ARGUMENTS}) +endfunction() + +macro(PRINTALL match) + get_cmake_property(_variableNames VARIABLES) + list(SORT _variableNames) + foreach(_variableName ${_variableNames}) + if("${_variableName}" MATCHES "${match}") + message(STATUS "${_variableName}=${${_variableName}}") + endif() + endforeach() +endmacro() + +macro(SET_GLOBAL_REPLACE SUBSTR VARNAME) + string(REPLACE ${SUBSTR} ${${VARNAME}} TEMP) + global_set(${VARNAME} ${TEMP}) +endmacro() + +function(GLOBAL_APPEND VARNAME) #We make this a function since we are setting variables #and want to use scope to avoid overwriting local variables - SET(TEMP ${${VARNAME}}) - LIST(APPEND TEMP ${ARGN}) - GLOBAL_SET(${VARNAME} ${TEMP}) -ENDFUNCTION() + set(TEMP ${${VARNAME}}) + list(APPEND TEMP ${ARGN}) + global_set(${VARNAME} ${TEMP}) +endfunction() diff --git a/lib/kokkos/cmake/gnu.cmake b/lib/kokkos/cmake/gnu.cmake index aa11fe87b1..e53b4a7bec 100644 --- a/lib/kokkos/cmake/gnu.cmake +++ b/lib/kokkos/cmake/gnu.cmake @@ -1,23 +1,21 @@ - -FUNCTION(kokkos_set_gnu_flags full_standard int_standard) - STRING(TOLOWER ${full_standard} FULL_LC_STANDARD) - STRING(TOLOWER ${int_standard} INT_LC_STANDARD) +function(kokkos_set_gnu_flags full_standard int_standard) + string(TOLOWER ${full_standard} FULL_LC_STANDARD) + string(TOLOWER ${int_standard} INT_LC_STANDARD) # The following three blocks of code were copied from # /Modules/Compiler/Intel-CXX.cmake from CMake 3.7.2 and then modified. - IF(CMAKE_CXX_SIMULATE_ID STREQUAL MSVC) - SET(_std -Qstd) - SET(_ext c++) - ELSE() - SET(_std -std) - SET(_ext gnu++) - ENDIF() - - IF (CMAKE_CXX_EXTENSIONS) - SET(KOKKOS_CXX_STANDARD_FLAG "-std=gnu++${FULL_LC_STANDARD}" PARENT_SCOPE) - SET(KOKKOS_CXX_INTERMEDIATE_STANDARD_FLAG "-std=gnu++${INT_LC_STANDARD}" PARENT_SCOPE) - ELSE() - SET(KOKKOS_CXX_STANDARD_FLAG "-std=c++${FULL_LC_STANDARD}" PARENT_SCOPE) - SET(KOKKOS_CXX_INTERMEDIATE_STANDARD_FLAG "-std=c++${INT_LC_STANDARD}" PARENT_SCOPE) - ENDIF() -ENDFUNCTION() + if(CMAKE_CXX_SIMULATE_ID STREQUAL MSVC) + set(_std -Qstd) + set(_ext c++) + else() + set(_std -std) + set(_ext gnu++) + endif() + if(CMAKE_CXX_EXTENSIONS) + set(KOKKOS_CXX_STANDARD_FLAG "-std=gnu++${FULL_LC_STANDARD}" PARENT_SCOPE) + set(KOKKOS_CXX_INTERMEDIATE_STANDARD_FLAG "-std=gnu++${INT_LC_STANDARD}" PARENT_SCOPE) + else() + set(KOKKOS_CXX_STANDARD_FLAG "-std=c++${FULL_LC_STANDARD}" PARENT_SCOPE) + set(KOKKOS_CXX_INTERMEDIATE_STANDARD_FLAG "-std=c++${INT_LC_STANDARD}" PARENT_SCOPE) + endif() +endfunction() diff --git a/lib/kokkos/cmake/intel.cmake b/lib/kokkos/cmake/intel.cmake index 7e6ee3358c..b7752caabd 100644 --- a/lib/kokkos/cmake/intel.cmake +++ b/lib/kokkos/cmake/intel.cmake @@ -1,18 +1,15 @@ - -FUNCTION(kokkos_set_intel_flags full_standard int_standard) - STRING(TOLOWER ${full_standard} FULL_LC_STANDARD) - STRING(TOLOWER ${int_standard} INT_LC_STANDARD) +function(kokkos_set_intel_flags full_standard int_standard) + string(TOLOWER ${full_standard} FULL_LC_STANDARD) + string(TOLOWER ${int_standard} INT_LC_STANDARD) # The following three blocks of code were copied from # /Modules/Compiler/Intel-CXX.cmake from CMake 3.18.1 and then modified. - IF(CMAKE_CXX_SIMULATE_ID STREQUAL MSVC) - SET(_std -Qstd) - SET(_ext c++) - ELSE() - SET(_std -std) - SET(_ext gnu++) - ENDIF() - SET(KOKKOS_CXX_STANDARD_FLAG "${_std}=c++${FULL_LC_STANDARD}" PARENT_SCOPE) - SET(KOKKOS_CXX_INTERMDIATE_STANDARD_FLAG "${_std}=${_ext}${INT_LC_STANDARD}" PARENT_SCOPE) -ENDFUNCTION() - - + if(CMAKE_CXX_SIMULATE_ID STREQUAL MSVC) + set(_std -Qstd) + set(_ext c++) + else() + set(_std -std) + set(_ext gnu++) + endif() + set(KOKKOS_CXX_STANDARD_FLAG "${_std}=c++${FULL_LC_STANDARD}" PARENT_SCOPE) + set(KOKKOS_CXX_INTERMDIATE_STANDARD_FLAG "${_std}=${_ext}${INT_LC_STANDARD}" PARENT_SCOPE) +endfunction() diff --git a/lib/kokkos/cmake/kokkos_arch.cmake b/lib/kokkos/cmake/kokkos_arch.cmake index 0b3d4044d0..ae45da806f 100644 --- a/lib/kokkos/cmake/kokkos_arch.cmake +++ b/lib/kokkos/cmake/kokkos_arch.cmake @@ -1,611 +1,732 @@ - -FUNCTION(KOKKOS_ARCH_OPTION SUFFIX DEV_TYPE DESCRIPTION DEPENDENCY) +function(KOKKOS_ARCH_OPTION SUFFIX DEV_TYPE DESCRIPTION DEPENDENCY) #all optimizations off by default - KOKKOS_DEPENDENT_OPTION(ARCH_${SUFFIX} "Optimize for ${DESCRIPTION} (${DEV_TYPE})" OFF "${DEPENDENCY}" OFF) - SET(KOKKOS_ARCH_${SUFFIX} ${KOKKOS_ARCH_${SUFFIX}} PARENT_SCOPE) - SET(KOKKOS_OPTION_KEYS ${KOKKOS_OPTION_KEYS} PARENT_SCOPE) - SET(KOKKOS_OPTION_VALUES ${KOKKOS_OPTION_VALUES} PARENT_SCOPE) - SET(KOKKOS_OPTION_TYPES ${KOKKOS_OPTION_TYPES} PARENT_SCOPE) - IF(KOKKOS_ARCH_${SUFFIX}) - LIST(APPEND KOKKOS_ENABLED_ARCH_LIST ${SUFFIX}) - SET(KOKKOS_ENABLED_ARCH_LIST ${KOKKOS_ENABLED_ARCH_LIST} PARENT_SCOPE) - ENDIF() -ENDFUNCTION() - + kokkos_dependent_option(ARCH_${SUFFIX} "Optimize for ${DESCRIPTION} (${DEV_TYPE})" OFF "${DEPENDENCY}" OFF) + set(KOKKOS_ARCH_${SUFFIX} ${KOKKOS_ARCH_${SUFFIX}} PARENT_SCOPE) + set(KOKKOS_OPTION_KEYS ${KOKKOS_OPTION_KEYS} PARENT_SCOPE) + set(KOKKOS_OPTION_VALUES ${KOKKOS_OPTION_VALUES} PARENT_SCOPE) + set(KOKKOS_OPTION_TYPES ${KOKKOS_OPTION_TYPES} PARENT_SCOPE) + if(KOKKOS_ARCH_${SUFFIX}) + list(APPEND KOKKOS_ENABLED_ARCH_LIST ${SUFFIX}) + set(KOKKOS_ENABLED_ARCH_LIST ${KOKKOS_ENABLED_ARCH_LIST} PARENT_SCOPE) + endif() +endfunction() # Make sure devices and compiler ID are done -KOKKOS_CFG_DEPENDS(ARCH COMPILER_ID) -KOKKOS_CFG_DEPENDS(ARCH DEVICES) -KOKKOS_CFG_DEPENDS(ARCH OPTIONS) +kokkos_cfg_depends(ARCH COMPILER_ID) +kokkos_cfg_depends(ARCH DEVICES) +kokkos_cfg_depends(ARCH OPTIONS) -KOKKOS_CHECK_DEPRECATED_OPTIONS( - ARCH_EPYC "Please replace EPYC with ZEN or ZEN2, depending on your platform" - ARCH_RYZEN "Please replace RYZEN with ZEN or ZEN2, depending on your platform" +kokkos_check_deprecated_options( + ARCH_EPYC "Please replace EPYC with ZEN or ZEN2, depending on your platform" ARCH_RYZEN + "Please replace RYZEN with ZEN or ZEN2, depending on your platform" ) #------------------------------------------------------------------------------- # List of possible host architectures. #------------------------------------------------------------------------------- -SET(KOKKOS_ARCH_LIST) +set(KOKKOS_ARCH_LIST) include(CheckCXXCompilerFlag) -KOKKOS_DEPRECATED_LIST(ARCH ARCH) +kokkos_deprecated_list(ARCH ARCH) -SET(HOST_ARCH_ALREADY_SPECIFIED "") -MACRO(DECLARE_AND_CHECK_HOST_ARCH ARCH LABEL) - KOKKOS_ARCH_OPTION(${ARCH} HOST "${LABEL}" TRUE) - IF(KOKKOS_ARCH_${ARCH}) - IF(HOST_ARCH_ALREADY_SPECIFIED) - MESSAGE(FATAL_ERROR "Multiple host architectures given! Already have ${HOST_ARCH_ALREADY_SPECIFIED}, but trying to add ${ARCH}. If you are re-running CMake, try clearing the cache and running again.") - ENDIF() - SET(HOST_ARCH_ALREADY_SPECIFIED ${ARCH}) - ENDIF() -ENDMACRO() +set(HOST_ARCH_ALREADY_SPECIFIED "") +macro(DECLARE_AND_CHECK_HOST_ARCH ARCH LABEL) + kokkos_arch_option(${ARCH} HOST "${LABEL}" TRUE) + if(KOKKOS_ARCH_${ARCH}) + if(HOST_ARCH_ALREADY_SPECIFIED) + message( + FATAL_ERROR + "Multiple host architectures given! Already have ${HOST_ARCH_ALREADY_SPECIFIED}, but trying to add ${ARCH}. If you are re-running CMake, try clearing the cache and running again." + ) + endif() + set(HOST_ARCH_ALREADY_SPECIFIED ${ARCH}) + endif() +endmacro() -DECLARE_AND_CHECK_HOST_ARCH(NATIVE "local machine") -DECLARE_AND_CHECK_HOST_ARCH(AMDAVX "AMD chip") -DECLARE_AND_CHECK_HOST_ARCH(ARMV80 "ARMv8.0 Compatible CPU") -DECLARE_AND_CHECK_HOST_ARCH(ARMV81 "ARMv8.1 Compatible CPU") -DECLARE_AND_CHECK_HOST_ARCH(ARMV8_THUNDERX "ARMv8 Cavium ThunderX CPU") -DECLARE_AND_CHECK_HOST_ARCH(ARMV8_THUNDERX2 "ARMv8 Cavium ThunderX2 CPU") -DECLARE_AND_CHECK_HOST_ARCH(A64FX "ARMv8.2 with SVE Support") -DECLARE_AND_CHECK_HOST_ARCH(ARMV9_GRACE "ARMv9 NVIDIA Grace CPU") -DECLARE_AND_CHECK_HOST_ARCH(SNB "Intel Sandy/Ivy Bridge CPUs") -DECLARE_AND_CHECK_HOST_ARCH(HSW "Intel Haswell CPUs") -DECLARE_AND_CHECK_HOST_ARCH(BDW "Intel Broadwell Xeon E-class CPUs") -DECLARE_AND_CHECK_HOST_ARCH(ICL "Intel Ice Lake Client CPUs (AVX512)") -DECLARE_AND_CHECK_HOST_ARCH(ICX "Intel Ice Lake Xeon Server CPUs (AVX512)") -DECLARE_AND_CHECK_HOST_ARCH(SKL "Intel Skylake Client CPUs") -DECLARE_AND_CHECK_HOST_ARCH(SKX "Intel Skylake Xeon Server CPUs (AVX512)") -DECLARE_AND_CHECK_HOST_ARCH(KNC "Intel Knights Corner Xeon Phi") -DECLARE_AND_CHECK_HOST_ARCH(KNL "Intel Knights Landing Xeon Phi") -DECLARE_AND_CHECK_HOST_ARCH(SPR "Intel Sapphire Rapids Xeon Server CPUs (AVX512)") -DECLARE_AND_CHECK_HOST_ARCH(POWER8 "IBM POWER8 CPUs") -DECLARE_AND_CHECK_HOST_ARCH(POWER9 "IBM POWER9 CPUs") -DECLARE_AND_CHECK_HOST_ARCH(ZEN "AMD Zen architecture") -DECLARE_AND_CHECK_HOST_ARCH(ZEN2 "AMD Zen2 architecture") -DECLARE_AND_CHECK_HOST_ARCH(ZEN3 "AMD Zen3 architecture") -DECLARE_AND_CHECK_HOST_ARCH(RISCV_SG2042 "SG2042 (RISC-V) CPUs") +declare_and_check_host_arch(NATIVE "local machine") +declare_and_check_host_arch(AMDAVX "AMD chip") +declare_and_check_host_arch(ARMV80 "ARMv8.0 Compatible CPU") +declare_and_check_host_arch(ARMV81 "ARMv8.1 Compatible CPU") +declare_and_check_host_arch(ARMV8_THUNDERX "ARMv8 Cavium ThunderX CPU") +declare_and_check_host_arch(ARMV8_THUNDERX2 "ARMv8 Cavium ThunderX2 CPU") +declare_and_check_host_arch(A64FX "ARMv8.2 with SVE Support") +declare_and_check_host_arch(ARMV9_GRACE "ARMv9 NVIDIA Grace CPU") +declare_and_check_host_arch(SNB "Intel Sandy/Ivy Bridge CPUs") +declare_and_check_host_arch(HSW "Intel Haswell CPUs") +declare_and_check_host_arch(BDW "Intel Broadwell Xeon E-class CPUs") +declare_and_check_host_arch(ICL "Intel Ice Lake Client CPUs (AVX512)") +declare_and_check_host_arch(ICX "Intel Ice Lake Xeon Server CPUs (AVX512)") +declare_and_check_host_arch(SKL "Intel Skylake Client CPUs") +declare_and_check_host_arch(SKX "Intel Skylake Xeon Server CPUs (AVX512)") +declare_and_check_host_arch(KNC "Intel Knights Corner Xeon Phi") +declare_and_check_host_arch(KNL "Intel Knights Landing Xeon Phi") +declare_and_check_host_arch(SPR "Intel Sapphire Rapids Xeon Server CPUs (AVX512)") +declare_and_check_host_arch(POWER8 "IBM POWER8 CPUs") +declare_and_check_host_arch(POWER9 "IBM POWER9 CPUs") +declare_and_check_host_arch(ZEN "AMD Zen architecture") +declare_and_check_host_arch(ZEN2 "AMD Zen2 architecture") +declare_and_check_host_arch(ZEN3 "AMD Zen3 architecture") +declare_and_check_host_arch(RISCV_SG2042 "SG2042 (RISC-V) CPUs") +declare_and_check_host_arch(RISCV_RVA22V "RVA22V (RISC-V) CPUs") -IF(Kokkos_ENABLE_CUDA OR Kokkos_ENABLE_OPENMPTARGET OR Kokkos_ENABLE_OPENACC OR Kokkos_ENABLE_SYCL) - SET(KOKKOS_SHOW_CUDA_ARCHS ON) -ENDIF() +if(Kokkos_ENABLE_CUDA + OR Kokkos_ENABLE_OPENMPTARGET + OR Kokkos_ENABLE_OPENACC + OR Kokkos_ENABLE_SYCL +) + set(KOKKOS_SHOW_CUDA_ARCHS ON) +endif() -KOKKOS_ARCH_OPTION(KEPLER30 GPU "NVIDIA Kepler generation CC 3.0" "KOKKOS_SHOW_CUDA_ARCHS") -KOKKOS_ARCH_OPTION(KEPLER32 GPU "NVIDIA Kepler generation CC 3.2" "KOKKOS_SHOW_CUDA_ARCHS") -KOKKOS_ARCH_OPTION(KEPLER35 GPU "NVIDIA Kepler generation CC 3.5" "KOKKOS_SHOW_CUDA_ARCHS") -KOKKOS_ARCH_OPTION(KEPLER37 GPU "NVIDIA Kepler generation CC 3.7" "KOKKOS_SHOW_CUDA_ARCHS") -KOKKOS_ARCH_OPTION(MAXWELL50 GPU "NVIDIA Maxwell generation CC 5.0" "KOKKOS_SHOW_CUDA_ARCHS") -KOKKOS_ARCH_OPTION(MAXWELL52 GPU "NVIDIA Maxwell generation CC 5.2" "KOKKOS_SHOW_CUDA_ARCHS") -KOKKOS_ARCH_OPTION(MAXWELL53 GPU "NVIDIA Maxwell generation CC 5.3" "KOKKOS_SHOW_CUDA_ARCHS") -KOKKOS_ARCH_OPTION(PASCAL60 GPU "NVIDIA Pascal generation CC 6.0" "KOKKOS_SHOW_CUDA_ARCHS") -KOKKOS_ARCH_OPTION(PASCAL61 GPU "NVIDIA Pascal generation CC 6.1" "KOKKOS_SHOW_CUDA_ARCHS") -KOKKOS_ARCH_OPTION(VOLTA70 GPU "NVIDIA Volta generation CC 7.0" "KOKKOS_SHOW_CUDA_ARCHS") -KOKKOS_ARCH_OPTION(VOLTA72 GPU "NVIDIA Volta generation CC 7.2" "KOKKOS_SHOW_CUDA_ARCHS") -KOKKOS_ARCH_OPTION(TURING75 GPU "NVIDIA Turing generation CC 7.5" "KOKKOS_SHOW_CUDA_ARCHS") -KOKKOS_ARCH_OPTION(AMPERE80 GPU "NVIDIA Ampere generation CC 8.0" "KOKKOS_SHOW_CUDA_ARCHS") -KOKKOS_ARCH_OPTION(AMPERE86 GPU "NVIDIA Ampere generation CC 8.6" "KOKKOS_SHOW_CUDA_ARCHS") -KOKKOS_ARCH_OPTION(ADA89 GPU "NVIDIA Ada generation CC 8.9" "KOKKOS_SHOW_CUDA_ARCHS") -KOKKOS_ARCH_OPTION(HOPPER90 GPU "NVIDIA Hopper generation CC 9.0" "KOKKOS_SHOW_CUDA_ARCHS") +kokkos_arch_option(KEPLER30 GPU "NVIDIA Kepler generation CC 3.0" "KOKKOS_SHOW_CUDA_ARCHS") +kokkos_arch_option(KEPLER32 GPU "NVIDIA Kepler generation CC 3.2" "KOKKOS_SHOW_CUDA_ARCHS") +kokkos_arch_option(KEPLER35 GPU "NVIDIA Kepler generation CC 3.5" "KOKKOS_SHOW_CUDA_ARCHS") +kokkos_arch_option(KEPLER37 GPU "NVIDIA Kepler generation CC 3.7" "KOKKOS_SHOW_CUDA_ARCHS") +kokkos_arch_option(MAXWELL50 GPU "NVIDIA Maxwell generation CC 5.0" "KOKKOS_SHOW_CUDA_ARCHS") +kokkos_arch_option(MAXWELL52 GPU "NVIDIA Maxwell generation CC 5.2" "KOKKOS_SHOW_CUDA_ARCHS") +kokkos_arch_option(MAXWELL53 GPU "NVIDIA Maxwell generation CC 5.3" "KOKKOS_SHOW_CUDA_ARCHS") +kokkos_arch_option(PASCAL60 GPU "NVIDIA Pascal generation CC 6.0" "KOKKOS_SHOW_CUDA_ARCHS") +kokkos_arch_option(PASCAL61 GPU "NVIDIA Pascal generation CC 6.1" "KOKKOS_SHOW_CUDA_ARCHS") +kokkos_arch_option(VOLTA70 GPU "NVIDIA Volta generation CC 7.0" "KOKKOS_SHOW_CUDA_ARCHS") +kokkos_arch_option(VOLTA72 GPU "NVIDIA Volta generation CC 7.2" "KOKKOS_SHOW_CUDA_ARCHS") +kokkos_arch_option(TURING75 GPU "NVIDIA Turing generation CC 7.5" "KOKKOS_SHOW_CUDA_ARCHS") +kokkos_arch_option(AMPERE80 GPU "NVIDIA Ampere generation CC 8.0" "KOKKOS_SHOW_CUDA_ARCHS") +kokkos_arch_option(AMPERE86 GPU "NVIDIA Ampere generation CC 8.6" "KOKKOS_SHOW_CUDA_ARCHS") +kokkos_arch_option(ADA89 GPU "NVIDIA Ada generation CC 8.9" "KOKKOS_SHOW_CUDA_ARCHS") +kokkos_arch_option(HOPPER90 GPU "NVIDIA Hopper generation CC 9.0" "KOKKOS_SHOW_CUDA_ARCHS") -IF(Kokkos_ENABLE_HIP OR Kokkos_ENABLE_OPENMPTARGET OR Kokkos_ENABLE_OPENACC OR Kokkos_ENABLE_SYCL) - SET(KOKKOS_SHOW_HIP_ARCHS ON) -ENDIF() +if(Kokkos_ENABLE_HIP + OR Kokkos_ENABLE_OPENMPTARGET + OR Kokkos_ENABLE_OPENACC + OR Kokkos_ENABLE_SYCL +) + set(KOKKOS_SHOW_HIP_ARCHS ON) +endif() # AMD archs ordered in decreasing priority of autodetection -LIST(APPEND SUPPORTED_AMD_GPUS MI300 MI300) -LIST(APPEND SUPPORTED_AMD_ARCHS AMD_GFX942 AMD_GFX940) -LIST(APPEND CORRESPONDING_AMD_FLAGS gfx942 gfx940) -LIST(APPEND SUPPORTED_AMD_GPUS MI200 MI200 MI100 MI100) -LIST(APPEND SUPPORTED_AMD_ARCHS VEGA90A AMD_GFX90A VEGA908 AMD_GFX908) -LIST(APPEND CORRESPONDING_AMD_FLAGS gfx90a gfx90a gfx908 gfx908) -LIST(APPEND SUPPORTED_AMD_GPUS MI50/60 MI50/60) -LIST(APPEND SUPPORTED_AMD_ARCHS VEGA906 AMD_GFX906) -LIST(APPEND CORRESPONDING_AMD_FLAGS gfx906 gfx906) -LIST(APPEND SUPPORTED_AMD_GPUS PHOENIX RX7900XTX V620/W6800 V620/W6800) -LIST(APPEND SUPPORTED_AMD_ARCHS AMD_GFX1103 AMD_GFX1100 NAVI1030 AMD_GFX1030) -LIST(APPEND CORRESPONDING_AMD_FLAGS gfx1103 gfx1100 gfx1030 gfx1030) +list(APPEND SUPPORTED_AMD_GPUS MI300 MI300A MI300) +list(APPEND SUPPORTED_AMD_ARCHS AMD_GFX942 AMD_GFX942_APU AMD_GFX940) +list(APPEND CORRESPONDING_AMD_FLAGS gfx942 gfx942 gfx940) +list(APPEND SUPPORTED_AMD_GPUS MI200 MI200 MI100 MI100) +list(APPEND SUPPORTED_AMD_ARCHS VEGA90A AMD_GFX90A VEGA908 AMD_GFX908) +list(APPEND CORRESPONDING_AMD_FLAGS gfx90a gfx90a gfx908 gfx908) +list(APPEND SUPPORTED_AMD_GPUS MI50/60 MI50/60) +list(APPEND SUPPORTED_AMD_ARCHS VEGA906 AMD_GFX906) +list(APPEND CORRESPONDING_AMD_FLAGS gfx906 gfx906) +list(APPEND SUPPORTED_AMD_GPUS PHOENIX RX7900XTX V620/W6800 V620/W6800) +list(APPEND SUPPORTED_AMD_ARCHS AMD_GFX1103 AMD_GFX1100 NAVI1030 AMD_GFX1030) +list(APPEND CORRESPONDING_AMD_FLAGS gfx1103 gfx1100 gfx1030 gfx1030) #FIXME CAN BE REPLACED WITH LIST_ZIP IN CMAKE 3.17 -FOREACH(ARCH IN LISTS SUPPORTED_AMD_ARCHS) - LIST(FIND SUPPORTED_AMD_ARCHS ${ARCH} LIST_INDEX) - LIST(GET SUPPORTED_AMD_GPUS ${LIST_INDEX} GPU) - LIST(GET CORRESPONDING_AMD_FLAGS ${LIST_INDEX} FLAG) - KOKKOS_ARCH_OPTION(${ARCH} GPU "AMD GPU ${GPU} ${FLAG}" "KOKKOS_SHOW_HIP_ARCHS") -ENDFOREACH() +foreach(ARCH IN LISTS SUPPORTED_AMD_ARCHS) + list(FIND SUPPORTED_AMD_ARCHS ${ARCH} LIST_INDEX) + list(GET SUPPORTED_AMD_GPUS ${LIST_INDEX} GPU) + list(GET CORRESPONDING_AMD_FLAGS ${LIST_INDEX} FLAG) + kokkos_arch_option(${ARCH} GPU "AMD GPU ${GPU} ${FLAG}" "KOKKOS_SHOW_HIP_ARCHS") +endforeach() -IF(Kokkos_ENABLE_SYCL OR Kokkos_ENABLE_OPENMPTARGET) - SET(KOKKOS_SHOW_SYCL_ARCHS ON) -ENDIF() +if(Kokkos_ENABLE_SYCL OR Kokkos_ENABLE_OPENMPTARGET) + set(KOKKOS_SHOW_SYCL_ARCHS ON) +endif() -KOKKOS_ARCH_OPTION(INTEL_GEN GPU "SPIR64-based devices, e.g. Intel GPUs, using JIT" "KOKKOS_SHOW_SYCL_ARCHS") -KOKKOS_ARCH_OPTION(INTEL_DG1 GPU "Intel Iris XeMAX GPU" "KOKKOS_SHOW_SYCL_ARCHS") -KOKKOS_ARCH_OPTION(INTEL_GEN9 GPU "Intel GPU Gen9" "KOKKOS_SHOW_SYCL_ARCHS") -KOKKOS_ARCH_OPTION(INTEL_GEN11 GPU "Intel GPU Gen11" "KOKKOS_SHOW_SYCL_ARCHS") -KOKKOS_ARCH_OPTION(INTEL_GEN12LP GPU "Intel GPU Gen12LP" "KOKKOS_SHOW_SYCL_ARCHS") -KOKKOS_ARCH_OPTION(INTEL_XEHP GPU "Intel GPU Xe-HP" "KOKKOS_SHOW_SYCL_ARCHS") -KOKKOS_ARCH_OPTION(INTEL_PVC GPU "Intel GPU Ponte Vecchio" "KOKKOS_SHOW_SYCL_ARCHS") +kokkos_arch_option(INTEL_GEN GPU "SPIR64-based devices, e.g. Intel GPUs, using JIT" "KOKKOS_SHOW_SYCL_ARCHS") +kokkos_arch_option(INTEL_DG1 GPU "Intel Iris XeMAX GPU" "KOKKOS_SHOW_SYCL_ARCHS") +kokkos_arch_option(INTEL_GEN9 GPU "Intel GPU Gen9" "KOKKOS_SHOW_SYCL_ARCHS") +kokkos_arch_option(INTEL_GEN11 GPU "Intel GPU Gen11" "KOKKOS_SHOW_SYCL_ARCHS") +kokkos_arch_option(INTEL_GEN12LP GPU "Intel GPU Gen12LP" "KOKKOS_SHOW_SYCL_ARCHS") +kokkos_arch_option(INTEL_XEHP GPU "Intel GPU Xe-HP" "KOKKOS_SHOW_SYCL_ARCHS") +kokkos_arch_option(INTEL_PVC GPU "Intel GPU Ponte Vecchio" "KOKKOS_SHOW_SYCL_ARCHS") -IF(KOKKOS_ENABLE_COMPILER_WARNINGS) - SET(COMMON_WARNINGS - "-Wall" "-Wextra" "-Wunused-parameter" "-Wshadow" "-pedantic" - "-Wsign-compare" "-Wtype-limits" "-Wuninitialized") +if(KOKKOS_ENABLE_COMPILER_WARNINGS) + set(COMMON_WARNINGS + "-Wall" + "-Wextra" + "-Wunused-parameter" + "-Wshadow" + "-pedantic" + "-Wsign-compare" + "-Wtype-limits" + "-Wuninitialized" + "-Wsuggest-override" + ) # NOTE KOKKOS_ prefixed variable (all uppercase) is not set yet because TPLs are processed after ARCH - IF(Kokkos_ENABLE_LIBQUADMATH) + if(Kokkos_ENABLE_LIBQUADMATH) # warning: non-standard suffix on floating constant [-Wpedantic] - LIST(REMOVE_ITEM COMMON_WARNINGS "-pedantic") - ENDIF() + list(REMOVE_ITEM COMMON_WARNINGS "-pedantic") + endif() # NVHPC compiler does not support -Wtype-limits. - IF(KOKKOS_ENABLE_OPENACC) - IF(KOKKOS_CXX_COMPILER_ID STREQUAL NVHPC) - LIST(REMOVE_ITEM COMMON_WARNINGS "-Wtype-limits") - ENDIF() - ENDIF() + if(KOKKOS_ENABLE_OPENACC) + if(KOKKOS_CXX_COMPILER_ID STREQUAL NVHPC) + list(REMOVE_ITEM COMMON_WARNINGS "-Wtype-limits") + endif() + endif() - IF(KOKKOS_CXX_COMPILER_ID STREQUAL Clang) - LIST(APPEND COMMON_WARNINGS "-Wimplicit-fallthrough") - ENDIF() + # ICPC doesn't support -Wsuggest-override + if(KOKKOS_CXX_COMPILER_ID STREQUAL Intel) + list(REMOVE_ITEM COMMON_WARNINGS "-Wsuggest-override") + endif() - SET(GNU_WARNINGS "-Wempty-body" "-Wclobbered" "-Wignored-qualifiers" - ${COMMON_WARNINGS}) - IF(KOKKOS_CXX_COMPILER_ID STREQUAL GNU) - LIST(APPEND GNU_WARNINGS "-Wimplicit-fallthrough") - ENDIF() + if(KOKKOS_CXX_COMPILER_ID STREQUAL Clang) + list(APPEND COMMON_WARNINGS "-Wimplicit-fallthrough") + endif() + + set(GNU_WARNINGS "-Wempty-body" "-Wclobbered" "-Wignored-qualifiers" ${COMMON_WARNINGS}) + if(KOKKOS_CXX_COMPILER_ID STREQUAL GNU) + list(APPEND GNU_WARNINGS "-Wimplicit-fallthrough") + endif() # Not using COMPILER_SPECIFIC_FLAGS function so the warning flags are not passed downstream - IF(CMAKE_CXX_COMPILER_ID STREQUAL GNU) - STRING(REPLACE ";" " " WARNING_FLAGS "${GNU_WARNINGS}") - ELSEIF(CMAKE_CXX_COMPILER_ID STREQUAL NVHPC) + if(CMAKE_CXX_COMPILER_ID STREQUAL GNU) + string(REPLACE ";" " " WARNING_FLAGS "${GNU_WARNINGS}") + elseif(CMAKE_CXX_COMPILER_ID STREQUAL NVHPC) # FIXME_NVHPC - ELSE() - STRING(REPLACE ";" " " WARNING_FLAGS "${COMMON_WARNINGS}") - ENDIF() - SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${WARNING_FLAGS}") -ENDIF() - + else() + string(REPLACE ";" " " WARNING_FLAGS "${COMMON_WARNINGS}") + endif() + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${WARNING_FLAGS}") +endif() #------------------------------- KOKKOS_CUDA_OPTIONS --------------------------- #clear anything that might be in the cache -GLOBAL_SET(KOKKOS_CUDA_OPTIONS) +global_set(KOKKOS_CUDA_OPTIONS) # Construct the Makefile options -IF(KOKKOS_CXX_COMPILER_ID STREQUAL NVIDIA) - GLOBAL_APPEND(KOKKOS_CUDA_OPTIONS "-extended-lambda") - GLOBAL_APPEND(KOKKOS_CUDA_OPTIONS "-Wext-lambda-captures-this") -ENDIF() +if(KOKKOS_CXX_COMPILER_ID STREQUAL NVIDIA) + global_append(KOKKOS_CUDA_OPTIONS "-extended-lambda") + global_append(KOKKOS_CUDA_OPTIONS "-Wext-lambda-captures-this") +endif() -IF (KOKKOS_ENABLE_CUDA_CONSTEXPR) - IF(KOKKOS_CXX_COMPILER_ID STREQUAL NVIDIA) - GLOBAL_APPEND(KOKKOS_CUDA_OPTIONS "-expt-relaxed-constexpr") - ENDIF() -ENDIF() +if(KOKKOS_ENABLE_CUDA_CONSTEXPR) + if(KOKKOS_CXX_COMPILER_ID STREQUAL NVIDIA) + global_append(KOKKOS_CUDA_OPTIONS "-expt-relaxed-constexpr") + endif() +endif() -IF (KOKKOS_CXX_COMPILER_ID STREQUAL Clang) - SET(CUDA_ARCH_FLAG "--cuda-gpu-arch") - GLOBAL_APPEND(KOKKOS_CUDA_OPTIONS -x cuda) +if(KOKKOS_CXX_COMPILER_ID STREQUAL Clang) + set(CUDA_ARCH_FLAG "--cuda-gpu-arch") + global_append(KOKKOS_CUDA_OPTIONS -x cuda) # Kokkos_CUDA_DIR has priority over CUDAToolkit_BIN_DIR - IF (Kokkos_CUDA_DIR) - GLOBAL_APPEND(KOKKOS_CUDA_OPTIONS --cuda-path=${Kokkos_CUDA_DIR}) - ELSEIF(CUDAToolkit_BIN_DIR) - GLOBAL_APPEND(KOKKOS_CUDA_OPTIONS --cuda-path=${CUDAToolkit_BIN_DIR}/..) - ENDIF() -ELSEIF(KOKKOS_CXX_COMPILER_ID STREQUAL NVIDIA) - SET(CUDA_ARCH_FLAG "-arch") -ENDIF() - -IF (KOKKOS_CXX_COMPILER_ID STREQUAL NVIDIA) - STRING(TOUPPER "${CMAKE_BUILD_TYPE}" _UPPERCASE_CMAKE_BUILD_TYPE) - IF (KOKKOS_ENABLE_DEBUG OR _UPPERCASE_CMAKE_BUILD_TYPE STREQUAL "DEBUG") - GLOBAL_APPEND(KOKKOS_CUDA_OPTIONS -lineinfo) - ENDIF() - UNSET(_UPPERCASE_CMAKE_BUILD_TYPE) -ENDIF() + if(Kokkos_CUDA_DIR) + global_append(KOKKOS_CUDA_OPTIONS --cuda-path=${Kokkos_CUDA_DIR}) + elseif(CUDAToolkit_BIN_DIR) + global_append(KOKKOS_CUDA_OPTIONS --cuda-path=${CUDAToolkit_BIN_DIR}/..) + endif() +elseif(KOKKOS_CXX_COMPILER_ID STREQUAL NVIDIA) + set(CUDA_ARCH_FLAG "-arch") +endif() +if(KOKKOS_CXX_COMPILER_ID STREQUAL NVIDIA) + string(TOUPPER "${CMAKE_BUILD_TYPE}" _UPPERCASE_CMAKE_BUILD_TYPE) + if(KOKKOS_ENABLE_DEBUG OR _UPPERCASE_CMAKE_BUILD_TYPE STREQUAL "DEBUG") + global_append(KOKKOS_CUDA_OPTIONS -lineinfo) + endif() + unset(_UPPERCASE_CMAKE_BUILD_TYPE) +endif() #------------------------------- KOKKOS_HIP_OPTIONS --------------------------- -KOKKOS_OPTION(IMPL_AMDGPU_FLAGS "" STRING "Set compiler flags for AMD GPUs") -KOKKOS_OPTION(IMPL_AMDGPU_LINK "" STRING "Set linker flags for AMD GPUs") -MARK_AS_ADVANCED(Kokkos_IMPL_AMDGPU_FLAGS) -MARK_AS_ADVANCED(Kokkos_IMPL_AMDGPU_LINK) +kokkos_option(IMPL_AMDGPU_FLAGS "" STRING "Set compiler flags for AMD GPUs") +kokkos_option(IMPL_AMDGPU_LINK "" STRING "Set linker flags for AMD GPUs") +mark_as_advanced(Kokkos_IMPL_AMDGPU_FLAGS) +mark_as_advanced(Kokkos_IMPL_AMDGPU_LINK) #clear anything that might be in the cache -GLOBAL_SET(KOKKOS_AMDGPU_OPTIONS) -IF(KOKKOS_ENABLE_HIP) - SET(AMDGPU_ARCH_FLAG "--offload-arch") - IF(NOT KOKKOS_CXX_COMPILER_ID STREQUAL HIPCC) - IF (NOT CMAKE_CXX_STANDARD) - MESSAGE(FATAL_ERROR "Kokkos requires CMAKE_CXX_STANDARD to set to 17 or higher") - ENDIF() - GLOBAL_APPEND(KOKKOS_AMDGPU_OPTIONS -xhip) - IF(DEFINED ENV{ROCM_PATH}) - GLOBAL_APPEND(KOKKOS_AMDGPU_OPTIONS --rocm-path=$ENV{ROCM_PATH}) - ENDIF() - ENDIF() -ENDIF() +global_set(KOKKOS_AMDGPU_OPTIONS) +if(KOKKOS_ENABLE_HIP) + set(AMDGPU_ARCH_FLAG "--offload-arch") + if(NOT KOKKOS_CXX_COMPILER_ID STREQUAL HIPCC) + if(NOT CMAKE_CXX_STANDARD) + message(FATAL_ERROR "Kokkos requires CMAKE_CXX_STANDARD to set to 17 or higher") + endif() + global_append(KOKKOS_AMDGPU_OPTIONS -xhip) + if(DEFINED ENV{ROCM_PATH}) + global_append(KOKKOS_AMDGPU_OPTIONS --rocm-path=$ENV{ROCM_PATH}) + endif() + endif() +endif() +if(KOKKOS_ARCH_NATIVE) + if(KOKKOS_CXX_HOST_COMPILER_ID STREQUAL "MSVC") + message(FATAL_ERROR "MSVC doesn't support ARCH_NATIVE!") + endif() -IF(KOKKOS_ARCH_NATIVE) - IF(KOKKOS_CXX_HOST_COMPILER_ID STREQUAL "MSVC") - MESSAGE(FATAL_ERROR "MSVC doesn't support ARCH_NATIVE!") - ENDIF() + string(TOUPPER "${CMAKE_SYSTEM_PROCESSOR}" KOKKOS_UC_SYSTEM_PROCESSOR) + if(KOKKOS_UC_SYSTEM_PROCESSOR MATCHES "(X86)|(AMD64)") + set(KOKKOS_NATIVE_FLAGS "-march=native;-mtune=native") + else() + set(KOKKOS_NATIVE_FLAGS "-mcpu=native") + endif() + compiler_specific_flags(COMPILER_ID KOKKOS_CXX_HOST_COMPILER_ID NVHPC -tp=native DEFAULT ${KOKKOS_NATIVE_FLAGS}) +endif() - STRING(TOUPPER "${CMAKE_SYSTEM_PROCESSOR}" KOKKOS_UC_SYSTEM_PROCESSOR) - IF(KOKKOS_UC_SYSTEM_PROCESSOR MATCHES "(X86)|(AMD64)") - SET(KOKKOS_NATIVE_FLAGS "-march=native;-mtune=native") - ELSE() - SET(KOKKOS_NATIVE_FLAGS "-mcpu=native") - ENDIF() - COMPILER_SPECIFIC_FLAGS( - COMPILER_ID KOKKOS_CXX_HOST_COMPILER_ID - NVHPC -tp=native - DEFAULT ${KOKKOS_NATIVE_FLAGS} +if(KOKKOS_ARCH_ARMV80) + set(KOKKOS_ARCH_ARM_NEON ON) + compiler_specific_flags( + COMPILER_ID + KOKKOS_CXX_HOST_COMPILER_ID + Cray + NO-VALUE-SPECIFIED + MSVC + /arch:armv8.0 + NVHPC + NO-VALUE-SPECIFIED + DEFAULT + -march=armv8-a ) -ENDIF() +endif() -IF (KOKKOS_ARCH_ARMV80) - SET(KOKKOS_ARCH_ARM_NEON ON) - COMPILER_SPECIFIC_FLAGS( - COMPILER_ID KOKKOS_CXX_HOST_COMPILER_ID - Cray NO-VALUE-SPECIFIED - MSVC /arch:armv8.0 - NVHPC NO-VALUE-SPECIFIED - DEFAULT -march=armv8-a +if(KOKKOS_ARCH_ARMV81) + set(KOKKOS_ARCH_ARM_NEON ON) + compiler_specific_flags( + COMPILER_ID + KOKKOS_CXX_HOST_COMPILER_ID + Cray + NO-VALUE-SPECIFIED + MSVC + /arch:armv8.1 + NVHPC + NO-VALUE-SPECIFIED + DEFAULT + -march=armv8.1-a ) -ENDIF() +endif() -IF (KOKKOS_ARCH_ARMV81) - SET(KOKKOS_ARCH_ARM_NEON ON) - COMPILER_SPECIFIC_FLAGS( - COMPILER_ID KOKKOS_CXX_HOST_COMPILER_ID - Cray NO-VALUE-SPECIFIED - MSVC /arch:armv8.1 - NVHPC NO-VALUE-SPECIFIED - DEFAULT -march=armv8.1-a +if(KOKKOS_ARCH_ARMV8_THUNDERX) + set(KOKKOS_ARCH_ARM_NEON ON) + set(KOKKOS_ARCH_ARMV80 ON) #Not a cache variable + compiler_specific_flags( + COMPILER_ID + KOKKOS_CXX_HOST_COMPILER_ID + Cray + NO-VALUE-SPECIFIED + MSVC + /arch:armv8.0 + NVHPC + NO-VALUE-SPECIFIED + DEFAULT + -march=armv8-a + -mtune=thunderx ) -ENDIF() +endif() -IF (KOKKOS_ARCH_ARMV8_THUNDERX) - SET(KOKKOS_ARCH_ARM_NEON ON) - SET(KOKKOS_ARCH_ARMV80 ON) #Not a cache variable - COMPILER_SPECIFIC_FLAGS( - COMPILER_ID KOKKOS_CXX_HOST_COMPILER_ID - Cray NO-VALUE-SPECIFIED - MSVC /arch:armv8.0 - NVHPC NO-VALUE-SPECIFIED - DEFAULT -march=armv8-a -mtune=thunderx +if(KOKKOS_ARCH_ARMV8_THUNDERX2) + set(KOKKOS_ARCH_ARM_NEON ON) + set(KOKKOS_ARCH_ARMV81 ON) #Not a cache variable + compiler_specific_flags( + COMPILER_ID + KOKKOS_CXX_HOST_COMPILER_ID + Cray + NO-VALUE-SPECIFIED + MSVC + /arch:armv8.1 + NVHPC + NO-VALUE-SPECIFIED + DEFAULT + -mcpu=thunderx2t99 + -mtune=thunderx2t99 ) -ENDIF() +endif() -IF (KOKKOS_ARCH_ARMV8_THUNDERX2) - SET(KOKKOS_ARCH_ARM_NEON ON) - SET(KOKKOS_ARCH_ARMV81 ON) #Not a cache variable - COMPILER_SPECIFIC_FLAGS( - COMPILER_ID KOKKOS_CXX_HOST_COMPILER_ID - Cray NO-VALUE-SPECIFIED - MSVC /arch:armv8.1 - NVHPC NO-VALUE-SPECIFIED - DEFAULT -mcpu=thunderx2t99 -mtune=thunderx2t99 +if(KOKKOS_ARCH_A64FX) + set(KOKKOS_ARCH_ARM_NEON ON) + compiler_specific_flags( + COMPILER_ID + KOKKOS_CXX_HOST_COMPILER_ID + Clang + -march=armv8.2-a+sve + -msve-vector-bits=512 + GNU + -march=armv8.2-a+sve + -msve-vector-bits=512 + MSVC + NO-VALUE-SPECIFIED + NVHPC + NO-VALUE-SPECIFIED + DEFAULT + -march=armv8.2-a+sve ) -ENDIF() +endif() -IF (KOKKOS_ARCH_A64FX) - SET(KOKKOS_ARCH_ARM_NEON ON) - COMPILER_SPECIFIC_FLAGS( - COMPILER_ID KOKKOS_CXX_HOST_COMPILER_ID - Clang -march=armv8.2-a+sve -msve-vector-bits=512 - GNU -march=armv8.2-a+sve -msve-vector-bits=512 - MSVC NO-VALUE-SPECIFIED - NVHPC NO-VALUE-SPECIFIED - DEFAULT -march=armv8.2-a+sve - ) -ENDIF() - -IF (KOKKOS_ARCH_ARMV9_GRACE) - SET(KOKKOS_ARCH_ARM_NEON ON) +if(KOKKOS_ARCH_ARMV9_GRACE) + set(KOKKOS_ARCH_ARM_NEON ON) check_cxx_compiler_flag("-mcpu=neoverse-n2" COMPILER_SUPPORTS_NEOVERSE_N2) check_cxx_compiler_flag("-msve-vector-bits=128" COMPILER_SUPPORTS_SVE_VECTOR_BITS) - IF (COMPILER_SUPPORTS_NEOVERSE_N2 AND COMPILER_SUPPORTS_SVE_VECTOR_BITS) - COMPILER_SPECIFIC_FLAGS( - COMPILER_ID KOKKOS_CXX_HOST_COMPILER_ID - DEFAULT -mcpu=neoverse-n2 -msve-vector-bits=128 - ) - ELSE() - MESSAGE(WARNING "Compiler does not support ARMv9 Grace architecture") - ENDIF() -ENDIF() + if(COMPILER_SUPPORTS_NEOVERSE_N2 AND COMPILER_SUPPORTS_SVE_VECTOR_BITS) + compiler_specific_flags(COMPILER_ID KOKKOS_CXX_HOST_COMPILER_ID DEFAULT -mcpu=neoverse-n2 -msve-vector-bits=128) + else() + message(WARNING "Compiler does not support ARMv9 Grace architecture") + endif() +endif() -IF (KOKKOS_ARCH_ZEN) - COMPILER_SPECIFIC_FLAGS( - COMPILER_ID KOKKOS_CXX_HOST_COMPILER_ID - Intel -mavx2 - MSVC /arch:AVX2 - NVHPC -tp=zen - DEFAULT -march=znver1 -mtune=znver1 +if(KOKKOS_ARCH_ZEN) + compiler_specific_flags( + COMPILER_ID + KOKKOS_CXX_HOST_COMPILER_ID + Intel + -mavx2 + MSVC + /arch:AVX2 + NVHPC + -tp=zen + DEFAULT + -march=znver1 + -mtune=znver1 ) - SET(KOKKOS_ARCH_AMD_ZEN ON) - SET(KOKKOS_ARCH_AVX2 ON) -ENDIF() + set(KOKKOS_ARCH_AMD_ZEN ON) + set(KOKKOS_ARCH_AVX2 ON) +endif() -IF (KOKKOS_ARCH_ZEN2) - COMPILER_SPECIFIC_FLAGS( - COMPILER_ID KOKKOS_CXX_HOST_COMPILER_ID - Intel -mavx2 - MSVC /arch:AVX2 - NVHPC -tp=zen2 - DEFAULT -march=znver2 -mtune=znver2 +if(KOKKOS_ARCH_ZEN2) + compiler_specific_flags( + COMPILER_ID + KOKKOS_CXX_HOST_COMPILER_ID + Intel + -mavx2 + MSVC + /arch:AVX2 + NVHPC + -tp=zen2 + DEFAULT + -march=znver2 + -mtune=znver2 ) - SET(KOKKOS_ARCH_AMD_ZEN2 ON) - SET(KOKKOS_ARCH_AVX2 ON) -ENDIF() + set(KOKKOS_ARCH_AMD_ZEN2 ON) + set(KOKKOS_ARCH_AVX2 ON) +endif() -IF (KOKKOS_ARCH_ZEN3) - COMPILER_SPECIFIC_FLAGS( - COMPILER_ID KOKKOS_CXX_HOST_COMPILER_ID - Intel -mavx2 - MSVC /arch:AVX2 - NVHPC -tp=zen2 - DEFAULT -march=znver3 -mtune=znver3 +if(KOKKOS_ARCH_ZEN3) + compiler_specific_flags( + COMPILER_ID + KOKKOS_CXX_HOST_COMPILER_ID + Intel + -mavx2 + MSVC + /arch:AVX2 + NVHPC + -tp=zen2 + DEFAULT + -march=znver3 + -mtune=znver3 ) - SET(KOKKOS_ARCH_AMD_ZEN3 ON) - SET(KOKKOS_ARCH_AVX2 ON) -ENDIF() + set(KOKKOS_ARCH_AMD_ZEN3 ON) + set(KOKKOS_ARCH_AVX2 ON) +endif() -IF (KOKKOS_ARCH_SNB OR KOKKOS_ARCH_AMDAVX) - SET(KOKKOS_ARCH_AVX ON) - COMPILER_SPECIFIC_FLAGS( - COMPILER_ID KOKKOS_CXX_HOST_COMPILER_ID - Cray NO-VALUE-SPECIFIED - Intel -mavx - MSVC /arch:AVX - NVHPC -tp=sandybridge - DEFAULT -mavx +if(KOKKOS_ARCH_SNB OR KOKKOS_ARCH_AMDAVX) + set(KOKKOS_ARCH_AVX ON) + compiler_specific_flags( + COMPILER_ID + KOKKOS_CXX_HOST_COMPILER_ID + Cray + NO-VALUE-SPECIFIED + Intel + -mavx + MSVC + /arch:AVX + NVHPC + -tp=sandybridge + DEFAULT + -mavx ) -ENDIF() +endif() -IF (KOKKOS_ARCH_HSW) - SET(KOKKOS_ARCH_AVX2 ON) - COMPILER_SPECIFIC_FLAGS( - COMPILER_ID KOKKOS_CXX_HOST_COMPILER_ID - Cray NO-VALUE-SPECIFIED - Intel -xCORE-AVX2 - MSVC /arch:AVX2 - NVHPC -tp=haswell - DEFAULT -march=core-avx2 -mtune=core-avx2 +if(KOKKOS_ARCH_HSW) + set(KOKKOS_ARCH_AVX2 ON) + compiler_specific_flags( + COMPILER_ID + KOKKOS_CXX_HOST_COMPILER_ID + Cray + NO-VALUE-SPECIFIED + Intel + -xCORE-AVX2 + MSVC + /arch:AVX2 + NVHPC + -tp=haswell + DEFAULT + -march=core-avx2 + -mtune=core-avx2 ) -ENDIF() +endif() -IF (KOKKOS_ARCH_RISCV_SG2042) - IF(NOT - (KOKKOS_CXX_COMPILER_ID STREQUAL GNU - AND KOKKOS_CXX_COMPILER_VERSION VERSION_GREATER_EQUAL 12) - OR - (KOKKOS_CXX_COMPILER_ID STREQUAL Clang - AND KOKKOS_CXX_COMPILER_VERSION VERSION_GREATER_EQUAL 14) +if(KOKKOS_ARCH_RISCV_SG2042) + if(NOT (KOKKOS_CXX_COMPILER_ID STREQUAL GNU AND KOKKOS_CXX_COMPILER_VERSION VERSION_GREATER_EQUAL 12) + OR (KOKKOS_CXX_COMPILER_ID STREQUAL Clang AND KOKKOS_CXX_COMPILER_VERSION VERSION_GREATER_EQUAL 14) ) - MESSAGE(SEND_ERROR "Only gcc >= 12 and clang >= 14 support RISC-V.") - ENDIF() - COMPILER_SPECIFIC_FLAGS( - COMPILER_ID KOKKOS_CXX_HOST_COMPILER_ID - DEFAULT -march=rv64imafdcv - ) -ENDIF() + message(SEND_ERROR "Only gcc >= 12 and clang >= 14 support RISC-V.") + endif() + compiler_specific_flags(COMPILER_ID KOKKOS_CXX_HOST_COMPILER_ID DEFAULT -march=rv64imafdcv) +endif() - -IF (KOKKOS_ARCH_BDW) - SET(KOKKOS_ARCH_AVX2 ON) - COMPILER_SPECIFIC_FLAGS( - COMPILER_ID KOKKOS_CXX_HOST_COMPILER_ID - Cray NO-VALUE-SPECIFIED - Intel -xCORE-AVX2 - MSVC /arch:AVX2 - NVHPC -tp=haswell - DEFAULT -march=core-avx2 -mtune=core-avx2 -mrtm +if(KOKKOS_ARCH_RISCV_RVA22V) + if(NOT (KOKKOS_CXX_COMPILER_ID STREQUAL GNU AND KOKKOS_CXX_COMPILER_VERSION VERSION_GREATER_EQUAL 12) + OR (KOKKOS_CXX_COMPILER_ID STREQUAL Clang AND KOKKOS_CXX_COMPILER_VERSION VERSION_GREATER_EQUAL 14) ) -ENDIF() + message(SEND_ERROR "Only gcc >= 12 and clang >= 14 support RISC-V.") + endif() + compiler_specific_flags( + COMPILER_ID KOKKOS_CXX_HOST_COMPILER_ID DEFAULT + -march=rv64imafdcv_sscofpmf_sstc_svpbmt_zicbom_zicboz_zicbop_zihintpause + ) +endif() -IF (KOKKOS_ARCH_KNL) +if(KOKKOS_ARCH_BDW) + set(KOKKOS_ARCH_AVX2 ON) + compiler_specific_flags( + COMPILER_ID + KOKKOS_CXX_HOST_COMPILER_ID + Cray + NO-VALUE-SPECIFIED + Intel + -xCORE-AVX2 + MSVC + /arch:AVX2 + NVHPC + -tp=haswell + DEFAULT + -march=core-avx2 + -mtune=core-avx2 + -mrtm + ) +endif() + +if(KOKKOS_ARCH_KNL) #avx512-mic - SET(KOKKOS_ARCH_AVX512MIC ON) #not a cache variable - COMPILER_SPECIFIC_FLAGS( - COMPILER_ID KOKKOS_CXX_HOST_COMPILER_ID - Cray NO-VALUE-SPECIFIED - Intel -xMIC-AVX512 - MSVC /arch:AVX512 - NVHPC -tp=knl - DEFAULT -march=knl -mtune=knl + set(KOKKOS_ARCH_AVX512MIC ON) #not a cache variable + compiler_specific_flags( + COMPILER_ID + KOKKOS_CXX_HOST_COMPILER_ID + Cray + NO-VALUE-SPECIFIED + Intel + -xMIC-AVX512 + MSVC + /arch:AVX512 + NVHPC + -tp=knl + DEFAULT + -march=knl + -mtune=knl ) -ENDIF() +endif() -IF (KOKKOS_ARCH_KNC) - COMPILER_SPECIFIC_FLAGS( - COMPILER_ID KOKKOS_CXX_HOST_COMPILER_ID - MSVC NO-VALUE-SPECIFIED - DEFAULT -mmic - ) -ENDIF() +if(KOKKOS_ARCH_KNC) + compiler_specific_flags(COMPILER_ID KOKKOS_CXX_HOST_COMPILER_ID MSVC NO-VALUE-SPECIFIED DEFAULT -mmic) +endif() -IF (KOKKOS_ARCH_SKL) - COMPILER_SPECIFIC_FLAGS( - COMPILER_ID KOKKOS_CXX_HOST_COMPILER_ID - Cray NO-VALUE-SPECIFIED - Intel -xSKYLAKE - MSVC /arch:AVX2 - NVHPC -tp=skylake - DEFAULT -march=skylake -mtune=skylake +if(KOKKOS_ARCH_SKL) + compiler_specific_flags( + COMPILER_ID + KOKKOS_CXX_HOST_COMPILER_ID + Cray + NO-VALUE-SPECIFIED + Intel + -xSKYLAKE + MSVC + /arch:AVX2 + NVHPC + -tp=skylake + DEFAULT + -march=skylake + -mtune=skylake ) -ENDIF() +endif() -IF (KOKKOS_ARCH_SKX) - SET(KOKKOS_ARCH_AVX512XEON ON) - COMPILER_SPECIFIC_FLAGS( - COMPILER_ID KOKKOS_CXX_HOST_COMPILER_ID - Cray NO-VALUE-SPECIFIED - Intel -xCORE-AVX512 - MSVC /arch:AVX512 - NVHPC -tp=skylake - DEFAULT -march=skylake-avx512 -mtune=skylake-avx512 +if(KOKKOS_ARCH_SKX) + set(KOKKOS_ARCH_AVX512XEON ON) + compiler_specific_flags( + COMPILER_ID + KOKKOS_CXX_HOST_COMPILER_ID + Cray + NO-VALUE-SPECIFIED + Intel + -xCORE-AVX512 + MSVC + /arch:AVX512 + NVHPC + -tp=skylake + DEFAULT + -march=skylake-avx512 + -mtune=skylake-avx512 ) -ENDIF() +endif() -IF (KOKKOS_ARCH_ICL) - SET(KOKKOS_ARCH_AVX512XEON ON) - COMPILER_SPECIFIC_FLAGS( - COMPILER_ID KOKKOS_CXX_HOST_COMPILER_ID - MSVC /arch:AVX512 - DEFAULT -march=icelake-client -mtune=icelake-client +if(KOKKOS_ARCH_ICL) + set(KOKKOS_ARCH_AVX512XEON ON) + compiler_specific_flags( + COMPILER_ID + KOKKOS_CXX_HOST_COMPILER_ID + MSVC + /arch:AVX512 + DEFAULT + -march=icelake-client + -mtune=icelake-client ) -ENDIF() +endif() -IF (KOKKOS_ARCH_ICX) - SET(KOKKOS_ARCH_AVX512XEON ON) - COMPILER_SPECIFIC_FLAGS( - COMPILER_ID KOKKOS_CXX_HOST_COMPILER_ID - MSVC /arch:AVX512 - DEFAULT -march=icelake-server -mtune=icelake-server +if(KOKKOS_ARCH_ICX) + set(KOKKOS_ARCH_AVX512XEON ON) + compiler_specific_flags( + COMPILER_ID + KOKKOS_CXX_HOST_COMPILER_ID + MSVC + /arch:AVX512 + DEFAULT + -march=icelake-server + -mtune=icelake-server ) -ENDIF() +endif() -IF (KOKKOS_ARCH_SPR) - SET(KOKKOS_ARCH_AVX512XEON ON) - COMPILER_SPECIFIC_FLAGS( - COMPILER_ID KOKKOS_CXX_HOST_COMPILER_ID - MSVC /arch:AVX512 - DEFAULT -march=sapphirerapids -mtune=sapphirerapids +if(KOKKOS_ARCH_SPR) + set(KOKKOS_ARCH_AVX512XEON ON) + compiler_specific_flags( + COMPILER_ID + KOKKOS_CXX_HOST_COMPILER_ID + MSVC + /arch:AVX512 + DEFAULT + -march=sapphirerapids + -mtune=sapphirerapids ) -ENDIF() +endif() -IF (KOKKOS_ARCH_POWER7) - COMPILER_SPECIFIC_FLAGS( - COMPILER_ID KOKKOS_CXX_HOST_COMPILER_ID - MSVC NO-VALUE-SPECIFIED - NVHPC NO-VALUE-SPECIFIED - DEFAULT -mcpu=power7 -mtune=power7 +if(KOKKOS_ARCH_POWER7) + compiler_specific_flags( + COMPILER_ID + KOKKOS_CXX_HOST_COMPILER_ID + MSVC + NO-VALUE-SPECIFIED + NVHPC + NO-VALUE-SPECIFIED + DEFAULT + -mcpu=power7 + -mtune=power7 ) -ENDIF() +endif() -IF (KOKKOS_ARCH_POWER8) - COMPILER_SPECIFIC_FLAGS( - COMPILER_ID KOKKOS_CXX_HOST_COMPILER_ID - MSVC NO-VALUE-SPECIFIED - NVHPC -tp=pwr8 - DEFAULT -mcpu=power8 -mtune=power8 +if(KOKKOS_ARCH_POWER8) + compiler_specific_flags( + COMPILER_ID + KOKKOS_CXX_HOST_COMPILER_ID + MSVC + NO-VALUE-SPECIFIED + NVHPC + -tp=pwr8 + DEFAULT + -mcpu=power8 + -mtune=power8 ) -ENDIF() +endif() -IF (KOKKOS_ARCH_POWER9) - COMPILER_SPECIFIC_FLAGS( - COMPILER_ID KOKKOS_CXX_HOST_COMPILER_ID - MSVC NO-VALUE-SPECIFIED - NVHPC -tp=pwr9 - DEFAULT -mcpu=power9 -mtune=power9 +if(KOKKOS_ARCH_POWER9) + compiler_specific_flags( + COMPILER_ID + KOKKOS_CXX_HOST_COMPILER_ID + MSVC + NO-VALUE-SPECIFIED + NVHPC + -tp=pwr9 + DEFAULT + -mcpu=power9 + -mtune=power9 ) -ENDIF() +endif() # If Kokkos_ARCH_NATIVE is enabled, we are trying to autodetect # the SIMD capabilities based on compiler macros. -IF (KOKKOS_ARCH_NATIVE) +if(KOKKOS_ARCH_NATIVE) # Make sure to rerun the checks if compile options have changed - IF(NOT "${KOKKOS_COMPILE_OPTIONS}" STREQUAL "${KOKKOS_COMPILE_OPTIONS_SAVED}") - SET(KOKKOS_COMPILE_OPTIONS_SAVED "${KOKKOS_COMPILE_OPTIONS}" CACHE INTERNAL "") + if(NOT "${KOKKOS_COMPILE_OPTIONS}" STREQUAL "${KOKKOS_COMPILE_OPTIONS_SAVED}") + set(KOKKOS_COMPILE_OPTIONS_SAVED "${KOKKOS_COMPILE_OPTIONS}" CACHE INTERNAL "") - SET(CMAKE_REQUIRED_QUIET ON) - SET(CMAKE_REQUIRED_FLAGS "${KOKKOS_COMPILE_OPTIONS}") - INCLUDE(CheckCXXSymbolExists) + set(CMAKE_REQUIRED_QUIET ON) + set(CMAKE_REQUIRED_FLAGS "${KOKKOS_COMPILE_OPTIONS}") + include(CheckCXXSymbolExists) - UNSET(KOKKOS_COMPILER_HAS_AVX512 CACHE) - CHECK_CXX_SYMBOL_EXISTS(__AVX512F__ "" KOKKOS_COMPILER_HAS_AVX512) - UNSET(KOKKOS_COMPILER_HAS_AVX2 CACHE) - CHECK_CXX_SYMBOL_EXISTS(__AVX2__ "" KOKKOS_COMPILER_HAS_AVX2) - UNSET(KOKKOS_COMPILER_HAS_ARM_NEON CACHE) - CHECK_CXX_SYMBOL_EXISTS(__ARM_NEON "" KOKKOS_COMPILER_HAS_ARM_NEON) - UNSET(KOKKOS_COMPILER_HAS_AVX CACHE) - CHECK_CXX_SYMBOL_EXISTS(__AVX__ "" KOKKOS_COMPILER_HAS_AVX) - SET(CMAKE_REQUIRED_FLAGS "${KOKKOS_COMPILE_OPTIONS}") + unset(KOKKOS_COMPILER_HAS_AVX512 CACHE) + check_cxx_symbol_exists(__AVX512F__ "" KOKKOS_COMPILER_HAS_AVX512) + unset(KOKKOS_COMPILER_HAS_AVX2 CACHE) + check_cxx_symbol_exists(__AVX2__ "" KOKKOS_COMPILER_HAS_AVX2) + unset(KOKKOS_COMPILER_HAS_ARM_NEON CACHE) + check_cxx_symbol_exists(__ARM_NEON "" KOKKOS_COMPILER_HAS_ARM_NEON) + unset(KOKKOS_COMPILER_HAS_AVX CACHE) + check_cxx_symbol_exists(__AVX__ "" KOKKOS_COMPILER_HAS_AVX) + set(CMAKE_REQUIRED_FLAGS "${KOKKOS_COMPILE_OPTIONS}") - UNSET(CMAKE_REQUIRED_QUIET) - UNSET(CMAKE_REQUIRED_FLAGS) - ENDIF() + unset(CMAKE_REQUIRED_QUIET) + unset(CMAKE_REQUIRED_FLAGS) + endif() # Only define one of these macros for now # to be uniform with what we are doing for other architectures. - IF(KOKKOS_COMPILER_HAS_AVX512) - MESSAGE(STATUS "SIMD: AVX512 detected") - SET(KOKKOS_ARCH_AVX512XEON ON) - ELSEIF(KOKKOS_COMPILER_HAS_AVX2) - MESSAGE(STATUS "SIMD: AVX2 detected") - SET(KOKKOS_ARCH_AVX2 ON) - ELSEIF(KOKKOS_COMPILER_HAS_ARM_NEON) - MESSAGE(STATUS "SIMD: ARM_NEON detected") - SET(KOKKOS_ARCH_ARM_NEON ON) - ELSEIF(KOKKOS_COMPILER_HAS_AVX) - MESSAGE(STATUS "SIMD: AVX detected") - SET(KOKKOS_ARCH_AVX ON) - ENDIF() -ENDIF() + if(KOKKOS_COMPILER_HAS_AVX512) + message(STATUS "SIMD: AVX512 detected") + set(KOKKOS_ARCH_AVX512XEON ON) + elseif(KOKKOS_COMPILER_HAS_AVX2) + message(STATUS "SIMD: AVX2 detected") + set(KOKKOS_ARCH_AVX2 ON) + elseif(KOKKOS_COMPILER_HAS_ARM_NEON) + message(STATUS "SIMD: ARM_NEON detected") + set(KOKKOS_ARCH_ARM_NEON ON) + elseif(KOKKOS_COMPILER_HAS_AVX) + message(STATUS "SIMD: AVX detected") + set(KOKKOS_ARCH_AVX ON) + endif() +endif() # FIXME_NVHPC nvc++ doesn't seem to support AVX512. -IF (KOKKOS_CXX_HOST_COMPILER_ID STREQUAL NVHPC) - SET(KOKKOS_ARCH_AVX512XEON OFF) -ENDIF() +if(KOKKOS_CXX_HOST_COMPILER_ID STREQUAL NVHPC) + set(KOKKOS_ARCH_AVX512XEON OFF) +endif() # FIXME_NVCC nvcc doesn't seem to support Arm Neon. -IF(KOKKOS_ARCH_ARM_NEON AND KOKKOS_CXX_COMPILER_ID STREQUAL NVIDIA) - UNSET(KOKKOS_ARCH_ARM_NEON) -ENDIF() +if(KOKKOS_ARCH_ARM_NEON AND KOKKOS_CXX_COMPILER_ID STREQUAL NVIDIA) + unset(KOKKOS_ARCH_ARM_NEON) +endif() -IF (NOT KOKKOS_COMPILE_LANGUAGE STREQUAL CUDA) - IF (KOKKOS_ENABLE_CUDA_RELOCATABLE_DEVICE_CODE) - COMPILER_SPECIFIC_FLAGS( - Clang -fcuda-rdc - NVIDIA --relocatable-device-code=true - ) - ENDIF() -ENDIF() +if(NOT KOKKOS_COMPILE_LANGUAGE STREQUAL CUDA) + if(KOKKOS_ENABLE_CUDA_RELOCATABLE_DEVICE_CODE) + compiler_specific_flags(Clang -fcuda-rdc NVIDIA --relocatable-device-code=true) + endif() +endif() # Clang needs mcx16 option enabled for Windows atomic functions -IF (CMAKE_CXX_COMPILER_ID STREQUAL Clang AND WIN32) - COMPILER_SPECIFIC_OPTIONS( - Clang -mcx16 - ) -ENDIF() +if(CMAKE_CXX_COMPILER_ID STREQUAL Clang AND WIN32) + compiler_specific_options(Clang -mcx16) +endif() # MSVC ABI has many deprecation warnings, so ignore them -IF (CMAKE_CXX_COMPILER_ID STREQUAL "MSVC" OR "x${CMAKE_CXX_SIMULATE_ID}" STREQUAL "xMSVC") - COMPILER_SPECIFIC_DEFS( - Clang _CRT_SECURE_NO_WARNINGS - ) -ENDIF() - +if(CMAKE_CXX_COMPILER_ID STREQUAL "MSVC" OR "x${CMAKE_CXX_SIMULATE_ID}" STREQUAL "xMSVC") + compiler_specific_defs(Clang _CRT_SECURE_NO_WARNINGS) +endif() #Right now we cannot get the compiler ID when cross-compiling, so just check #that HIP is enabled -IF (KOKKOS_ENABLE_HIP) - IF (KOKKOS_ENABLE_HIP_RELOCATABLE_DEVICE_CODE) - COMPILER_SPECIFIC_FLAGS( - DEFAULT -fgpu-rdc - ) - IF (NOT KOKKOS_CXX_COMPILER_ID STREQUAL HIPCC AND NOT KOKKOS_IMPL_AMDGPU_FLAGS) - COMPILER_SPECIFIC_LINK_OPTIONS( - DEFAULT --hip-link - ) - ENDIF() - ELSE() - COMPILER_SPECIFIC_FLAGS( - DEFAULT -fno-gpu-rdc - ) - ENDIF() -ENDIF() +if(KOKKOS_ENABLE_HIP) + if(KOKKOS_ENABLE_HIP_RELOCATABLE_DEVICE_CODE) + compiler_specific_flags(DEFAULT -fgpu-rdc) + if(NOT KOKKOS_CXX_COMPILER_ID STREQUAL HIPCC AND NOT KOKKOS_IMPL_AMDGPU_FLAGS) + compiler_specific_link_options(DEFAULT --hip-link) + endif() + else() + compiler_specific_flags(DEFAULT -fno-gpu-rdc) + endif() +endif() -IF (KOKKOS_ENABLE_SYCL) - COMPILER_SPECIFIC_FLAGS( - DEFAULT -fsycl -fno-sycl-id-queries-fit-in-int -fsycl-dead-args-optimization - ) - COMPILER_SPECIFIC_OPTIONS( - DEFAULT -fsycl-unnamed-lambda - ) -ENDIF() +if(KOKKOS_ENABLE_SYCL) + compiler_specific_flags(DEFAULT -fsycl -fno-sycl-id-queries-fit-in-int -fsycl-dead-args-optimization) + compiler_specific_options(DEFAULT -fsycl-unnamed-lambda) + if(KOKKOS_CXX_COMPILER_ID STREQUAL IntelLLVM AND KOKKOS_CXX_COMPILER_VERSION VERSION_LESS 2024.1.0) + # Before oneAPI 2024.1.0 passing -fno-sycl didn't work properly + if(NOT KOKKOS_ENABLE_SYCL_RELOCATABLE_DEVICE_CODE) + message(FATAL_ERROR "Kokkos_ENABLE_SYCL_RELOCATABLE_DEVICE_CODE=OFF requires oneAPI 2024.1.0 or later") + endif() + elseif(KOKKOS_ENABLE_SYCL_RELOCATABLE_DEVICE_CODE) + compiler_specific_options(DEFAULT -fsycl-rdc) + else() + compiler_specific_options(DEFAULT -fno-sycl-rdc) + endif() +endif() # Check support for device_global variables # FIXME_SYCL If SYCL_EXT_ONEAPI_DEVICE_GLOBAL is defined, we can use device @@ -613,17 +734,18 @@ ENDIF() # implementation. Otherwise, the feature is not supported when building shared # libraries. Thus, we don't even check for support if shared libraries are # requested and SYCL_EXT_ONEAPI_DEVICE_GLOBAL is not defined. -IF(KOKKOS_ENABLE_SYCL) - STRING(REPLACE ";" " " CMAKE_REQUIRED_FLAGS "${KOKKOS_COMPILE_OPTIONS}") - INCLUDE(CheckCXXSymbolExists) - CHECK_CXX_SYMBOL_EXISTS(SYCL_EXT_ONEAPI_DEVICE_GLOBAL "sycl/sycl.hpp" KOKKOS_IMPL_HAVE_SYCL_EXT_ONEAPI_DEVICE_GLOBAL) - IF (KOKKOS_IMPL_HAVE_SYCL_EXT_ONEAPI_DEVICE_GLOBAL) - SET(KOKKOS_IMPL_SYCL_DEVICE_GLOBAL_SUPPORTED ON) +if(KOKKOS_ENABLE_SYCL) + string(REPLACE ";" " " CMAKE_REQUIRED_FLAGS "${KOKKOS_COMPILE_OPTIONS}") + include(CheckCXXSymbolExists) + check_cxx_symbol_exists(SYCL_EXT_ONEAPI_DEVICE_GLOBAL "sycl/sycl.hpp" KOKKOS_IMPL_HAVE_SYCL_EXT_ONEAPI_DEVICE_GLOBAL) + if(KOKKOS_IMPL_HAVE_SYCL_EXT_ONEAPI_DEVICE_GLOBAL) + set(KOKKOS_IMPL_SYCL_DEVICE_GLOBAL_SUPPORTED ON) # Use the non-separable compilation implementation to support shared libraries as well. - COMPILER_SPECIFIC_FLAGS(DEFAULT -DDESUL_SYCL_DEVICE_GLOBAL_SUPPORTED) - ELSEIF(NOT BUILD_SHARED_LIBS) - INCLUDE(CheckCXXSourceCompiles) - CHECK_CXX_SOURCE_COMPILES(" + compiler_specific_flags(DEFAULT -DDESUL_SYCL_DEVICE_GLOBAL_SUPPORTED) + elseif(NOT BUILD_SHARED_LIBS AND KOKKOS_ENABLE_SYCL_RELOCATABLE_DEVICE_CODE) + include(CheckCXXSourceCompiles) + check_cxx_source_compiles( + " #include using namespace sycl::ext::oneapi::experimental; using namespace sycl; @@ -638,548 +760,617 @@ IF(KOKKOS_ENABLE_SYCL) int main(){ return 0; } " - KOKKOS_IMPL_SYCL_DEVICE_GLOBAL_SUPPORTED) + KOKKOS_IMPL_SYCL_DEVICE_GLOBAL_SUPPORTED + ) - IF(KOKKOS_IMPL_SYCL_DEVICE_GLOBAL_SUPPORTED) + if(KOKKOS_IMPL_SYCL_DEVICE_GLOBAL_SUPPORTED) # Only the separable compilation implementation is supported. - COMPILER_SPECIFIC_FLAGS( - DEFAULT -fsycl-device-code-split=off -DDESUL_SYCL_DEVICE_GLOBAL_SUPPORTED + compiler_specific_flags(DEFAULT -fsycl-device-code-split=off -DDESUL_SYCL_DEVICE_GLOBAL_SUPPORTED) + endif() + endif() + + check_cxx_symbol_exists(SYCL_EXT_ONEAPI_GRAPH "sycl/sycl.hpp" KOKKOS_IMPL_HAVE_SYCL_EXT_ONEAPI_GRAPH) +endif() + +set(CUDA_ARCH_ALREADY_SPECIFIED "") +function(CHECK_CUDA_ARCH ARCH FLAG) + if(KOKKOS_ARCH_${ARCH}) + if(CUDA_ARCH_ALREADY_SPECIFIED) + message( + FATAL_ERROR + "Multiple GPU architectures given! Already have ${CUDA_ARCH_ALREADY_SPECIFIED}, but trying to add ${ARCH}. If you are re-running CMake, try clearing the cache and running again." ) - ENDIF() - ENDIF() -ENDIF() - -SET(CUDA_ARCH_ALREADY_SPECIFIED "") -FUNCTION(CHECK_CUDA_ARCH ARCH FLAG) - IF(KOKKOS_ARCH_${ARCH}) - IF(CUDA_ARCH_ALREADY_SPECIFIED) - MESSAGE(FATAL_ERROR "Multiple GPU architectures given! Already have ${CUDA_ARCH_ALREADY_SPECIFIED}, but trying to add ${ARCH}. If you are re-running CMake, try clearing the cache and running again.") - ENDIF() - SET(CUDA_ARCH_ALREADY_SPECIFIED ${ARCH} PARENT_SCOPE) - IF (NOT KOKKOS_ENABLE_CUDA AND NOT KOKKOS_ENABLE_OPENMPTARGET AND NOT KOKKOS_ENABLE_SYCL AND NOT KOKKOS_ENABLE_OPENACC) - MESSAGE(WARNING "Given CUDA arch ${ARCH}, but Kokkos_ENABLE_CUDA, Kokkos_ENABLE_SYCL, Kokkos_ENABLE_OPENACC, and Kokkos_ENABLE_OPENMPTARGET are OFF. Option will be ignored.") - UNSET(KOKKOS_ARCH_${ARCH} PARENT_SCOPE) - ELSE() - IF(KOKKOS_ENABLE_CUDA) - STRING(REPLACE "sm_" "" CMAKE_ARCH ${FLAG}) - SET(KOKKOS_CUDA_ARCHITECTURES ${CMAKE_ARCH}) - SET(KOKKOS_CUDA_ARCHITECTURES ${CMAKE_ARCH} PARENT_SCOPE) - ENDIF() - SET(KOKKOS_CUDA_ARCH_FLAG ${FLAG} PARENT_SCOPE) - IF(KOKKOS_ENABLE_COMPILE_AS_CMAKE_LANGUAGE) - SET(CMAKE_CUDA_ARCHITECTURES ${KOKKOS_CUDA_ARCHITECTURES} PARENT_SCOPE) - ELSE() - GLOBAL_APPEND(KOKKOS_CUDA_OPTIONS "${CUDA_ARCH_FLAG}=${FLAG}") - IF(KOKKOS_ENABLE_CUDA_RELOCATABLE_DEVICE_CODE OR KOKKOS_CXX_COMPILER_ID STREQUAL NVIDIA) - GLOBAL_APPEND(KOKKOS_LINK_OPTIONS "${CUDA_ARCH_FLAG}=${FLAG}") - ENDIF() - ENDIF() - ENDIF() - ENDIF() - LIST(APPEND KOKKOS_CUDA_ARCH_FLAGS ${FLAG}) - SET(KOKKOS_CUDA_ARCH_FLAGS ${KOKKOS_CUDA_ARCH_FLAGS} PARENT_SCOPE) - LIST(APPEND KOKKOS_CUDA_ARCH_LIST ${ARCH}) - SET(KOKKOS_CUDA_ARCH_LIST ${KOKKOS_CUDA_ARCH_LIST} PARENT_SCOPE) -ENDFUNCTION() - + endif() + set(CUDA_ARCH_ALREADY_SPECIFIED ${ARCH} PARENT_SCOPE) + if(NOT KOKKOS_ENABLE_CUDA + AND NOT KOKKOS_ENABLE_OPENMPTARGET + AND NOT KOKKOS_ENABLE_SYCL + AND NOT KOKKOS_ENABLE_OPENACC + ) + message( + WARNING + "Given CUDA arch ${ARCH}, but Kokkos_ENABLE_CUDA, Kokkos_ENABLE_SYCL, Kokkos_ENABLE_OPENACC, and Kokkos_ENABLE_OPENMPTARGET are OFF. Option will be ignored." + ) + unset(KOKKOS_ARCH_${ARCH} PARENT_SCOPE) + else() + if(KOKKOS_ENABLE_CUDA) + string(REPLACE "sm_" "" CMAKE_ARCH ${FLAG}) + set(KOKKOS_CUDA_ARCHITECTURES ${CMAKE_ARCH}) + set(KOKKOS_CUDA_ARCHITECTURES ${CMAKE_ARCH} PARENT_SCOPE) + endif() + set(KOKKOS_CUDA_ARCH_FLAG ${FLAG} PARENT_SCOPE) + if(KOKKOS_ENABLE_COMPILE_AS_CMAKE_LANGUAGE) + set(CMAKE_CUDA_ARCHITECTURES ${KOKKOS_CUDA_ARCHITECTURES} PARENT_SCOPE) + else() + global_append(KOKKOS_CUDA_OPTIONS "${CUDA_ARCH_FLAG}=${FLAG}") + if(KOKKOS_ENABLE_CUDA_RELOCATABLE_DEVICE_CODE OR KOKKOS_CXX_COMPILER_ID STREQUAL NVIDIA) + global_append(KOKKOS_LINK_OPTIONS "${CUDA_ARCH_FLAG}=${FLAG}") + endif() + endif() + endif() + endif() + list(APPEND KOKKOS_CUDA_ARCH_FLAGS ${FLAG}) + set(KOKKOS_CUDA_ARCH_FLAGS ${KOKKOS_CUDA_ARCH_FLAGS} PARENT_SCOPE) + list(APPEND KOKKOS_CUDA_ARCH_LIST ${ARCH}) + set(KOKKOS_CUDA_ARCH_LIST ${KOKKOS_CUDA_ARCH_LIST} PARENT_SCOPE) +endfunction() #These will define KOKKOS_CUDA_ARCH_FLAG #to the corresponding flag name if ON -CHECK_CUDA_ARCH(KEPLER30 sm_30) -CHECK_CUDA_ARCH(KEPLER32 sm_32) -CHECK_CUDA_ARCH(KEPLER35 sm_35) -CHECK_CUDA_ARCH(KEPLER37 sm_37) -CHECK_CUDA_ARCH(MAXWELL50 sm_50) -CHECK_CUDA_ARCH(MAXWELL52 sm_52) -CHECK_CUDA_ARCH(MAXWELL53 sm_53) -CHECK_CUDA_ARCH(PASCAL60 sm_60) -CHECK_CUDA_ARCH(PASCAL61 sm_61) -CHECK_CUDA_ARCH(VOLTA70 sm_70) -CHECK_CUDA_ARCH(VOLTA72 sm_72) -CHECK_CUDA_ARCH(TURING75 sm_75) -CHECK_CUDA_ARCH(AMPERE80 sm_80) -CHECK_CUDA_ARCH(AMPERE86 sm_86) -CHECK_CUDA_ARCH(ADA89 sm_89) -CHECK_CUDA_ARCH(HOPPER90 sm_90) +check_cuda_arch(KEPLER30 sm_30) +check_cuda_arch(KEPLER32 sm_32) +check_cuda_arch(KEPLER35 sm_35) +check_cuda_arch(KEPLER37 sm_37) +check_cuda_arch(MAXWELL50 sm_50) +check_cuda_arch(MAXWELL52 sm_52) +check_cuda_arch(MAXWELL53 sm_53) +check_cuda_arch(PASCAL60 sm_60) +check_cuda_arch(PASCAL61 sm_61) +check_cuda_arch(VOLTA70 sm_70) +check_cuda_arch(VOLTA72 sm_72) +check_cuda_arch(TURING75 sm_75) +check_cuda_arch(AMPERE80 sm_80) +check_cuda_arch(AMPERE86 sm_86) +check_cuda_arch(ADA89 sm_89) +check_cuda_arch(HOPPER90 sm_90) -SET(AMDGPU_ARCH_ALREADY_SPECIFIED "") -FUNCTION(CHECK_AMDGPU_ARCH ARCH FLAG) - IF(KOKKOS_ARCH_${ARCH}) - IF(AMDGPU_ARCH_ALREADY_SPECIFIED) - MESSAGE(FATAL_ERROR "Multiple GPU architectures given! Already have ${AMDGPU_ARCH_ALREADY_SPECIFIED}, but trying to add ${ARCH}. If you are re-running CMake, try clearing the cache and running again.") - ENDIF() - SET(AMDGPU_ARCH_ALREADY_SPECIFIED ${ARCH} PARENT_SCOPE) - IF (NOT KOKKOS_ENABLE_HIP AND NOT KOKKOS_ENABLE_OPENMPTARGET AND NOT KOKKOS_ENABLE_OPENACC AND NOT KOKKOS_ENABLE_SYCL) - MESSAGE(WARNING "Given AMD GPU architecture ${ARCH}, but Kokkos_ENABLE_HIP, Kokkos_ENABLE_SYCL, Kokkos_ENABLE_OPENACC, and Kokkos_ENABLE_OPENMPTARGET are OFF. Option will be ignored.") - UNSET(KOKKOS_ARCH_${ARCH} PARENT_SCOPE) - ELSE() - IF(KOKKOS_ENABLE_HIP) - SET(KOKKOS_HIP_ARCHITECTURES ${FLAG} PARENT_SCOPE) - ENDIF() - IF(NOT KOKKOS_IMPL_AMDGPU_FLAGS) - SET(KOKKOS_AMDGPU_ARCH_FLAG ${FLAG} PARENT_SCOPE) - GLOBAL_APPEND(KOKKOS_AMDGPU_OPTIONS "${AMDGPU_ARCH_FLAG}=${FLAG}") - ENDIF() - IF(KOKKOS_ENABLE_HIP_RELOCATABLE_DEVICE_CODE) - GLOBAL_APPEND(KOKKOS_LINK_OPTIONS "${AMDGPU_ARCH_FLAG}=${FLAG}") - ENDIF() - ENDIF() - ENDIF() -ENDFUNCTION() +set(AMDGPU_ARCH_ALREADY_SPECIFIED "") +function(CHECK_AMDGPU_ARCH ARCH FLAG) + if(KOKKOS_ARCH_${ARCH}) + if(AMDGPU_ARCH_ALREADY_SPECIFIED) + message( + FATAL_ERROR + "Multiple GPU architectures given! Already have ${AMDGPU_ARCH_ALREADY_SPECIFIED}, but trying to add ${ARCH}. If you are re-running CMake, try clearing the cache and running again." + ) + endif() + set(AMDGPU_ARCH_ALREADY_SPECIFIED ${ARCH} PARENT_SCOPE) + if(NOT KOKKOS_ENABLE_HIP + AND NOT KOKKOS_ENABLE_OPENMPTARGET + AND NOT KOKKOS_ENABLE_OPENACC + AND NOT KOKKOS_ENABLE_SYCL + ) + message( + WARNING + "Given AMD GPU architecture ${ARCH}, but Kokkos_ENABLE_HIP, Kokkos_ENABLE_SYCL, Kokkos_ENABLE_OPENACC, and Kokkos_ENABLE_OPENMPTARGET are OFF. Option will be ignored." + ) + unset(KOKKOS_ARCH_${ARCH} PARENT_SCOPE) + else() + if(KOKKOS_ENABLE_HIP) + set(KOKKOS_HIP_ARCHITECTURES ${FLAG} PARENT_SCOPE) + endif() + if(NOT KOKKOS_IMPL_AMDGPU_FLAGS) + set(KOKKOS_AMDGPU_ARCH_FLAG ${FLAG} PARENT_SCOPE) + global_append(KOKKOS_AMDGPU_OPTIONS "${AMDGPU_ARCH_FLAG}=${FLAG}") + endif() + if(KOKKOS_ENABLE_HIP_RELOCATABLE_DEVICE_CODE) + global_append(KOKKOS_LINK_OPTIONS "${AMDGPU_ARCH_FLAG}=${FLAG}") + endif() + endif() + endif() +endfunction() #These will define KOKKOS_AMDGPU_ARCH_FLAG #to the corresponding flag name if ON -FOREACH(ARCH IN LISTS SUPPORTED_AMD_ARCHS) - LIST(FIND SUPPORTED_AMD_ARCHS ${ARCH} LIST_INDEX) - LIST(GET CORRESPONDING_AMD_FLAGS ${LIST_INDEX} FLAG) - CHECK_AMDGPU_ARCH(${ARCH} ${FLAG}) -ENDFOREACH() +foreach(ARCH IN LISTS SUPPORTED_AMD_ARCHS) + list(FIND SUPPORTED_AMD_ARCHS ${ARCH} LIST_INDEX) + list(GET CORRESPONDING_AMD_FLAGS ${LIST_INDEX} FLAG) + check_amdgpu_arch(${ARCH} ${FLAG}) +endforeach() -IF(KOKKOS_IMPL_AMDGPU_FLAGS) - IF (NOT AMDGPU_ARCH_ALREADY_SPECIFIED) - MESSAGE(FATAL_ERROR "When IMPL_AMDGPU_FLAGS is set the architecture autodectection is disabled. " - "Please explicitly set the GPU architecture.") - ENDIF() - GLOBAL_APPEND(KOKKOS_AMDGPU_OPTIONS "${KOKKOS_IMPL_AMDGPU_FLAGS}") - GLOBAL_APPEND(KOKKOS_LINK_OPTIONS "${KOKKOS_IMPL_AMDGPU_LINK}") -ENDIF() +if(KOKKOS_IMPL_AMDGPU_FLAGS) + if(NOT AMDGPU_ARCH_ALREADY_SPECIFIED) + message(FATAL_ERROR "When IMPL_AMDGPU_FLAGS is set the architecture autodectection is disabled. " + "Please explicitly set the GPU architecture." + ) + endif() + global_append(KOKKOS_AMDGPU_OPTIONS "${KOKKOS_IMPL_AMDGPU_FLAGS}") + global_append(KOKKOS_LINK_OPTIONS "${KOKKOS_IMPL_AMDGPU_LINK}") +endif() -MACRO(SET_AND_CHECK_AMD_ARCH ARCH FLAG) - KOKKOS_SET_OPTION(ARCH_${ARCH} ON) - CHECK_AMDGPU_ARCH(${ARCH} ${FLAG}) - LIST(APPEND KOKKOS_ENABLED_ARCH_LIST ${ARCH}) -ENDMACRO() +macro(SET_AND_CHECK_AMD_ARCH ARCH FLAG) + kokkos_set_option(ARCH_${ARCH} ON) + check_amdgpu_arch(${ARCH} ${FLAG}) + list(APPEND KOKKOS_ENABLED_ARCH_LIST ${ARCH}) +endmacro() -MACRO(CHECK_MULTIPLE_INTEL_ARCH) - IF(KOKKOS_ARCH_INTEL_GPU) - MESSAGE(FATAL_ERROR "Specifying multiple Intel GPU architectures is not allowed!") - ENDIF() - SET(KOKKOS_ARCH_INTEL_GPU ON) -ENDMACRO() +macro(CHECK_MULTIPLE_INTEL_ARCH) + if(KOKKOS_ARCH_INTEL_GPU) + message(FATAL_ERROR "Specifying multiple Intel GPU architectures is not allowed!") + endif() + set(KOKKOS_ARCH_INTEL_GPU ON) +endmacro() -IF(KOKKOS_ARCH_INTEL_GEN) - CHECK_MULTIPLE_INTEL_ARCH() -ENDIF() -IF(KOKKOS_ARCH_INTEL_DG1) - CHECK_MULTIPLE_INTEL_ARCH() -ENDIF() -IF(KOKKOS_ARCH_INTEL_GEN9) - CHECK_MULTIPLE_INTEL_ARCH() -ENDIF() -IF(KOKKOS_ARCH_INTEL_GEN11) - CHECK_MULTIPLE_INTEL_ARCH() -ENDIF() -IF(KOKKOS_ARCH_INTEL_GEN12LP) - CHECK_MULTIPLE_INTEL_ARCH() -ENDIF() -IF(KOKKOS_ARCH_INTEL_XEHP) - CHECK_MULTIPLE_INTEL_ARCH() -ENDIF() -IF(KOKKOS_ARCH_INTEL_PVC) - CHECK_MULTIPLE_INTEL_ARCH() -ENDIF() +if(KOKKOS_ARCH_INTEL_GEN) + check_multiple_intel_arch() +endif() +if(KOKKOS_ARCH_INTEL_DG1) + check_multiple_intel_arch() +endif() +if(KOKKOS_ARCH_INTEL_GEN9) + check_multiple_intel_arch() +endif() +if(KOKKOS_ARCH_INTEL_GEN11) + check_multiple_intel_arch() +endif() +if(KOKKOS_ARCH_INTEL_GEN12LP) + check_multiple_intel_arch() +endif() +if(KOKKOS_ARCH_INTEL_XEHP) + check_multiple_intel_arch() +endif() +if(KOKKOS_ARCH_INTEL_PVC) + check_multiple_intel_arch() +endif() -IF (KOKKOS_ENABLE_OPENMPTARGET) - SET(CLANG_CUDA_ARCH ${KOKKOS_CUDA_ARCH_FLAG}) - IF (CLANG_CUDA_ARCH) - IF(KOKKOS_CLANG_IS_CRAY) - COMPILER_SPECIFIC_FLAGS( - Cray -fopenmp +if(KOKKOS_ENABLE_OPENMP) + compiler_specific_link_options(CrayClang -fopenmp) +endif() + +if(KOKKOS_ENABLE_OPENMPTARGET) + set(CLANG_CUDA_ARCH ${KOKKOS_CUDA_ARCH_FLAG}) + if(CLANG_CUDA_ARCH) + if(KOKKOS_CLANG_IS_CRAY) + compiler_specific_flags(Cray -fopenmp) + else() + string(REPLACE "sm_" "cc" NVHPC_CUDA_ARCH ${CLANG_CUDA_ARCH}) + compiler_specific_flags( + Clang -Xopenmp-target -march=${CLANG_CUDA_ARCH} -fopenmp-targets=nvptx64 NVHPC -gpu=${NVHPC_CUDA_ARCH} ) - ELSE() - STRING(REPLACE "sm_" "cc" NVHPC_CUDA_ARCH ${CLANG_CUDA_ARCH}) - COMPILER_SPECIFIC_FLAGS( - Clang -Xopenmp-target -march=${CLANG_CUDA_ARCH} -fopenmp-targets=nvptx64 - NVHPC -gpu=${NVHPC_CUDA_ARCH} - ) - ENDIF() - ENDIF() - SET(CLANG_AMDGPU_ARCH ${KOKKOS_AMDGPU_ARCH_FLAG}) - IF (CLANG_AMDGPU_ARCH) - COMPILER_SPECIFIC_FLAGS( + endif() + endif() + set(CLANG_AMDGPU_ARCH ${KOKKOS_AMDGPU_ARCH_FLAG}) + if(CLANG_AMDGPU_ARCH) + compiler_specific_flags( Clang -Xopenmp-target=amdgcn-amd-amdhsa -march=${CLANG_AMDGPU_ARCH} -fopenmp-targets=amdgcn-amd-amdhsa ) - ENDIF() - IF (KOKKOS_ARCH_INTEL_GEN) - COMPILER_SPECIFIC_FLAGS( - IntelLLVM -fopenmp-targets=spir64 -D__STRICT_ANSI__ - ) - ELSE() - COMPILER_SPECIFIC_OPTIONS( - IntelLLVM -fopenmp-targets=spir64_gen -D__STRICT_ANSI__ - ) - IF(KOKKOS_ARCH_INTEL_GEN9) - COMPILER_SPECIFIC_LINK_OPTIONS( - IntelLLVM -fopenmp-targets=spir64_gen -Xopenmp-target-backend "-device gen9" - ) - ELSEIF(KOKKOS_ARCH_INTEL_GEN11) - COMPILER_SPECIFIC_LINK_OPTIONS( - IntelLLVM -fopenmp-targets=spir64_gen -Xopenmp-target-backend "-device gen11" - ) - ELSEIF(KOKKOS_ARCH_INTEL_GEN12LP) - COMPILER_SPECIFIC_LINK_OPTIONS( - IntelLLVM -fopenmp-targets=spir64_gen -Xopenmp-target-backend "-device gen12lp" - ) - ELSEIF(KOKKOS_ARCH_INTEL_DG1) - COMPILER_SPECIFIC_LINK_OPTIONS( - IntelLLVM -fopenmp-targets=spir64_gen -Xopenmp-target-backend "-device dg1" - ) - ELSEIF(KOKKOS_ARCH_INTEL_XEHP) - COMPILER_SPECIFIC_LINK_OPTIONS( - IntelLLVM -fopenmp-targets=spir64_gen -Xopenmp-target-backend "-device 12.50.4" - ) - ELSEIF(KOKKOS_ARCH_INTEL_PVC) - COMPILER_SPECIFIC_LINK_OPTIONS( - IntelLLVM -fopenmp-targets=spir64_gen -Xopenmp-target-backend "-device 12.60.7" - ) - ENDIF() - ENDIF() -ENDIF() + endif() + if(KOKKOS_ARCH_INTEL_GEN) + compiler_specific_flags(IntelLLVM -fopenmp-targets=spir64 -D__STRICT_ANSI__) + else() + compiler_specific_options(IntelLLVM -fopenmp-targets=spir64_gen -D__STRICT_ANSI__) + if(KOKKOS_ARCH_INTEL_GEN9) + compiler_specific_link_options(IntelLLVM -fopenmp-targets=spir64_gen -Xopenmp-target-backend "-device gen9") + elseif(KOKKOS_ARCH_INTEL_GEN11) + compiler_specific_link_options(IntelLLVM -fopenmp-targets=spir64_gen -Xopenmp-target-backend "-device gen11") + elseif(KOKKOS_ARCH_INTEL_GEN12LP) + compiler_specific_link_options(IntelLLVM -fopenmp-targets=spir64_gen -Xopenmp-target-backend "-device gen12lp") + elseif(KOKKOS_ARCH_INTEL_DG1) + compiler_specific_link_options(IntelLLVM -fopenmp-targets=spir64_gen -Xopenmp-target-backend "-device dg1") + elseif(KOKKOS_ARCH_INTEL_XEHP) + compiler_specific_link_options(IntelLLVM -fopenmp-targets=spir64_gen -Xopenmp-target-backend "-device 12.50.4") + elseif(KOKKOS_ARCH_INTEL_PVC) + compiler_specific_link_options(IntelLLVM -fopenmp-targets=spir64_gen -Xopenmp-target-backend "-device 12.60.7") + endif() + endif() +endif() -IF (KOKKOS_ENABLE_OPENACC) - IF(KOKKOS_CUDA_ARCH_FLAG) - SET(CLANG_CUDA_ARCH ${KOKKOS_CUDA_ARCH_FLAG}) - STRING(REPLACE "sm_" "cc" NVHPC_CUDA_ARCH ${KOKKOS_CUDA_ARCH_FLAG}) - COMPILER_SPECIFIC_FLAGS( - NVHPC -acc -gpu=${NVHPC_CUDA_ARCH} - Clang -Xopenmp-target=nvptx64-nvidia-cuda -march=${CLANG_CUDA_ARCH} - -fopenmp-targets=nvptx64-nvidia-cuda - ) - ELSEIF(KOKKOS_AMDGPU_ARCH_FLAG) - COMPILER_SPECIFIC_FLAGS( - Clang -Xopenmp-target=amdgcn-amd-amdhsa -march=${KOKKOS_AMDGPU_ARCH_FLAG} - -fopenmp-targets=amdgcn-amd-amdhsa - ) - ELSE() - COMPILER_SPECIFIC_FLAGS( - NVHPC -acc - ) - ENDIF() -ENDIF() - -IF (KOKKOS_ENABLE_SYCL) - IF(CUDA_ARCH_ALREADY_SPECIFIED) - IF(KOKKOS_ENABLE_UNSUPPORTED_ARCHS) - COMPILER_SPECIFIC_FLAGS( - DEFAULT -fsycl-targets=nvptx64-nvidia-cuda -Xsycl-target-backend=nvptx64-nvidia-cuda --cuda-gpu-arch=${KOKKOS_CUDA_ARCH_FLAG} +if(KOKKOS_ENABLE_OPENACC) + if(KOKKOS_CUDA_ARCH_FLAG) + if(KOKKOS_ENABLE_OPENACC_FORCE_HOST_AS_DEVICE) + message( + FATAL_ERROR + "If a GPU architecture is specified, Kokkos_ENABLE_OPENACC_FORCE_HOST_AS_DEVICE option cannot be used. Disable the Kokkos_ENABLE_OPENACC_FORCE_HOST_AS_DEVICE option." ) - ELSE() - MESSAGE(SEND_ERROR "Setting a CUDA architecture for SYCL is only allowed with Kokkos_ENABLE_UNSUPPORTED_ARCHS=ON!") - ENDIF() - ELSEIF(AMDGPU_ARCH_ALREADY_SPECIFIED) - IF(KOKKOS_ENABLE_UNSUPPORTED_ARCHS) - COMPILER_SPECIFIC_FLAGS( + endif() + set(CLANG_CUDA_ARCH ${KOKKOS_CUDA_ARCH_FLAG}) + string(REPLACE "sm_" "cc" NVHPC_CUDA_ARCH ${KOKKOS_CUDA_ARCH_FLAG}) + compiler_specific_flags( + NVHPC + -acc + -gpu=${NVHPC_CUDA_ARCH} + Clang + -Xopenmp-target=nvptx64-nvidia-cuda + -march=${CLANG_CUDA_ARCH} + -fopenmp-targets=nvptx64-nvidia-cuda + ) + if(DEFINED ENV{CUDA_PATH}) + compiler_specific_link_options(Clang -L$ENV{CUDA_PATH}/lib64) + endif() + compiler_specific_libs(Clang -lcudart NVHPC -cuda) + elseif(KOKKOS_AMDGPU_ARCH_FLAG) + if(KOKKOS_ENABLE_OPENACC_FORCE_HOST_AS_DEVICE) + message( + FATAL_ERROR + "If a GPU architecture is specified, Kokkos_ENABLE_OPENACC_FORCE_HOST_AS_DEVICE option cannot be used. Disable the Kokkos_ENABLE_OPENACC_FORCE_HOST_AS_DEVICE option." + ) + endif() + compiler_specific_flags( + Clang -Xopenmp-target=amdgcn-amd-amdhsa -march=${KOKKOS_AMDGPU_ARCH_FLAG} -fopenmp-targets=amdgcn-amd-amdhsa + ) + if(DEFINED ENV{ROCM_PATH}) + compiler_specific_flags(Clang -I$ENV{ROCM_PATH}/include) + compiler_specific_link_options(Clang -L$ENV{ROCM_PATH}/lib) + endif() + compiler_specific_libs(Clang -lamdhip64) + elseif(KOKKOS_ENABLE_OPENACC_FORCE_HOST_AS_DEVICE) + # Compile for kernel execution on the host. In that case, + # memory is shared between the OpenACC space and the host space. + compiler_specific_flags(NVHPC -acc=multicore) + else() + # Automatic fallback mode; try to offload any available GPU, and fall back + # to the host CPU if no available GPU is found. + compiler_specific_flags(NVHPC -acc=gpu,multicore) + message( + STATUS + "No OpenACC target device is specificed; the OpenACC backend will be executed in an automatic fallback mode." + ) + endif() +endif() + +if(KOKKOS_ENABLE_SYCL) + if(CUDA_ARCH_ALREADY_SPECIFIED) + if(KOKKOS_ENABLE_UNSUPPORTED_ARCHS) + compiler_specific_flags( + DEFAULT -fsycl-targets=nvptx64-nvidia-cuda -Xsycl-target-backend=nvptx64-nvidia-cuda + --cuda-gpu-arch=${KOKKOS_CUDA_ARCH_FLAG} + ) + else() + message( + SEND_ERROR "Setting a CUDA architecture for SYCL is only allowed with Kokkos_ENABLE_UNSUPPORTED_ARCHS=ON!" + ) + endif() + elseif(AMDGPU_ARCH_ALREADY_SPECIFIED) + if(KOKKOS_ENABLE_UNSUPPORTED_ARCHS) + compiler_specific_flags( DEFAULT -fsycl-targets=amdgcn-amd-amdhsa -Xsycl-target-backend --offload-arch=${KOKKOS_AMDGPU_ARCH_FLAG} ) - ELSE() - MESSAGE(SEND_ERROR "Setting a AMDGPU architecture for SYCL is only allowed with Kokkos_ENABLE_UNSUPPORTED_ARCHS=ON!") - ENDIF() - ELSEIF(KOKKOS_ARCH_INTEL_GEN) - COMPILER_SPECIFIC_FLAGS( - DEFAULT -fsycl-targets=spir64 - ) - ELSE() - COMPILER_SPECIFIC_OPTIONS( - DEFAULT -fsycl-targets=spir64_gen - ) - IF(KOKKOS_ARCH_INTEL_GEN9) - COMPILER_SPECIFIC_LINK_OPTIONS( - DEFAULT -fsycl-targets=spir64_gen -Xsycl-target-backend "-device gen9" + else() + message( + SEND_ERROR "Setting a AMDGPU architecture for SYCL is only allowed with Kokkos_ENABLE_UNSUPPORTED_ARCHS=ON!" ) - ELSEIF(KOKKOS_ARCH_INTEL_GEN11) - COMPILER_SPECIFIC_LINK_OPTIONS( - DEFAULT -fsycl-targets=spir64_gen -Xsycl-target-backend "-device gen11" - ) - ELSEIF(KOKKOS_ARCH_INTEL_GEN12LP) - COMPILER_SPECIFIC_LINK_OPTIONS( - DEFAULT -fsycl-targets=spir64_gen -Xsycl-target-backend "-device gen12lp" - ) - ELSEIF(KOKKOS_ARCH_INTEL_DG1) - COMPILER_SPECIFIC_LINK_OPTIONS( - DEFAULT -fsycl-targets=spir64_gen -Xsycl-target-backend "-device dg1" - ) - ELSEIF(KOKKOS_ARCH_INTEL_XEHP) - COMPILER_SPECIFIC_LINK_OPTIONS( - DEFAULT -fsycl-targets=spir64_gen -Xsycl-target-backend "-device 12.50.4" - ) - ELSEIF(KOKKOS_ARCH_INTEL_PVC) - COMPILER_SPECIFIC_LINK_OPTIONS( - DEFAULT -fsycl-targets=spir64_gen -Xsycl-target-backend "-device 12.60.7" - ) - ENDIF() - ENDIF() -ENDIF() + endif() + elseif(KOKKOS_ARCH_INTEL_GEN) + compiler_specific_flags(DEFAULT -fsycl-targets=spir64) + elseif(KOKKOS_ARCH_INTEL_GPU) + set(SYCL_TARGET_FLAG -fsycl-targets=spir64_gen) -IF(KOKKOS_ENABLE_CUDA AND NOT CUDA_ARCH_ALREADY_SPECIFIED) + if(KOKKOS_ARCH_INTEL_GEN9) + set(SYCL_TARGET_BACKEND_FLAG -Xsycl-target-backend "-device gen9") + elseif(KOKKOS_ARCH_INTEL_GEN11) + set(SYCL_TARGET_BACKEND_FLAG -Xsycl-target-backend "-device gen11") + elseif(KOKKOS_ARCH_INTEL_GEN12LP) + set(SYCL_TARGET_BACKEND_FLAG -Xsycl-target-backend "-device gen12lp") + elseif(KOKKOS_ARCH_INTEL_DG1) + set(SYCL_TARGET_BACKEND_FLAG -Xsycl-target-backend "-device dg1") + elseif(KOKKOS_ARCH_INTEL_XEHP) + set(SYCL_TARGET_BACKEND_FLAG -Xsycl-target-backend "-device 12.50.4") + elseif(KOKKOS_ARCH_INTEL_PVC) + set(SYCL_TARGET_BACKEND_FLAG -Xsycl-target-backend "-device 12.60.7") + endif() + + if(Kokkos_ENABLE_SYCL_RELOCATABLE_DEVICE_CODE) + compiler_specific_options(DEFAULT ${SYCL_TARGET_FLAG}) + compiler_specific_link_options(DEFAULT ${SYCL_TARGET_FLAG} ${SYCL_TARGET_BACKEND_FLAG}) + else() + compiler_specific_options(DEFAULT ${SYCL_TARGET_FLAG} ${SYCL_TARGET_BACKEND_FLAG}) + endif() + endif() +endif() + +if(KOKKOS_ENABLE_CUDA AND NOT CUDA_ARCH_ALREADY_SPECIFIED) # Try to autodetect the CUDA Compute Capability by asking the device - SET(_BINARY_TEST_DIR ${CMAKE_CURRENT_BINARY_DIR}/cmake/compile_tests/CUDAComputeCapabilityWorkdir) - FILE(REMOVE_RECURSE ${_BINARY_TEST_DIR}) - FILE(MAKE_DIRECTORY ${_BINARY_TEST_DIR}) + set(_BINARY_TEST_DIR ${CMAKE_CURRENT_BINARY_DIR}/cmake/compile_tests/CUDAComputeCapabilityWorkdir) + file(REMOVE_RECURSE ${_BINARY_TEST_DIR}) + file(MAKE_DIRECTORY ${_BINARY_TEST_DIR}) - TRY_RUN( - _RESULT - _COMPILE_RESULT - ${_BINARY_TEST_DIR} - ${CMAKE_CURRENT_SOURCE_DIR}/cmake/compile_tests/cuda_compute_capability.cc - COMPILE_DEFINITIONS -DSM_ONLY - RUN_OUTPUT_VARIABLE _CUDA_COMPUTE_CAPABILITY) + try_run(_RESULT _COMPILE_RESULT ${_BINARY_TEST_DIR} + ${CMAKE_CURRENT_SOURCE_DIR}/cmake/compile_tests/cuda_compute_capability.cc COMPILE_DEFINITIONS -DSM_ONLY + RUN_OUTPUT_VARIABLE _CUDA_COMPUTE_CAPABILITY + ) # if user is using kokkos_compiler_launcher, above will fail. - IF(NOT _COMPILE_RESULT OR NOT _RESULT EQUAL 0) + if(NOT _COMPILE_RESULT OR NOT _RESULT EQUAL 0) # check to see if CUDA is not already enabled (may happen when Kokkos is subproject) - GET_PROPERTY(_ENABLED_LANGUAGES GLOBAL PROPERTY ENABLED_LANGUAGES) + get_property(_ENABLED_LANGUAGES GLOBAL PROPERTY ENABLED_LANGUAGES) # language has to be fully enabled, just checking for CMAKE_CUDA_COMPILER isn't enough - IF(NOT "CUDA" IN_LIST _ENABLED_LANGUAGES) + if(NOT "CUDA" IN_LIST _ENABLED_LANGUAGES) # make sure the user knows that we aren't using CUDA compiler for anything else - MESSAGE(STATUS "CUDA auto-detection of architecture failed with ${CMAKE_CXX_COMPILER}. Enabling CUDA language ONLY to auto-detect architecture...") - INCLUDE(CheckLanguage) - CHECK_LANGUAGE(CUDA) - IF(CMAKE_CUDA_COMPILER) - ENABLE_LANGUAGE(CUDA) - ELSE() - MESSAGE(STATUS "CUDA language could not be enabled") - ENDIF() - ENDIF() + message( + STATUS + "CUDA auto-detection of architecture failed with ${CMAKE_CXX_COMPILER}. Enabling CUDA language ONLY to auto-detect architecture..." + ) + include(CheckLanguage) + check_language(CUDA) + if(CMAKE_CUDA_COMPILER) + enable_language(CUDA) + else() + message(STATUS "CUDA language could not be enabled") + endif() + endif() # if CUDA was enabled, this will be defined - IF(CMAKE_CUDA_COMPILER) + if(CMAKE_CUDA_COMPILER) # copy our test to .cu so cmake compiles as CUDA - CONFIGURE_FILE( + configure_file( ${CMAKE_CURRENT_SOURCE_DIR}/cmake/compile_tests/cuda_compute_capability.cc - ${CMAKE_CURRENT_BINARY_DIR}/compile_tests/cuda_compute_capability.cu - COPYONLY + ${CMAKE_CURRENT_BINARY_DIR}/compile_tests/cuda_compute_capability.cu COPYONLY ) # run test again - TRY_RUN( - _RESULT - _COMPILE_RESULT - ${_BINARY_TEST_DIR} - ${CMAKE_CURRENT_BINARY_DIR}/compile_tests/cuda_compute_capability.cu - COMPILE_DEFINITIONS -DSM_ONLY - RUN_OUTPUT_VARIABLE _CUDA_COMPUTE_CAPABILITY) - ENDIF() - ENDIF() + try_run(_RESULT _COMPILE_RESULT ${_BINARY_TEST_DIR} + ${CMAKE_CURRENT_BINARY_DIR}/compile_tests/cuda_compute_capability.cu COMPILE_DEFINITIONS -DSM_ONLY + RUN_OUTPUT_VARIABLE _CUDA_COMPUTE_CAPABILITY + ) + endif() + endif() - LIST(FIND KOKKOS_CUDA_ARCH_FLAGS sm_${_CUDA_COMPUTE_CAPABILITY} FLAG_INDEX) - IF(_COMPILE_RESULT AND _RESULT EQUAL 0 AND NOT FLAG_INDEX EQUAL -1) - MESSAGE(STATUS "Detected CUDA Compute Capability ${_CUDA_COMPUTE_CAPABILITY}") - LIST(GET KOKKOS_CUDA_ARCH_LIST ${FLAG_INDEX} ARCHITECTURE) - KOKKOS_SET_OPTION(ARCH_${ARCHITECTURE} ON) - CHECK_CUDA_ARCH(${ARCHITECTURE} sm_${_CUDA_COMPUTE_CAPABILITY}) - LIST(APPEND KOKKOS_ENABLED_ARCH_LIST ${ARCHITECTURE}) - ELSE() - MESSAGE(SEND_ERROR "CUDA enabled but no NVIDIA GPU architecture currently enabled and auto-detection failed. " - "Please give one -DKokkos_ARCH_{..}=ON' to enable an NVIDIA GPU architecture.\n" - "You can yourself try to compile ${CMAKE_CURRENT_SOURCE_DIR}/cmake/compile_tests/cuda_compute_capability.cc and run the executable. " - "If you are cross-compiling, you should try to do this on a compute node.") - ENDIF() -ENDIF() + list(FIND KOKKOS_CUDA_ARCH_FLAGS sm_${_CUDA_COMPUTE_CAPABILITY} FLAG_INDEX) + if(_COMPILE_RESULT AND _RESULT EQUAL 0 AND NOT FLAG_INDEX EQUAL -1) + message(STATUS "Detected CUDA Compute Capability ${_CUDA_COMPUTE_CAPABILITY}") + list(GET KOKKOS_CUDA_ARCH_LIST ${FLAG_INDEX} ARCHITECTURE) + kokkos_set_option(ARCH_${ARCHITECTURE} ON) + check_cuda_arch(${ARCHITECTURE} sm_${_CUDA_COMPUTE_CAPABILITY}) + list(APPEND KOKKOS_ENABLED_ARCH_LIST ${ARCHITECTURE}) + else() + message( + SEND_ERROR + "CUDA enabled but no NVIDIA GPU architecture currently enabled and auto-detection failed. " + "Please give one -DKokkos_ARCH_{..}=ON' to enable an NVIDIA GPU architecture.\n" + "You can yourself try to compile ${CMAKE_CURRENT_SOURCE_DIR}/cmake/compile_tests/cuda_compute_capability.cc and run the executable. " + "If you are cross-compiling, you should try to do this on a compute node." + ) + endif() +endif() #Regardless of version, make sure we define the general architecture name -IF (KOKKOS_ARCH_KEPLER30 OR KOKKOS_ARCH_KEPLER32 OR KOKKOS_ARCH_KEPLER35 OR KOKKOS_ARCH_KEPLER37) - SET(KOKKOS_ARCH_KEPLER ON) -ENDIF() +if(KOKKOS_ARCH_KEPLER30 + OR KOKKOS_ARCH_KEPLER32 + OR KOKKOS_ARCH_KEPLER35 + OR KOKKOS_ARCH_KEPLER37 +) + set(KOKKOS_ARCH_KEPLER ON) +endif() #Regardless of version, make sure we define the general architecture name -IF (KOKKOS_ARCH_MAXWELL50 OR KOKKOS_ARCH_MAXWELL52 OR KOKKOS_ARCH_MAXWELL53) - SET(KOKKOS_ARCH_MAXWELL ON) -ENDIF() +if(KOKKOS_ARCH_MAXWELL50 OR KOKKOS_ARCH_MAXWELL52 OR KOKKOS_ARCH_MAXWELL53) + set(KOKKOS_ARCH_MAXWELL ON) +endif() #Regardless of version, make sure we define the general architecture name -IF (KOKKOS_ARCH_PASCAL60 OR KOKKOS_ARCH_PASCAL61) - SET(KOKKOS_ARCH_PASCAL ON) -ENDIF() +if(KOKKOS_ARCH_PASCAL60 OR KOKKOS_ARCH_PASCAL61) + set(KOKKOS_ARCH_PASCAL ON) +endif() #Regardless of version, make sure we define the general architecture name -IF (KOKKOS_ARCH_VOLTA70 OR KOKKOS_ARCH_VOLTA72) - SET(KOKKOS_ARCH_VOLTA ON) -ENDIF() +if(KOKKOS_ARCH_VOLTA70 OR KOKKOS_ARCH_VOLTA72) + set(KOKKOS_ARCH_VOLTA ON) +endif() -IF (KOKKOS_ARCH_AMPERE80 OR KOKKOS_ARCH_AMPERE86) - SET(KOKKOS_ARCH_AMPERE ON) -ENDIF() +if(KOKKOS_ARCH_AMPERE80 OR KOKKOS_ARCH_AMPERE86) + set(KOKKOS_ARCH_AMPERE ON) +endif() -IF (KOKKOS_ARCH_HOPPER90) - SET(KOKKOS_ARCH_HOPPER ON) -ENDIF() +if(KOKKOS_ARCH_HOPPER90) + set(KOKKOS_ARCH_HOPPER ON) +endif() + +function(CHECK_AMD_APU ARCH) + set(BINARY_TEST_DIR ${CMAKE_CURRENT_BINARY_DIR}/cmake/compile_tests/AmdApuWorkdir) + file(REMOVE_RECURSE ${BINARY_TEST_DIR}) + file(MAKE_DIRECTORY ${BINARY_TEST_DIR}) + + try_run(RESULT COMPILE_RESULT ${BINARY_TEST_DIR} ${CMAKE_CURRENT_SOURCE_DIR}/cmake/compile_tests/amd_apu.cc + RUN_OUTPUT_VARIABLE AMD_APU + ) + + if(NOT COMPILE_RESULT OR NOT RESULT EQUAL 0) + message(SEND_ERROR "Autodetection of AMD APU failed." + "Please manually specify one AMD GPU architecture via -DKokkos_ARCH_{..}=ON'." + ) + endif() + + if(AMD_APU) + set(${ARCH} AMD_GFX942_APU PARENT_SCOPE) + endif() +endfunction() #HIP detection of gpu arch -IF(KOKKOS_ENABLE_HIP AND NOT AMDGPU_ARCH_ALREADY_SPECIFIED AND NOT KOKKOS_IMPL_AMDGPU_FLAGS) - FIND_PROGRAM(ROCM_ENUMERATOR rocm_agent_enumerator) - IF(NOT ROCM_ENUMERATOR) - MESSAGE(FATAL_ERROR "Autodetection of AMD GPU architecture not possible as " - "rocm_agent_enumerator could not be found. " - "Please specify an arch manually via -DKokkos_ARCH_{..}=ON") - ELSE() - EXECUTE_PROCESS(COMMAND ${ROCM_ENUMERATOR} OUTPUT_VARIABLE GPU_ARCHS) - STRING(LENGTH "${GPU_ARCHS}" len_str) +if(KOKKOS_ENABLE_HIP AND NOT AMDGPU_ARCH_ALREADY_SPECIFIED AND NOT KOKKOS_IMPL_AMDGPU_FLAGS) + find_program(ROCM_ENUMERATOR rocm_agent_enumerator) + if(NOT ROCM_ENUMERATOR) + message( + FATAL_ERROR "Autodetection of AMD GPU architecture not possible as " "rocm_agent_enumerator could not be found. " + "Please specify an arch manually via -DKokkos_ARCH_{..}=ON" + ) + else() + execute_process(COMMAND ${ROCM_ENUMERATOR} OUTPUT_VARIABLE GPU_ARCHS) + string(LENGTH "${GPU_ARCHS}" len_str) # enumerator always output gfx000 as the first line - IF(${len_str} LESS 8) - MESSAGE(SEND_ERROR "HIP enabled but no AMD GPU architecture could be automatically detected. " - "Please manually specify one AMD GPU architecture via -DKokkos_ARCH_{..}=ON'.") - # check for known gpu archs, otherwise error out - ELSE() - SET(AMD_ARCH_DETECTED "") - FOREACH(ARCH IN LISTS SUPPORTED_AMD_ARCHS) - LIST(FIND SUPPORTED_AMD_ARCHS ${ARCH} LIST_INDEX) - LIST(GET CORRESPONDING_AMD_FLAGS ${LIST_INDEX} FLAG) - STRING(REGEX MATCH "(${FLAG})" DETECTED_GPU_ARCH ${GPU_ARCHS}) - IF("${DETECTED_GPU_ARCH}" STREQUAL "${FLAG}") - SET_AND_CHECK_AMD_ARCH(${ARCH} ${FLAG}) - SET(AMD_ARCH_DETECTED ${ARCH}) - BREAK() - ENDIF() - ENDFOREACH() - IF("${AMD_ARCH_DETECTED}" STREQUAL "") - MESSAGE(FATAL_ERROR "HIP enabled but no automatically detected AMD GPU architecture " - "is supported. " - "Please manually specify one AMD GPU architecture via -DKokkos_ARCH_{..}=ON'.") - ENDIF() - ENDIF() - ENDIF() -ENDIF() + if(${len_str} LESS 8) + message(SEND_ERROR "HIP enabled but no AMD GPU architecture could be automatically detected. " + "Please manually specify one AMD GPU architecture via -DKokkos_ARCH_{..}=ON'." + ) + # check for known gpu archs, otherwise error out + else() + set(AMD_ARCH_DETECTED "") + foreach(ARCH IN LISTS SUPPORTED_AMD_ARCHS) + list(FIND SUPPORTED_AMD_ARCHS ${ARCH} LIST_INDEX) + list(GET CORRESPONDING_AMD_FLAGS ${LIST_INDEX} FLAG) + string(REGEX MATCH "(${FLAG})" DETECTED_GPU_ARCH ${GPU_ARCHS}) + if("${DETECTED_GPU_ARCH}" STREQUAL "${FLAG}") + # If we detected gfx942, we need to discriminate between APU and discrete GPU + if(FLAG STREQUAL "gfx942") + check_amd_apu(ARCH) + endif() + set_and_check_amd_arch(${ARCH} ${FLAG}) + set(AMD_ARCH_DETECTED ${ARCH}) + break() + endif() + endforeach() + if("${AMD_ARCH_DETECTED}" STREQUAL "") + message(FATAL_ERROR "HIP enabled but no automatically detected AMD GPU architecture " "is supported. " + "Please manually specify one AMD GPU architecture via -DKokkos_ARCH_{..}=ON'." + ) + endif() + endif() + endif() +endif() -FOREACH(ARCH IN LISTS SUPPORTED_AMD_ARCHS) - IF (KOKKOS_ARCH_${ARCH}) - STRING(REGEX MATCH "90A" IS_90A ${ARCH}) - IF(IS_90A) - SET(KOKKOS_ARCH_AMD_GFX90A ON) - SET(KOKKOS_ARCH_VEGA90A ON) - BREAK() - ENDIF() - STRING(REGEX MATCH "908" IS_908 ${ARCH}) - IF(IS_908) - SET(KOKKOS_ARCH_AMD_GFX908 ON) - SET(KOKKOS_ARCH_VEGA908 ON) - BREAK() - ENDIF() - STRING(REGEX MATCH "906" IS_906 ${ARCH}) - IF(IS_906) - SET(KOKKOS_ARCH_AMD_GFX906 ON) - SET(KOKKOS_ARCH_VEGA906 ON) - BREAK() - ENDIF() - STRING(REGEX MATCH "1100" IS_1100 ${ARCH}) - IF(IS_1100) - SET(KOKKOS_ARCH_AMD_GFX1100 ON) - SET(KOKKOS_ARCH_NAVI1100 ON) - BREAK() - ENDIF() - STRING(REGEX MATCH "1030" IS_1030 ${ARCH}) - IF(IS_1030) - SET(KOKKOS_ARCH_AMD_GFX1030 ON) - SET(KOKKOS_ARCH_NAVI1030 ON) - BREAK() - ENDIF() - ENDIF() -ENDFOREACH() +foreach(ARCH IN LISTS SUPPORTED_AMD_ARCHS) + if(KOKKOS_ARCH_${ARCH}) + string(REGEX MATCH "90A" IS_90A ${ARCH}) + if(IS_90A) + set(KOKKOS_ARCH_AMD_GFX90A ON) + set(KOKKOS_ARCH_VEGA90A ON) + break() + endif() + string(REGEX MATCH "908" IS_908 ${ARCH}) + if(IS_908) + set(KOKKOS_ARCH_AMD_GFX908 ON) + set(KOKKOS_ARCH_VEGA908 ON) + break() + endif() + string(REGEX MATCH "906" IS_906 ${ARCH}) + if(IS_906) + set(KOKKOS_ARCH_AMD_GFX906 ON) + set(KOKKOS_ARCH_VEGA906 ON) + break() + endif() + string(REGEX MATCH "1100" IS_1100 ${ARCH}) + if(IS_1100) + set(KOKKOS_ARCH_AMD_GFX1100 ON) + set(KOKKOS_ARCH_NAVI1100 ON) + break() + endif() + string(REGEX MATCH "1030" IS_1030 ${ARCH}) + if(IS_1030) + set(KOKKOS_ARCH_AMD_GFX1030 ON) + set(KOKKOS_ARCH_NAVI1030 ON) + break() + endif() + endif() +endforeach() #Regardless of version, make sure we define the general architecture name -FOREACH(ARCH IN LISTS SUPPORTED_AMD_ARCHS) - IF (KOKKOS_ARCH_${ARCH}) - SET(KOKKOS_ARCH_AMD_GPU ON) - STRING(REGEX MATCH "(VEGA)" IS_VEGA ${ARCH}) - IF(IS_VEGA) - SET(KOKKOS_ARCH_VEGA ON) - BREAK() - ENDIF() - STRING(REGEX MATCH "(NAVI)" IS_NAVI ${ARCH}) - IF(IS_NAVI) - SET(KOKKOS_ARCH_NAVI ON) - BREAK() - ENDIF() - ENDIF() -ENDFOREACH() +foreach(ARCH IN LISTS SUPPORTED_AMD_ARCHS) + if(KOKKOS_ARCH_${ARCH}) + list(FIND SUPPORTED_AMD_ARCHS ${ARCH} LIST_INDEX) + list(GET CORRESPONDING_AMD_FLAGS ${LIST_INDEX} FLAG) + set(KOKKOS_ARCH_AMD_GPU "${FLAG}") + string(REGEX MATCH "(VEGA)" IS_VEGA ${ARCH}) + if(IS_VEGA) + set(KOKKOS_ARCH_VEGA ON) + break() + endif() + string(REGEX MATCH "(NAVI)" IS_NAVI ${ARCH}) + if(IS_NAVI) + set(KOKKOS_ARCH_NAVI ON) + break() + endif() + endif() +endforeach() #CMake verbose is kind of pointless #Let's just always print things -MESSAGE(STATUS "Built-in Execution Spaces:") +message(STATUS "Built-in Execution Spaces:") -FOREACH (_BACKEND Cuda OpenMPTarget HIP SYCL OpenACC) - STRING(TOUPPER ${_BACKEND} UC_BACKEND) - IF(KOKKOS_ENABLE_${UC_BACKEND}) - IF(_DEVICE_PARALLEL) - MESSAGE(FATAL_ERROR "Multiple device parallel execution spaces are not allowed! " - "Trying to enable execution space ${_BACKEND}, " - "but execution space ${_DEVICE_PARALLEL} is already enabled. " - "Remove the CMakeCache.txt file and re-configure.") - ENDIF() - IF (${_BACKEND} STREQUAL "Cuda") - IF(KOKKOS_ENABLE_CUDA_UVM) - MESSAGE(DEPRECATION "Setting Kokkos_ENABLE_CUDA_UVM is deprecated - use the portable Kokkos::SharedSpace as an explicit memory space in your code instead") - IF(KOKKOS_ENABLE_DEPRECATED_CODE_4) - SET(_DEFAULT_DEVICE_MEMSPACE "Kokkos::${_BACKEND}UVMSpace") - ELSE() - MESSAGE(FATAL_ERROR "Kokkos_ENABLE_DEPRECATED_CODE_4 must be set to use Kokkos_ENABLE_CUDA_UVM") - ENDIF() - ELSE() - SET(_DEFAULT_DEVICE_MEMSPACE "Kokkos::${_BACKEND}Space") - ENDIF() - SET(_DEVICE_PARALLEL "Kokkos::${_BACKEND}") - ELSEIF(${_BACKEND} STREQUAL "HIP") - SET(_DEFAULT_DEVICE_MEMSPACE "Kokkos::${_BACKEND}Space") - SET(_DEVICE_PARALLEL "Kokkos::${_BACKEND}") - ELSE() - SET(_DEFAULT_DEVICE_MEMSPACE "Kokkos::Experimental::${_BACKEND}Space") - SET(_DEVICE_PARALLEL "Kokkos::Experimental::${_BACKEND}") - ENDIF() - ENDIF() -ENDFOREACH() -IF(NOT _DEVICE_PARALLEL) - SET(_DEVICE_PARALLEL "NoTypeDefined") - SET(_DEFAULT_DEVICE_MEMSPACE "NoTypeDefined") -ENDIF() -MESSAGE(STATUS " Device Parallel: ${_DEVICE_PARALLEL}") +foreach(_BACKEND Cuda OpenMPTarget HIP SYCL OpenACC) + string(TOUPPER ${_BACKEND} UC_BACKEND) + if(KOKKOS_ENABLE_${UC_BACKEND}) + if(_DEVICE_PARALLEL) + message( + FATAL_ERROR + "Multiple device parallel execution spaces are not allowed! " + "Trying to enable execution space ${_BACKEND}, " + "but execution space ${_DEVICE_PARALLEL} is already enabled. " + "Remove the CMakeCache.txt file and re-configure." + ) + endif() + if(${_BACKEND} STREQUAL "Cuda") + if(KOKKOS_ENABLE_CUDA_UVM) + message( + DEPRECATION + "Setting Kokkos_ENABLE_CUDA_UVM is deprecated - use the portable Kokkos::SharedSpace as an explicit memory space in your code instead" + ) + if(NOT KOKKOS_ENABLE_DEPRECATED_CODE_4) + message(FATAL_ERROR "Kokkos_ENABLE_DEPRECATED_CODE_4 must be set to use Kokkos_ENABLE_CUDA_UVM") + endif() + endif() + set(_DEVICE_PARALLEL "Kokkos::${_BACKEND}") + elseif(${_BACKEND} STREQUAL "HIP" OR ${_BACKEND} STREQUAL "SYCL") + set(_DEVICE_PARALLEL "Kokkos::${_BACKEND}") + else() + set(_DEVICE_PARALLEL "Kokkos::Experimental::${_BACKEND}") + endif() + endif() +endforeach() +if(NOT _DEVICE_PARALLEL) + set(_DEVICE_PARALLEL "NoTypeDefined") +endif() +message(STATUS " Device Parallel: ${_DEVICE_PARALLEL}") -FOREACH (_BACKEND OpenMP Threads HPX) - STRING(TOUPPER ${_BACKEND} UC_BACKEND) - IF(KOKKOS_ENABLE_${UC_BACKEND}) - IF(_HOST_PARALLEL) - MESSAGE(FATAL_ERROR "Multiple host parallel execution spaces are not allowed! " - "Trying to enable execution space ${_BACKEND}, " - "but execution space ${_HOST_PARALLEL} is already enabled. " - "Remove the CMakeCache.txt file and re-configure.") - ENDIF() - IF (${_BACKEND} STREQUAL "HPX") - SET(_HOST_PARALLEL "Kokkos::Experimental::${_BACKEND}") - ELSE() - SET(_HOST_PARALLEL "Kokkos::${_BACKEND}") - ENDIF() - ENDIF() -ENDFOREACH() +foreach(_BACKEND OpenMP Threads HPX) + string(TOUPPER ${_BACKEND} UC_BACKEND) + if(KOKKOS_ENABLE_${UC_BACKEND}) + if(_HOST_PARALLEL) + message( + FATAL_ERROR + "Multiple host parallel execution spaces are not allowed! " "Trying to enable execution space ${_BACKEND}, " + "but execution space ${_HOST_PARALLEL} is already enabled. " + "Remove the CMakeCache.txt file and re-configure." + ) + endif() + if(${_BACKEND} STREQUAL "HPX") + set(_HOST_PARALLEL "Kokkos::Experimental::${_BACKEND}") + else() + set(_HOST_PARALLEL "Kokkos::${_BACKEND}") + endif() + endif() +endforeach() -IF(NOT _HOST_PARALLEL AND NOT KOKKOS_ENABLE_SERIAL) - MESSAGE(FATAL_ERROR "At least one host execution space must be enabled, " - "but no host parallel execution space was requested " - "and Kokkos_ENABLE_SERIAL=OFF.") -ENDIF() +if(NOT _HOST_PARALLEL AND NOT KOKKOS_ENABLE_SERIAL) + message(FATAL_ERROR "At least one host execution space must be enabled, " + "but no host parallel execution space was requested " "and Kokkos_ENABLE_SERIAL=OFF." + ) +endif() -IF(_HOST_PARALLEL) -MESSAGE(STATUS " Host Parallel: ${_HOST_PARALLEL}") -ELSE() - SET(_HOST_PARALLEL "NoTypeDefined") - MESSAGE(STATUS " Host Parallel: NoTypeDefined") -ENDIF() +if(_HOST_PARALLEL) + message(STATUS " Host Parallel: ${_HOST_PARALLEL}") +else() + set(_HOST_PARALLEL "NoTypeDefined") + message(STATUS " Host Parallel: NoTypeDefined") +endif() -IF(KOKKOS_ENABLE_SERIAL) - MESSAGE(STATUS " Host Serial: SERIAL") -ELSE() - MESSAGE(STATUS " Host Serial: NONE") -ENDIF() +if(KOKKOS_ENABLE_SERIAL) + message(STATUS " Host Serial: SERIAL") +else() + message(STATUS " Host Serial: NONE") +endif() -MESSAGE(STATUS "") -MESSAGE(STATUS "Architectures:") -FOREACH(Arch ${KOKKOS_ENABLED_ARCH_LIST}) - MESSAGE(STATUS " ${Arch}") -ENDFOREACH() +message(STATUS "") +message(STATUS "Architectures:") +foreach(Arch ${KOKKOS_ENABLED_ARCH_LIST}) + message(STATUS " ${Arch}") +endforeach() - -IF(KOKKOS_ENABLE_ATOMICS_BYPASS) - IF(NOT _HOST_PARALLEL STREQUAL "NoTypeDefined" OR NOT _DEVICE_PARALLEL STREQUAL "NoTypeDefined") - MESSAGE(FATAL_ERROR "Not allowed to disable atomics (via -DKokkos_ENABLE_AROMICS_BYPASS=ON) if neither a host parallel nor a device backend is enabled!") - ENDIF() - IF(NOT KOKKOS_ENABLE_SERIAL) - MESSAGE(FATAL_ERROR "Implementation bug") # safeguard - ENDIF() - MESSAGE(STATUS "Atomics: **DISABLED**") -ENDIF() +if(KOKKOS_ENABLE_ATOMICS_BYPASS) + if(NOT _HOST_PARALLEL STREQUAL "NoTypeDefined" OR NOT _DEVICE_PARALLEL STREQUAL "NoTypeDefined") + message( + FATAL_ERROR + "Disabling atomics (via -DKokkos_ENABLE_ATOMICS_BYPASS=ON) is not allowed if a host parallel or a device backend is enabled!" + ) + endif() + if(NOT KOKKOS_ENABLE_SERIAL) + message(FATAL_ERROR "Implementation bug") # safeguard + endif() + message(STATUS "Atomics: **DISABLED**") +endif() diff --git a/lib/kokkos/cmake/kokkos_check_env.cmake b/lib/kokkos/cmake/kokkos_check_env.cmake index a455a403b9..f1a309ff85 100644 --- a/lib/kokkos/cmake/kokkos_check_env.cmake +++ b/lib/kokkos/cmake/kokkos_check_env.cmake @@ -1,12 +1,15 @@ -SET(CRAYPE_VERSION $ENV{CRAYPE_VERSION}) -IF (CRAYPE_VERSION) - SET(KOKKOS_IS_CRAYPE TRUE) - SET(CRAYPE_LINK_TYPE $ENV{CRAYPE_LINK_TYPE}) - IF (CRAYPE_LINK_TYPE) - IF (NOT CRAYPE_LINK_TYPE STREQUAL "dynamic") - MESSAGE(WARNING "CRAYPE_LINK_TYPE is set to ${CRAYPE_LINK_TYPE}. Linking is likely to fail unless this is set to 'dynamic'") - ENDIF() - ELSE() - MESSAGE(WARNING "CRAYPE_LINK_TYPE is not set. Linking is likely to fail unless this is set to 'dynamic'") - ENDIF() -ENDIF() +set(CRAYPE_VERSION $ENV{CRAYPE_VERSION}) +if(CRAYPE_VERSION) + set(KOKKOS_IS_CRAYPE TRUE) + set(CRAYPE_LINK_TYPE $ENV{CRAYPE_LINK_TYPE}) + if(CRAYPE_LINK_TYPE) + if(NOT CRAYPE_LINK_TYPE STREQUAL "dynamic") + message( + WARNING + "CRAYPE_LINK_TYPE is set to ${CRAYPE_LINK_TYPE}. Linking is likely to fail unless this is set to 'dynamic'" + ) + endif() + else() + message(WARNING "CRAYPE_LINK_TYPE is not set. Linking is likely to fail unless this is set to 'dynamic'") + endif() +endif() diff --git a/lib/kokkos/cmake/kokkos_compiler_id.cmake b/lib/kokkos/cmake/kokkos_compiler_id.cmake index e8bfadb64e..010ed33ede 100644 --- a/lib/kokkos/cmake/kokkos_compiler_id.cmake +++ b/lib/kokkos/cmake/kokkos_compiler_id.cmake @@ -1,262 +1,273 @@ -KOKKOS_CFG_DEPENDS(COMPILER_ID NONE) +kokkos_cfg_depends(COMPILER_ID NONE) -SET(KOKKOS_CXX_COMPILER ${CMAKE_CXX_COMPILER}) -SET(KOKKOS_CXX_COMPILER_ID ${CMAKE_CXX_COMPILER_ID}) -SET(KOKKOS_CXX_COMPILER_VERSION ${CMAKE_CXX_COMPILER_VERSION}) +set(KOKKOS_CXX_COMPILER ${CMAKE_CXX_COMPILER}) +set(KOKKOS_CXX_COMPILER_ID ${CMAKE_CXX_COMPILER_ID}) +set(KOKKOS_CXX_COMPILER_VERSION ${CMAKE_CXX_COMPILER_VERSION}) -MACRO(kokkos_internal_have_compiler_nvcc) +macro(kokkos_internal_have_compiler_nvcc) # Check if the compiler is nvcc (which really means nvcc_wrapper). - EXECUTE_PROCESS(COMMAND ${ARGN} --version - OUTPUT_VARIABLE INTERNAL_COMPILER_VERSION - OUTPUT_STRIP_TRAILING_WHITESPACE) - STRING(REPLACE "\n" " - " INTERNAL_COMPILER_VERSION_ONE_LINE ${INTERNAL_COMPILER_VERSION} ) - STRING(FIND ${INTERNAL_COMPILER_VERSION_ONE_LINE} "nvcc" INTERNAL_COMPILER_VERSION_CONTAINS_NVCC) - STRING(REGEX REPLACE "^ +" "" INTERNAL_HAVE_COMPILER_NVCC "${INTERNAL_HAVE_COMPILER_NVCC}") - IF(${INTERNAL_COMPILER_VERSION_CONTAINS_NVCC} GREATER -1) - SET(INTERNAL_HAVE_COMPILER_NVCC true) - ELSE() - SET(INTERNAL_HAVE_COMPILER_NVCC false) - ENDIF() -ENDMACRO() + execute_process(COMMAND ${ARGN} --version OUTPUT_VARIABLE INTERNAL_COMPILER_VERSION OUTPUT_STRIP_TRAILING_WHITESPACE) + string(REPLACE "\n" " - " INTERNAL_COMPILER_VERSION_ONE_LINE ${INTERNAL_COMPILER_VERSION}) + string(FIND ${INTERNAL_COMPILER_VERSION_ONE_LINE} "nvcc" INTERNAL_COMPILER_VERSION_CONTAINS_NVCC) + string(REGEX REPLACE "^ +" "" INTERNAL_HAVE_COMPILER_NVCC "${INTERNAL_HAVE_COMPILER_NVCC}") + if(${INTERNAL_COMPILER_VERSION_CONTAINS_NVCC} GREATER -1) + set(INTERNAL_HAVE_COMPILER_NVCC true) + else() + set(INTERNAL_HAVE_COMPILER_NVCC false) + endif() +endmacro() -IF(Kokkos_ENABLE_CUDA) +if(Kokkos_ENABLE_CUDA) # kokkos_enable_options is not yet called so use lower case here - IF(Kokkos_ENABLE_COMPILE_AS_CMAKE_LANGUAGE) + if(Kokkos_ENABLE_COMPILE_AS_CMAKE_LANGUAGE) kokkos_internal_have_compiler_nvcc(${CMAKE_CUDA_COMPILER}) - ELSE() + else() # find kokkos_launch_compiler - FIND_PROGRAM(Kokkos_COMPILE_LAUNCHER - NAMES kokkos_launch_compiler - HINTS ${PROJECT_SOURCE_DIR} - PATHS ${PROJECT_SOURCE_DIR} - PATH_SUFFIXES bin) + find_program( + Kokkos_COMPILE_LAUNCHER + NAMES kokkos_launch_compiler + HINTS ${PROJECT_SOURCE_DIR} + PATHS ${PROJECT_SOURCE_DIR} + PATH_SUFFIXES bin + ) - FIND_PROGRAM(Kokkos_NVCC_WRAPPER - NAMES nvcc_wrapper - HINTS ${PROJECT_SOURCE_DIR} - PATHS ${PROJECT_SOURCE_DIR} - PATH_SUFFIXES bin) + find_program( + Kokkos_NVCC_WRAPPER + NAMES nvcc_wrapper + HINTS ${PROJECT_SOURCE_DIR} + PATHS ${PROJECT_SOURCE_DIR} + PATH_SUFFIXES bin + ) # Check if compiler was set to nvcc_wrapper kokkos_internal_have_compiler_nvcc(${CMAKE_CXX_COMPILER}) # If launcher was found and nvcc_wrapper was not specified as # compiler and `CMAKE_CXX_COMPILIER_LAUNCHER` is not set, set to use launcher. # Will ensure CMAKE_CXX_COMPILER is replaced by nvcc_wrapper - IF(Kokkos_COMPILE_LAUNCHER AND NOT INTERNAL_HAVE_COMPILER_NVCC AND NOT KOKKOS_CXX_COMPILER_ID STREQUAL Clang) - IF(CMAKE_CXX_COMPILER_LAUNCHER) - MESSAGE(FATAL_ERROR "Cannot use CMAKE_CXX_COMPILER_LAUNCHER if the CMAKE_CXX_COMPILER is not able to compile CUDA code, i.e. nvcc_wrapper or clang++!") - ENDIF() + if(Kokkos_COMPILE_LAUNCHER AND NOT INTERNAL_HAVE_COMPILER_NVCC AND NOT KOKKOS_CXX_COMPILER_ID STREQUAL Clang) + if(CMAKE_CXX_COMPILER_LAUNCHER) + message( + FATAL_ERROR + "Cannot use CMAKE_CXX_COMPILER_LAUNCHER if the CMAKE_CXX_COMPILER is not able to compile CUDA code, i.e. nvcc_wrapper or clang++!" + ) + endif() # the first argument to launcher is always the C++ compiler defined by cmake # if the second argument matches the C++ compiler, it forwards the rest of the # args to nvcc_wrapper kokkos_internal_have_compiler_nvcc( - ${Kokkos_COMPILE_LAUNCHER} ${Kokkos_NVCC_WRAPPER} ${CMAKE_CXX_COMPILER} ${CMAKE_CXX_COMPILER} -DKOKKOS_DEPENDENCE) - SET(INTERNAL_USE_COMPILER_LAUNCHER true) - ENDIF() - ENDIF() -ENDIF() + ${Kokkos_COMPILE_LAUNCHER} ${Kokkos_NVCC_WRAPPER} ${CMAKE_CXX_COMPILER} ${CMAKE_CXX_COMPILER} + -DKOKKOS_DEPENDENCE + ) + set(INTERNAL_USE_COMPILER_LAUNCHER true) + endif() + endif() +endif() -IF(INTERNAL_HAVE_COMPILER_NVCC) +if(INTERNAL_HAVE_COMPILER_NVCC) # Save the host compiler id before overwriting it. - SET(KOKKOS_CXX_HOST_COMPILER_ID ${KOKKOS_CXX_COMPILER_ID}) + set(KOKKOS_CXX_HOST_COMPILER_ID ${KOKKOS_CXX_COMPILER_ID}) # SET the compiler id to nvcc. We use the value used by CMake 3.8. - SET(KOKKOS_CXX_COMPILER_ID NVIDIA CACHE STRING INTERNAL FORCE) + set(KOKKOS_CXX_COMPILER_ID NVIDIA CACHE STRING INTERNAL FORCE) - STRING(REGEX MATCH "V[0-9]+\\.[0-9]+\\.[0-9]+" - TEMP_CXX_COMPILER_VERSION ${INTERNAL_COMPILER_VERSION_ONE_LINE}) - STRING(SUBSTRING ${TEMP_CXX_COMPILER_VERSION} 1 -1 TEMP_CXX_COMPILER_VERSION) - SET(KOKKOS_CXX_COMPILER_VERSION ${TEMP_CXX_COMPILER_VERSION} CACHE STRING INTERNAL FORCE) - MESSAGE(STATUS "Compiler Version: ${KOKKOS_CXX_COMPILER_VERSION}") - IF(INTERNAL_USE_COMPILER_LAUNCHER) - MESSAGE(STATUS "kokkos_launch_compiler (${Kokkos_COMPILE_LAUNCHER}) is enabled...") + string(REGEX MATCH "V[0-9]+\\.[0-9]+\\.[0-9]+" TEMP_CXX_COMPILER_VERSION ${INTERNAL_COMPILER_VERSION_ONE_LINE}) + string(SUBSTRING ${TEMP_CXX_COMPILER_VERSION} 1 -1 TEMP_CXX_COMPILER_VERSION) + set(KOKKOS_CXX_COMPILER_VERSION ${TEMP_CXX_COMPILER_VERSION} CACHE STRING INTERNAL FORCE) + message(STATUS "Compiler Version: ${KOKKOS_CXX_COMPILER_VERSION}") + if(INTERNAL_USE_COMPILER_LAUNCHER) + message(STATUS "kokkos_launch_compiler (${Kokkos_COMPILE_LAUNCHER}) is enabled...") kokkos_compilation(GLOBAL) - ENDIF() -ENDIF() + endif() +endif() -IF(Kokkos_ENABLE_HIP) +if(Kokkos_ENABLE_HIP) # get HIP version - EXECUTE_PROCESS(COMMAND ${CMAKE_CXX_COMPILER} --version - OUTPUT_VARIABLE INTERNAL_COMPILER_VERSION - OUTPUT_STRIP_TRAILING_WHITESPACE) + execute_process( + COMMAND ${CMAKE_CXX_COMPILER} --version OUTPUT_VARIABLE INTERNAL_COMPILER_VERSION OUTPUT_STRIP_TRAILING_WHITESPACE + ) - STRING(REPLACE "\n" " - " INTERNAL_COMPILER_VERSION_ONE_LINE ${INTERNAL_COMPILER_VERSION} ) + string(REPLACE "\n" " - " INTERNAL_COMPILER_VERSION_ONE_LINE ${INTERNAL_COMPILER_VERSION}) - STRING(FIND ${INTERNAL_COMPILER_VERSION_ONE_LINE} "HIP version" INTERNAL_COMPILER_VERSION_CONTAINS_HIP) - IF(INTERNAL_COMPILER_VERSION_CONTAINS_HIP GREATER -1) - SET(KOKKOS_CXX_COMPILER_ID HIPCC CACHE STRING INTERNAL FORCE) - ENDIF() + string(FIND ${INTERNAL_COMPILER_VERSION_ONE_LINE} "HIP version" INTERNAL_COMPILER_VERSION_CONTAINS_HIP) + if(INTERNAL_COMPILER_VERSION_CONTAINS_HIP GREATER -1) + set(KOKKOS_CXX_COMPILER_ID HIPCC CACHE STRING INTERNAL FORCE) + endif() - STRING(REGEX MATCH "[0-9]+\\.[0-9]+\\.[0-9]+" - TEMP_CXX_COMPILER_VERSION ${INTERNAL_COMPILER_VERSION_ONE_LINE}) - SET(KOKKOS_CXX_COMPILER_VERSION ${TEMP_CXX_COMPILER_VERSION} CACHE STRING INTERNAL FORCE) - MESSAGE(STATUS "Compiler Version: ${KOKKOS_CXX_COMPILER_VERSION}") -ENDIF() + string(REGEX MATCH "[0-9]+\\.[0-9]+\\.[0-9]+" TEMP_CXX_COMPILER_VERSION ${INTERNAL_COMPILER_VERSION_ONE_LINE}) + set(KOKKOS_CXX_COMPILER_VERSION ${TEMP_CXX_COMPILER_VERSION} CACHE STRING INTERNAL FORCE) + message(STATUS "Compiler Version: ${KOKKOS_CXX_COMPILER_VERSION}") +endif() -IF(KOKKOS_CXX_COMPILER_ID STREQUAL Clang) +if(KOKKOS_CXX_COMPILER_ID STREQUAL Clang) # The Cray compiler reports as Clang to most versions of CMake - EXECUTE_PROCESS(COMMAND ${CMAKE_CXX_COMPILER} --version - COMMAND grep -c Cray - OUTPUT_VARIABLE INTERNAL_HAVE_CRAY_COMPILER - OUTPUT_STRIP_TRAILING_WHITESPACE) - IF (INTERNAL_HAVE_CRAY_COMPILER) #not actually Clang - SET(KOKKOS_CLANG_IS_CRAY TRUE) - ENDIF() + execute_process( + COMMAND ${CMAKE_CXX_COMPILER} --version + COMMAND grep -c Cray + OUTPUT_VARIABLE INTERNAL_HAVE_CRAY_COMPILER + OUTPUT_STRIP_TRAILING_WHITESPACE + ) + if(INTERNAL_HAVE_CRAY_COMPILER) #not actually Clang + set(KOKKOS_CLANG_IS_CRAY TRUE) + set(KOKKOS_CXX_COMPILER_ID CrayClang) + endif() # The clang based Intel compiler reports as Clang to most versions of CMake - EXECUTE_PROCESS(COMMAND ${CMAKE_CXX_COMPILER} --version - COMMAND grep -c "DPC++\\|icpx" - OUTPUT_VARIABLE INTERNAL_HAVE_INTEL_COMPILER - OUTPUT_STRIP_TRAILING_WHITESPACE) - IF (INTERNAL_HAVE_INTEL_COMPILER) #not actually Clang - SET(KOKKOS_CLANG_IS_INTEL TRUE) - SET(KOKKOS_CXX_COMPILER_ID IntelLLVM CACHE STRING INTERNAL FORCE) - EXECUTE_PROCESS(COMMAND ${CMAKE_CXX_COMPILER} --version - OUTPUT_VARIABLE INTERNAL_CXX_COMPILER_VERSION - OUTPUT_STRIP_TRAILING_WHITESPACE) - STRING(REGEX MATCH "[0-9]+\\.[0-9]+\\.[0-9]+" - KOKKOS_CXX_COMPILER_VERSION ${INTERNAL_CXX_COMPILER_VERSION}) - ENDIF() -ENDIF() + execute_process( + COMMAND ${CMAKE_CXX_COMPILER} --version + COMMAND grep -c "DPC++\\|icpx" + OUTPUT_VARIABLE INTERNAL_HAVE_INTEL_COMPILER + OUTPUT_STRIP_TRAILING_WHITESPACE + ) + if(INTERNAL_HAVE_INTEL_COMPILER) #not actually Clang + set(KOKKOS_CLANG_IS_INTEL TRUE) + set(KOKKOS_CXX_COMPILER_ID IntelLLVM CACHE STRING INTERNAL FORCE) + execute_process( + COMMAND ${CMAKE_CXX_COMPILER} --version OUTPUT_VARIABLE INTERNAL_CXX_COMPILER_VERSION + OUTPUT_STRIP_TRAILING_WHITESPACE + ) + string(REGEX MATCH "[0-9]+\\.[0-9]+\\.[0-9]+" KOKKOS_CXX_COMPILER_VERSION ${INTERNAL_CXX_COMPILER_VERSION}) + endif() +endif() -IF(KOKKOS_CXX_COMPILER_ID STREQUAL Cray OR KOKKOS_CLANG_IS_CRAY) +if(KOKKOS_CXX_COMPILER_ID STREQUAL Cray OR KOKKOS_CLANG_IS_CRAY) # SET Cray's compiler version. - EXECUTE_PROCESS(COMMAND ${CMAKE_CXX_COMPILER} --version - OUTPUT_VARIABLE INTERNAL_CXX_COMPILER_VERSION - OUTPUT_STRIP_TRAILING_WHITESPACE) + execute_process( + COMMAND ${CMAKE_CXX_COMPILER} --version OUTPUT_VARIABLE INTERNAL_CXX_COMPILER_VERSION + OUTPUT_STRIP_TRAILING_WHITESPACE + ) - STRING(REGEX MATCH "[0-9]+\\.[0-9]+\\.[0-9]+" - TEMP_CXX_COMPILER_VERSION ${INTERNAL_CXX_COMPILER_VERSION}) - IF (KOKKOS_CLANG_IS_CRAY) - SET(KOKKOS_CLANG_CRAY_COMPILER_VERSION ${TEMP_CXX_COMPILER_VERSION}) - ELSE() - SET(KOKKOS_CXX_COMPILER_VERSION ${TEMP_CXX_COMPILER_VERSION} CACHE STRING INTERNAL FORCE) - ENDIF() -ENDIF() + string(REGEX MATCH "[0-9]+\\.[0-9]+\\.[0-9]+" TEMP_CXX_COMPILER_VERSION ${INTERNAL_CXX_COMPILER_VERSION}) + if(KOKKOS_CLANG_IS_CRAY) + set(KOKKOS_CLANG_CRAY_COMPILER_VERSION ${TEMP_CXX_COMPILER_VERSION}) + else() + set(KOKKOS_CXX_COMPILER_VERSION ${TEMP_CXX_COMPILER_VERSION} CACHE STRING INTERNAL FORCE) + endif() +endif() -IF(KOKKOS_CXX_COMPILER_ID STREQUAL Fujitsu) +if(KOKKOS_CXX_COMPILER_ID STREQUAL Fujitsu) # SET Fujitsus compiler version which is not detected by CMake - EXECUTE_PROCESS(COMMAND ${CMAKE_CXX_COMPILER} --version - OUTPUT_VARIABLE INTERNAL_CXX_COMPILER_VERSION - OUTPUT_STRIP_TRAILING_WHITESPACE) + execute_process( + COMMAND ${CMAKE_CXX_COMPILER} --version OUTPUT_VARIABLE INTERNAL_CXX_COMPILER_VERSION + OUTPUT_STRIP_TRAILING_WHITESPACE + ) - STRING(REGEX MATCH "[0-9]+\\.[0-9]+\\.[0-9]+" - TEMP_CXX_COMPILER_VERSION ${INTERNAL_CXX_COMPILER_VERSION}) - SET(KOKKOS_CXX_COMPILER_VERSION ${TEMP_CXX_COMPILER_VERSION} CACHE STRING INTERNAL FORCE) -ENDIF() + string(REGEX MATCH "[0-9]+\\.[0-9]+\\.[0-9]+" TEMP_CXX_COMPILER_VERSION ${INTERNAL_CXX_COMPILER_VERSION}) + set(KOKKOS_CXX_COMPILER_VERSION ${TEMP_CXX_COMPILER_VERSION} CACHE STRING INTERNAL FORCE) +endif() # Enforce the minimum compilers supported by Kokkos. -IF(NOT CMAKE_CXX_STANDARD) - SET(CMAKE_CXX_STANDARD 17) -ENDIF() -IF(CMAKE_CXX_STANDARD EQUAL 17) - SET(KOKKOS_CLANG_CPU_MINIMUM 8.0.0) - SET(KOKKOS_CLANG_CUDA_MINIMUM 10.0.0) - SET(KOKKOS_CLANG_OPENMPTARGET_MINIMUM 15.0.0) - SET(KOKKOS_GCC_MINIMUM 8.2.0) - SET(KOKKOS_INTEL_MINIMUM 19.0.5) - SET(KOKKOS_INTEL_LLVM_CPU_MINIMUM 2021.1.1) - SET(KOKKOS_INTEL_LLVM_SYCL_MINIMUM 2023.0.0) - SET(KOKKOS_NVCC_MINIMUM 11.0.0) - SET(KOKKOS_HIPCC_MINIMUM 5.2.0) - SET(KOKKOS_NVHPC_MINIMUM 22.3) - SET(KOKKOS_MSVC_MINIMUM 19.29) -ELSE() - SET(KOKKOS_CLANG_CPU_MINIMUM 14.0.0) - SET(KOKKOS_CLANG_CUDA_MINIMUM 14.0.0) - SET(KOKKOS_CLANG_OPENMPTARGET_MINIMUM 15.0.0) - SET(KOKKOS_GCC_MINIMUM 10.1.0) - SET(KOKKOS_INTEL_MINIMUM "not supported") - SET(KOKKOS_INTEL_LLVM_CPU_MINIMUM 2022.0.0) - SET(KOKKOS_INTEL_LLVM_SYCL_MINIMUM 2023.0.0) - SET(KOKKOS_NVCC_MINIMUM 12.0.0) - SET(KOKKOS_HIPCC_MINIMUM 5.2.0) - SET(KOKKOS_NVHPC_MINIMUM 22.3) - SET(KOKKOS_MSVC_MINIMUM 19.30) -ENDIF() +if(NOT CMAKE_CXX_STANDARD) + set(CMAKE_CXX_STANDARD 17) +endif() +if(CMAKE_CXX_STANDARD EQUAL 17) + set(KOKKOS_CLANG_CPU_MINIMUM 8.0.0) + set(KOKKOS_CLANG_CUDA_MINIMUM 10.0.0) + set(KOKKOS_CLANG_OPENMPTARGET_MINIMUM 15.0.0) + set(KOKKOS_GCC_MINIMUM 8.2.0) + set(KOKKOS_INTEL_MINIMUM 19.0.5) + set(KOKKOS_INTEL_LLVM_CPU_MINIMUM 2021.1.1) + set(KOKKOS_INTEL_LLVM_SYCL_MINIMUM 2023.0.0) + set(KOKKOS_NVCC_MINIMUM 11.0.0) + set(KOKKOS_HIPCC_MINIMUM 5.2.0) + set(KOKKOS_NVHPC_MINIMUM 22.3) + set(KOKKOS_MSVC_MINIMUM 19.29) +else() + set(KOKKOS_CLANG_CPU_MINIMUM 14.0.0) + set(KOKKOS_CLANG_CUDA_MINIMUM 14.0.0) + set(KOKKOS_CLANG_OPENMPTARGET_MINIMUM 15.0.0) + set(KOKKOS_GCC_MINIMUM 10.1.0) + set(KOKKOS_INTEL_MINIMUM "not supported") + set(KOKKOS_INTEL_LLVM_CPU_MINIMUM 2022.0.0) + set(KOKKOS_INTEL_LLVM_SYCL_MINIMUM 2023.0.0) + set(KOKKOS_NVCC_MINIMUM 12.0.0) + set(KOKKOS_HIPCC_MINIMUM 5.2.0) + set(KOKKOS_NVHPC_MINIMUM 22.3) + set(KOKKOS_MSVC_MINIMUM 19.30) +endif() -SET(KOKKOS_MESSAGE_TEXT "Compiler not supported by Kokkos for C++${CMAKE_CXX_STANDARD}. Required minimum compiler versions:") -SET(KOKKOS_MESSAGE_TEXT "${KOKKOS_MESSAGE_TEXT}\n Clang(CPU) ${KOKKOS_CLANG_CPU_MINIMUM}") -SET(KOKKOS_MESSAGE_TEXT "${KOKKOS_MESSAGE_TEXT}\n Clang(CUDA) ${KOKKOS_CLANG_CUDA_MINIMUM}") -SET(KOKKOS_MESSAGE_TEXT "${KOKKOS_MESSAGE_TEXT}\n Clang(OpenMPTarget) ${KOKKOS_CLANG_OPENMPTARGET_MINIMUM}") -SET(KOKKOS_MESSAGE_TEXT "${KOKKOS_MESSAGE_TEXT}\n GCC ${KOKKOS_GCC_MINIMUM}") -SET(KOKKOS_MESSAGE_TEXT "${KOKKOS_MESSAGE_TEXT}\n Intel ${KOKKOS_INTEL_MINIMUM}") -SET(KOKKOS_MESSAGE_TEXT "${KOKKOS_MESSAGE_TEXT}\n IntelLLVM(CPU) ${KOKKOS_INTEL_LLVM_CPU_MINIMUM}") -SET(KOKKOS_MESSAGE_TEXT "${KOKKOS_MESSAGE_TEXT}\n IntelLLVM(SYCL) ${KOKKOS_INTEL_LLVM_SYCL_MINIMUM}") -SET(KOKKOS_MESSAGE_TEXT "${KOKKOS_MESSAGE_TEXT}\n NVCC ${KOKKOS_NVCC_MINIMUM}") -SET(KOKKOS_MESSAGE_TEXT "${KOKKOS_MESSAGE_TEXT}\n HIPCC ${KOKKOS_HIPCC_MINIMUM}") -SET(KOKKOS_MESSAGE_TEXT "${KOKKOS_MESSAGE_TEXT}\n NVHPC/PGI ${KOKKOS_NVHPC_MINIMUM}") -SET(KOKKOS_MESSAGE_TEXT "${KOKKOS_MESSAGE_TEXT}\n MSVC ${KOKKOS_MSVC_MINIMUM}") -SET(KOKKOS_MESSAGE_TEXT "${KOKKOS_MESSAGE_TEXT}\n XL/XLClang not supported") -SET(KOKKOS_MESSAGE_TEXT "${KOKKOS_MESSAGE_TEXT}\nCompiler: ${KOKKOS_CXX_COMPILER_ID} ${KOKKOS_CXX_COMPILER_VERSION}\n") +set(KOKKOS_MESSAGE_TEXT + "Compiler not supported by Kokkos for C++${CMAKE_CXX_STANDARD}. Required minimum compiler versions:" +) +set(KOKKOS_MESSAGE_TEXT "${KOKKOS_MESSAGE_TEXT}\n Clang(CPU) ${KOKKOS_CLANG_CPU_MINIMUM}") +set(KOKKOS_MESSAGE_TEXT "${KOKKOS_MESSAGE_TEXT}\n Clang(CUDA) ${KOKKOS_CLANG_CUDA_MINIMUM}") +set(KOKKOS_MESSAGE_TEXT "${KOKKOS_MESSAGE_TEXT}\n Clang(OpenMPTarget) ${KOKKOS_CLANG_OPENMPTARGET_MINIMUM}") +set(KOKKOS_MESSAGE_TEXT "${KOKKOS_MESSAGE_TEXT}\n GCC ${KOKKOS_GCC_MINIMUM}") +set(KOKKOS_MESSAGE_TEXT "${KOKKOS_MESSAGE_TEXT}\n Intel ${KOKKOS_INTEL_MINIMUM}") +set(KOKKOS_MESSAGE_TEXT "${KOKKOS_MESSAGE_TEXT}\n IntelLLVM(CPU) ${KOKKOS_INTEL_LLVM_CPU_MINIMUM}") +set(KOKKOS_MESSAGE_TEXT "${KOKKOS_MESSAGE_TEXT}\n IntelLLVM(SYCL) ${KOKKOS_INTEL_LLVM_SYCL_MINIMUM}") +set(KOKKOS_MESSAGE_TEXT "${KOKKOS_MESSAGE_TEXT}\n NVCC ${KOKKOS_NVCC_MINIMUM}") +set(KOKKOS_MESSAGE_TEXT "${KOKKOS_MESSAGE_TEXT}\n HIPCC ${KOKKOS_HIPCC_MINIMUM}") +set(KOKKOS_MESSAGE_TEXT "${KOKKOS_MESSAGE_TEXT}\n NVHPC/PGI ${KOKKOS_NVHPC_MINIMUM}") +set(KOKKOS_MESSAGE_TEXT "${KOKKOS_MESSAGE_TEXT}\n MSVC ${KOKKOS_MSVC_MINIMUM}") +set(KOKKOS_MESSAGE_TEXT "${KOKKOS_MESSAGE_TEXT}\n XL/XLClang not supported") +set(KOKKOS_MESSAGE_TEXT "${KOKKOS_MESSAGE_TEXT}\nCompiler: ${KOKKOS_CXX_COMPILER_ID} ${KOKKOS_CXX_COMPILER_VERSION}\n") -IF(KOKKOS_CXX_COMPILER_ID STREQUAL Clang AND NOT Kokkos_ENABLE_CUDA) - IF(KOKKOS_CXX_COMPILER_VERSION VERSION_LESS ${KOKKOS_CLANG_CPU_MINIMUM}) - MESSAGE(FATAL_ERROR "${KOKKOS_MESSAGE_TEXT}") - ENDIF() -ELSEIF(KOKKOS_CXX_COMPILER_ID STREQUAL Clang AND Kokkos_ENABLE_CUDA) - IF(KOKKOS_CXX_COMPILER_VERSION VERSION_LESS ${KOKKOS_CLANG_CUDA_MINIMUM}) - MESSAGE(FATAL_ERROR "${KOKKOS_MESSAGE_TEXT}") - ENDIF() -ELSEIF(KOKKOS_CXX_COMPILER_ID STREQUAL GNU) - IF(KOKKOS_CXX_COMPILER_VERSION VERSION_LESS ${KOKKOS_GCC_MINIMUM}) - MESSAGE(FATAL_ERROR "${KOKKOS_MESSAGE_TEXT}") - ENDIF() -ELSEIF(KOKKOS_CXX_COMPILER_ID STREQUAL Intel) - IF((NOT CMAKE_CXX_STANDARD EQUAL 17) OR (KOKKOS_CXX_COMPILER_VERSION VERSION_LESS ${KOKKOS_INTEL_MINIMUM})) - MESSAGE(FATAL_ERROR "${KOKKOS_MESSAGE_TEXT}") - ENDIF() -ELSEIF(KOKKOS_CXX_COMPILER_ID STREQUAL IntelLLVM AND NOT Kokkos_ENABLE_SYCL) - IF(KOKKOS_CXX_COMPILER_VERSION VERSION_LESS ${KOKKOS_INTEL_LLVM_CPU_MINIMUM}) - MESSAGE(FATAL_ERROR "${KOKKOS_MESSAGE_TEXT}") - ENDIF() -ELSEIF(KOKKOS_CXX_COMPILER_ID STREQUAL IntelLLVM AND Kokkos_ENABLE_SYCL) - IF(KOKKOS_CXX_COMPILER_VERSION VERSION_LESS ${KOKKOS_INTEL_LLVM_SYCL_MINIMUM}) - MESSAGE(FATAL_ERROR "${KOKKOS_MESSAGE_TEXT}") - ENDIF() -ELSEIF(KOKKOS_CXX_COMPILER_ID STREQUAL NVIDIA) - IF(KOKKOS_CXX_COMPILER_VERSION VERSION_LESS ${KOKKOS_NVCC_MINIMUM}) - MESSAGE(FATAL_ERROR "${KOKKOS_MESSAGE_TEXT}") - ENDIF() - SET(CMAKE_CXX_EXTENSIONS OFF CACHE BOOL "Kokkos turns off CXX extensions" FORCE) -ELSEIF(KOKKOS_CXX_COMPILER_ID STREQUAL HIPCC) - IF(KOKKOS_CXX_COMPILER_VERSION VERSION_LESS ${KOKKOS_HIPCC_MINIMUM}) - MESSAGE(FATAL_ERROR "${KOKKOS_MESSAGE_TEXT}") - ENDIF() -ELSEIF(KOKKOS_CXX_COMPILER_ID STREQUAL PGI OR KOKKOS_CXX_COMPILER_ID STREQUAL NVHPC) - IF(KOKKOS_CXX_COMPILER_VERSION VERSION_LESS ${KOKKOS_NVHPC_MINIMUM}) - MESSAGE(FATAL_ERROR "${KOKKOS_MESSAGE_TEXT}") - ENDIF() +if(KOKKOS_CXX_COMPILER_ID STREQUAL Clang AND NOT Kokkos_ENABLE_CUDA) + if(KOKKOS_CXX_COMPILER_VERSION VERSION_LESS ${KOKKOS_CLANG_CPU_MINIMUM}) + message(FATAL_ERROR "${KOKKOS_MESSAGE_TEXT}") + endif() +elseif(KOKKOS_CXX_COMPILER_ID STREQUAL Clang AND Kokkos_ENABLE_CUDA) + if(KOKKOS_CXX_COMPILER_VERSION VERSION_LESS ${KOKKOS_CLANG_CUDA_MINIMUM}) + message(FATAL_ERROR "${KOKKOS_MESSAGE_TEXT}") + endif() +elseif(KOKKOS_CXX_COMPILER_ID STREQUAL GNU) + if(KOKKOS_CXX_COMPILER_VERSION VERSION_LESS ${KOKKOS_GCC_MINIMUM}) + message(FATAL_ERROR "${KOKKOS_MESSAGE_TEXT}") + endif() +elseif(KOKKOS_CXX_COMPILER_ID STREQUAL Intel) + if((NOT CMAKE_CXX_STANDARD EQUAL 17) OR (KOKKOS_CXX_COMPILER_VERSION VERSION_LESS ${KOKKOS_INTEL_MINIMUM})) + message(FATAL_ERROR "${KOKKOS_MESSAGE_TEXT}") + endif() +elseif(KOKKOS_CXX_COMPILER_ID STREQUAL IntelLLVM AND NOT Kokkos_ENABLE_SYCL) + if(KOKKOS_CXX_COMPILER_VERSION VERSION_LESS ${KOKKOS_INTEL_LLVM_CPU_MINIMUM}) + message(FATAL_ERROR "${KOKKOS_MESSAGE_TEXT}") + endif() +elseif(KOKKOS_CXX_COMPILER_ID STREQUAL IntelLLVM AND Kokkos_ENABLE_SYCL) + if(KOKKOS_CXX_COMPILER_VERSION VERSION_LESS ${KOKKOS_INTEL_LLVM_SYCL_MINIMUM}) + message(FATAL_ERROR "${KOKKOS_MESSAGE_TEXT}") + endif() +elseif(KOKKOS_CXX_COMPILER_ID STREQUAL NVIDIA) + if(KOKKOS_CXX_COMPILER_VERSION VERSION_LESS ${KOKKOS_NVCC_MINIMUM}) + message(FATAL_ERROR "${KOKKOS_MESSAGE_TEXT}") + endif() + set(CMAKE_CXX_EXTENSIONS OFF CACHE BOOL "Kokkos turns off CXX extensions" FORCE) +elseif(KOKKOS_CXX_COMPILER_ID STREQUAL HIPCC) + if(KOKKOS_CXX_COMPILER_VERSION VERSION_LESS ${KOKKOS_HIPCC_MINIMUM}) + message(FATAL_ERROR "${KOKKOS_MESSAGE_TEXT}") + endif() +elseif(KOKKOS_CXX_COMPILER_ID STREQUAL PGI OR KOKKOS_CXX_COMPILER_ID STREQUAL NVHPC) + if(KOKKOS_CXX_COMPILER_VERSION VERSION_LESS ${KOKKOS_NVHPC_MINIMUM}) + message(FATAL_ERROR "${KOKKOS_MESSAGE_TEXT}") + endif() # Treat PGI internally as NVHPC to simplify handling both compilers. # Before CMake 3.20 NVHPC was identified as PGI, nvc++ is # backward-compatible to pgc++. - SET(KOKKOS_CXX_COMPILER_ID NVHPC CACHE STRING INTERNAL FORCE) -ELSEIF(KOKKOS_CXX_COMPILER_ID STREQUAL "MSVC") - IF(KOKKOS_CXX_COMPILER_VERSION VERSION_LESS ${KOKKOS_MSVC_MINIMUM}) - MESSAGE(FATAL_ERROR "${KOKKOS_MESSAGE_TEXT}") - ENDIF() -ELSEIF(KOKKOS_CXX_COMPILER_ID STREQUAL XL OR KOKKOS_CXX_COMPILER_ID STREQUAL XLClang) - MESSAGE(FATAL_ERROR "${KOKKOS_MESSAGE_TEXT}") -ELSEIF(KOKKOS_CXX_COMPILER_ID STREQUAL Clang AND Kokkos_ENABLE_OPENMPTARGET) - IF(KOKKOS_CXX_COMPILER_VERSION VERSION_LESS KOKKOS_CLANG_OPENMPTARGET_MINIMUM) - MESSAGE(FATAL_ERROR "${KOKKOS_MESSAGE_TEXT}") - ENDIF() -ENDIF() + set(KOKKOS_CXX_COMPILER_ID NVHPC CACHE STRING INTERNAL FORCE) +elseif(KOKKOS_CXX_COMPILER_ID STREQUAL "MSVC") + if(KOKKOS_CXX_COMPILER_VERSION VERSION_LESS ${KOKKOS_MSVC_MINIMUM}) + message(FATAL_ERROR "${KOKKOS_MESSAGE_TEXT}") + endif() +elseif(KOKKOS_CXX_COMPILER_ID STREQUAL XL OR KOKKOS_CXX_COMPILER_ID STREQUAL XLClang) + message(FATAL_ERROR "${KOKKOS_MESSAGE_TEXT}") +elseif(KOKKOS_CXX_COMPILER_ID STREQUAL Clang AND Kokkos_ENABLE_OPENMPTARGET) + if(KOKKOS_CXX_COMPILER_VERSION VERSION_LESS KOKKOS_CLANG_OPENMPTARGET_MINIMUM) + message(FATAL_ERROR "${KOKKOS_MESSAGE_TEXT}") + endif() +endif() -IF(NOT DEFINED KOKKOS_CXX_HOST_COMPILER_ID) - SET(KOKKOS_CXX_HOST_COMPILER_ID ${KOKKOS_CXX_COMPILER_ID}) -ELSEIF(KOKKOS_CXX_HOST_COMPILER_ID STREQUAL PGI) - SET(KOKKOS_CXX_HOST_COMPILER_ID NVHPC CACHE STRING INTERNAL FORCE) -ENDIF() +if(NOT DEFINED KOKKOS_CXX_HOST_COMPILER_ID) + set(KOKKOS_CXX_HOST_COMPILER_ID ${KOKKOS_CXX_COMPILER_ID}) +elseif(KOKKOS_CXX_HOST_COMPILER_ID STREQUAL PGI) + set(KOKKOS_CXX_HOST_COMPILER_ID NVHPC CACHE STRING INTERNAL FORCE) +endif() -STRING(REPLACE "." ";" VERSION_LIST ${KOKKOS_CXX_COMPILER_VERSION}) -LIST(GET VERSION_LIST 0 KOKKOS_COMPILER_VERSION_MAJOR) -LIST(GET VERSION_LIST 1 KOKKOS_COMPILER_VERSION_MINOR) -LIST(LENGTH VERSION_LIST LIST_LENGTH) +string(REPLACE "." ";" VERSION_LIST ${KOKKOS_CXX_COMPILER_VERSION}) +list(GET VERSION_LIST 0 KOKKOS_COMPILER_VERSION_MAJOR) +list(GET VERSION_LIST 1 KOKKOS_COMPILER_VERSION_MINOR) +list(LENGTH VERSION_LIST LIST_LENGTH) # On Android, the compiler doesn't have a patch version, just a major/minor -IF(LIST_LENGTH GREATER 2) - LIST(GET VERSION_LIST 2 KOKKOS_COMPILER_VERSION_PATCH) -ELSE() - SET(KOKKOS_COMPILER_VERSION_PATCH 0) -ENDIF() - +if(LIST_LENGTH GREATER 2) + list(GET VERSION_LIST 2 KOKKOS_COMPILER_VERSION_PATCH) +else() + set(KOKKOS_COMPILER_VERSION_PATCH 0) +endif() diff --git a/lib/kokkos/cmake/kokkos_configure_trilinos.cmake b/lib/kokkos/cmake/kokkos_configure_trilinos.cmake new file mode 100644 index 0000000000..5aeef61e7b --- /dev/null +++ b/lib/kokkos/cmake/kokkos_configure_trilinos.cmake @@ -0,0 +1,38 @@ +if(CMAKE_PROJECT_NAME STREQUAL "Trilinos") + set(Kokkos_ENABLE_SERIAL ON CACHE BOOL "Whether to build Serial backend" FORCE) + + if(NOT ${Trilinos_ENABLE_OpenMP} STREQUAL "") + set(Kokkos_ENABLE_OPENMP ${Trilinos_ENABLE_OpenMP} CACHE BOOL "Whether to build OpenMP backend" FORCE) + else() + set(Kokkos_ENABLE_OPENMP OFF CACHE BOOL "Whether to build OpenMP backend" FORCE) + endif() + + if(NOT ${TPL_ENABLE_CUDA} STREQUAL "") + set(Kokkos_ENABLE_CUDA ${TPL_ENABLE_CUDA} CACHE BOOL "Whether to build CUDA backend" FORCE) + else() + set(Kokkos_ENABLE_CUDA OFF CACHE BOOL "Whether to build CUDA backend" FORCE) + endif() + + if(NOT ${TPL_ENABLE_HPX} STREQUAL "") + set(Kokkos_ENABLE_HPX ${TPL_ENABLE_HPX} CACHE BOOL "Whether to build HPX backend" FORCE) + else() + set(Kokkos_ENABLE_HPX OFF CACHE BOOL "Whether to build HPX backend" FORCE) + endif() + + if(NOT ${TPL_ENABLE_quadmath} STREQUAL "") + set(Kokkos_ENABLE_LIBQUADMATH ${TPL_ENABLE_quadmath} CACHE BOOL "Whether to enable the LIBQUADMATH library" FORCE) + else() + set(Kokkos_ENABLE_LIBQUADMATH OFF CACHE BOOL "Whether to enable the LIBQUADMATH library" FORCE) + endif() + + if(NOT ${TPL_ENABLE_DLlib} STREQUAL "") + set(Kokkos_ENABLE_LIBDL ${TPL_ENABLE_DLlib} CACHE BOOL "Whether to enable the LIBDL library" FORCE) + else() + set(Kokkos_ENABLE_LIBDL OFF CACHE BOOL "Whether to enable the LIBDL library" FORCE) + endif() + + set(Kokkos_ENABLE_COMPLEX_ALIGN OFF CACHE BOOL "Whether to align Kokkos::complex to 2*alignof(RealType)") + + # FIXME_TRILINOS We run into problems when trying to use an external GTest in Trilinos CI + set(CMAKE_DISABLE_FIND_PACKAGE_GTest ON) +endif() diff --git a/lib/kokkos/cmake/kokkos_corner_cases.cmake b/lib/kokkos/cmake/kokkos_corner_cases.cmake index ede2b4e0ca..530e9e8fd8 100644 --- a/lib/kokkos/cmake/kokkos_corner_cases.cmake +++ b/lib/kokkos/cmake/kokkos_corner_cases.cmake @@ -1,4 +1,8 @@ -IF (KOKKOS_CXX_COMPILER_ID STREQUAL NVIDIA AND KOKKOS_ENABLE_CUDA_CONSTEXPR AND KOKKOS_CXX_COMPILER_VERSION VERSION_LESS 11.2) - MESSAGE(WARNING "You have requested -DKokkos_ENABLE_CUDA_CONSTEXPR=ON for NVCC ${KOKKOS_CXX_COMPILER_VERSION} which is known to trigger compiler bugs before NVCC version 11.2. See https://github.com/kokkos/kokkos/issues/3496") -ENDIF() - +if(KOKKOS_CXX_COMPILER_ID STREQUAL NVIDIA AND KOKKOS_ENABLE_CUDA_CONSTEXPR AND KOKKOS_CXX_COMPILER_VERSION VERSION_LESS + 11.2 +) + message( + WARNING + "You have requested -DKokkos_ENABLE_CUDA_CONSTEXPR=ON for NVCC ${KOKKOS_CXX_COMPILER_VERSION} which is known to trigger compiler bugs before NVCC version 11.2. See https://github.com/kokkos/kokkos/issues/3496" + ) +endif() diff --git a/lib/kokkos/cmake/kokkos_enable_devices.cmake b/lib/kokkos/cmake/kokkos_enable_devices.cmake index c7d189285c..40c2d3ea8a 100644 --- a/lib/kokkos/cmake/kokkos_enable_devices.cmake +++ b/lib/kokkos/cmake/kokkos_enable_devices.cmake @@ -1,128 +1,132 @@ - -FUNCTION(KOKKOS_DEVICE_OPTION SUFFIX DEFAULT DEV_TYPE DOCSTRING) - KOKKOS_OPTION(ENABLE_${SUFFIX} ${DEFAULT} BOOL ${DOCSTRING}) - STRING(TOUPPER ${SUFFIX} UC_NAME) - IF (KOKKOS_ENABLE_${UC_NAME}) - LIST(APPEND KOKKOS_ENABLED_DEVICES ${SUFFIX}) +function(KOKKOS_DEVICE_OPTION SUFFIX DEFAULT DEV_TYPE DOCSTRING) + kokkos_option(ENABLE_${SUFFIX} ${DEFAULT} BOOL ${DOCSTRING}) + string(TOUPPER ${SUFFIX} UC_NAME) + if(KOKKOS_ENABLE_${UC_NAME}) + list(APPEND KOKKOS_ENABLED_DEVICES ${SUFFIX}) #I hate that CMake makes me do this - SET(KOKKOS_ENABLED_DEVICES ${KOKKOS_ENABLED_DEVICES} PARENT_SCOPE) - ENDIF() - SET(KOKKOS_ENABLE_${UC_NAME} ${KOKKOS_ENABLE_${UC_NAME}} PARENT_SCOPE) - IF (KOKKOS_ENABLE_${UC_NAME} AND DEV_TYPE STREQUAL "HOST") - SET(KOKKOS_HAS_HOST ON PARENT_SCOPE) - ENDIF() -ENDFUNCTION() + set(KOKKOS_ENABLED_DEVICES ${KOKKOS_ENABLED_DEVICES} PARENT_SCOPE) + endif() + set(KOKKOS_ENABLE_${UC_NAME} ${KOKKOS_ENABLE_${UC_NAME}} PARENT_SCOPE) + if(KOKKOS_ENABLE_${UC_NAME} AND DEV_TYPE STREQUAL "HOST") + set(KOKKOS_HAS_HOST ON PARENT_SCOPE) + endif() +endfunction() -KOKKOS_CFG_DEPENDS(DEVICES NONE) +kokkos_cfg_depends(DEVICES NONE) # Put a check in just in case people are using this option -KOKKOS_DEPRECATED_LIST(DEVICES ENABLE) +kokkos_deprecated_list(DEVICES ENABLE) - -KOKKOS_DEVICE_OPTION(THREADS OFF HOST "Whether to build C++ threads backend") +kokkos_device_option(THREADS OFF HOST "Whether to build C++ threads backend") # detect clang++ / cl / clang-cl clashes -IF (CMAKE_CXX_COMPILER_ID STREQUAL Clang AND "x${CMAKE_CXX_SIMULATE_ID}" STREQUAL "xMSVC") +if(CMAKE_CXX_COMPILER_ID STREQUAL Clang AND "x${CMAKE_CXX_SIMULATE_ID}" STREQUAL "xMSVC") # this specific test requires CMake >= 3.15 - IF ("x${CMAKE_CXX_COMPILER_FRONTEND_VARIANT}" STREQUAL "xGNU") + if("x${CMAKE_CXX_COMPILER_FRONTEND_VARIANT}" STREQUAL "xGNU") # use pure clang++ instead of clang-cl - SET(KOKKOS_COMPILER_CLANG_MSVC OFF) - ELSE() + set(KOKKOS_COMPILER_CLANG_MSVC OFF) + else() # it defaults to clang-cl - SET(KOKKOS_COMPILER_CLANG_MSVC ON) - ENDIF() -ENDIF() - -IF(Trilinos_ENABLE_Kokkos AND Trilinos_ENABLE_OpenMP) - SET(OMP_DEFAULT ON) -ELSE() - SET(OMP_DEFAULT OFF) -ENDIF() -KOKKOS_DEVICE_OPTION(OPENMP ${OMP_DEFAULT} HOST "Whether to build OpenMP backend") + set(KOKKOS_COMPILER_CLANG_MSVC ON) + endif() +endif() +if(Trilinos_ENABLE_Kokkos AND Trilinos_ENABLE_OpenMP) + set(OMP_DEFAULT ON) +else() + set(OMP_DEFAULT OFF) +endif() +kokkos_device_option(OPENMP ${OMP_DEFAULT} HOST "Whether to build OpenMP backend") # We want this to default to OFF for cache reasons, but if no # host space is given, then activate serial -IF (KOKKOS_HAS_TRILINOS) - #However, Trilinos always wants Serial ON - SET(SERIAL_DEFAULT ON) -ELSEIF (KOKKOS_HAS_HOST) - SET(SERIAL_DEFAULT OFF) -ELSE() - SET(SERIAL_DEFAULT ON) - IF (NOT DEFINED Kokkos_ENABLE_SERIAL) - MESSAGE(STATUS "SERIAL backend is being turned on to ensure there is at least one Host space. To change this, you must enable another host execution space and configure with -DKokkos_ENABLE_SERIAL=OFF or change CMakeCache.txt") - ENDIF() -ENDIF() -KOKKOS_DEVICE_OPTION(SERIAL ${SERIAL_DEFAULT} HOST "Whether to build serial backend") +if(KOKKOS_HAS_HOST) + set(SERIAL_DEFAULT OFF) +else() + set(SERIAL_DEFAULT ON) + if(NOT DEFINED Kokkos_ENABLE_SERIAL) + message( + STATUS + "SERIAL backend is being turned on to ensure there is at least one Host space. To change this, you must enable another host execution space and configure with -DKokkos_ENABLE_SERIAL=OFF or change CMakeCache.txt" + ) + endif() +endif() +kokkos_device_option(SERIAL ${SERIAL_DEFAULT} HOST "Whether to build serial backend") -KOKKOS_DEVICE_OPTION(HPX OFF HOST "Whether to build HPX backend (experimental)") +kokkos_device_option(HPX OFF HOST "Whether to build HPX backend (experimental)") # Device backends have to come after host backends for header include order reasons # Without this we can't make e.g. CudaSpace accessible by HostSpace -KOKKOS_DEVICE_OPTION(OPENACC OFF DEVICE "Whether to build the OpenACC backend") -IF (KOKKOS_ENABLE_OPENACC) - COMPILER_SPECIFIC_FLAGS( - Clang -fopenacc -fopenacc-fake-async-wait - -Wno-openacc-and-cxx -Wno-openmp-mapping -Wno-unknown-cuda-version - -Wno-pass-failed +kokkos_device_option(OPENACC OFF DEVICE "Whether to build the OpenACC backend") +if(KOKKOS_ENABLE_OPENACC) + compiler_specific_flags( + Clang + -fopenacc + -fopenacc-fake-async-wait + -fopenacc-implicit-worker=vector + -Wno-openacc-and-cxx + -Wno-openmp-mapping + -Wno-unknown-cuda-version + -Wno-pass-failed ) - COMPILER_SPECIFIC_DEFS( - Clang KOKKOS_WORKAROUND_OPENMPTARGET_CLANG + compiler_specific_defs(Clang KOKKOS_WORKAROUND_OPENMPTARGET_CLANG) +endif() + +kokkos_device_option(OPENMPTARGET OFF DEVICE "Whether to build the OpenMP target backend") +if(KOKKOS_ENABLE_OPENMPTARGET) + set(ClangOpenMPFlag -fopenmp=libomp) + if(KOKKOS_CLANG_IS_CRAY) + set(ClangOpenMPFlag -fopenmp) + endif() + + compiler_specific_flags( + Clang + ${ClangOpenMPFlag} + -Wno-openmp-mapping + IntelLLVM + -fiopenmp + -Wno-openmp-mapping + NVHPC + -mp=gpu + DEFAULT + -fopenmp ) -ENDIF() + compiler_specific_defs(Clang KOKKOS_WORKAROUND_OPENMPTARGET_CLANG) + # Are there compilers which identify as Clang and need this library? + # COMPILER_SPECIFIC_LIBS( + # Clang -lopenmptarget + # ) + if(KOKKOS_CXX_STANDARD LESS 17) + message(FATAL_ERROR "OpenMPTarget backend requires C++17 or newer") + endif() +endif() -KOKKOS_DEVICE_OPTION(OPENMPTARGET OFF DEVICE "Whether to build the OpenMP target backend") -IF (KOKKOS_ENABLE_OPENMPTARGET) - SET(ClangOpenMPFlag -fopenmp=libomp) - IF(KOKKOS_CLANG_IS_CRAY) - SET(ClangOpenMPFlag -fopenmp) - ENDIF() +if(Trilinos_ENABLE_Kokkos AND TPL_ENABLE_CUDA) + set(CUDA_DEFAULT ON) +else() + set(CUDA_DEFAULT OFF) +endif() +kokkos_device_option(CUDA ${CUDA_DEFAULT} DEVICE "Whether to build CUDA backend") - COMPILER_SPECIFIC_FLAGS( - Clang ${ClangOpenMPFlag} -Wno-openmp-mapping - IntelLLVM -fiopenmp -Wno-openmp-mapping - NVHPC -mp=gpu - DEFAULT -fopenmp - ) - COMPILER_SPECIFIC_DEFS( - Clang KOKKOS_WORKAROUND_OPENMPTARGET_CLANG - ) -# Are there compilers which identify as Clang and need this library? -# COMPILER_SPECIFIC_LIBS( -# Clang -lopenmptarget -# ) - IF(KOKKOS_CXX_STANDARD LESS 17) - MESSAGE(FATAL_ERROR "OpenMPTarget backend requires C++17 or newer") - ENDIF() -ENDIF() +if(KOKKOS_ENABLE_CUDA) + global_set(KOKKOS_DONT_ALLOW_EXTENSIONS "CUDA enabled") + ## Cuda has extra setup requirements, turn on Kokkos_Setup_Cuda.hpp in macros + list(APPEND DEVICE_SETUP_LIST Cuda) +endif() -IF(Trilinos_ENABLE_Kokkos AND TPL_ENABLE_CUDA) - SET(CUDA_DEFAULT ON) -ELSE() - SET(CUDA_DEFAULT OFF) -ENDIF() -KOKKOS_DEVICE_OPTION(CUDA ${CUDA_DEFAULT} DEVICE "Whether to build CUDA backend") - -IF (KOKKOS_ENABLE_CUDA) - GLOBAL_SET(KOKKOS_DONT_ALLOW_EXTENSIONS "CUDA enabled") -## Cuda has extra setup requirements, turn on Kokkos_Setup_Cuda.hpp in macros - LIST(APPEND DEVICE_SETUP_LIST Cuda) -ENDIF() - -KOKKOS_DEVICE_OPTION(HIP OFF DEVICE "Whether to build HIP backend") +kokkos_device_option(HIP OFF DEVICE "Whether to build HIP backend") ## HIP has extra setup requirements, turn on Kokkos_Setup_HIP.hpp in macros -IF (KOKKOS_ENABLE_HIP) - LIST(APPEND DEVICE_SETUP_LIST HIP) -ENDIF() +if(KOKKOS_ENABLE_HIP) + list(APPEND DEVICE_SETUP_LIST HIP) +endif() -KOKKOS_DEVICE_OPTION(SYCL OFF DEVICE "Whether to build SYCL backend") +kokkos_device_option(SYCL OFF DEVICE "Whether to build SYCL backend") ## SYCL has extra setup requirements, turn on Kokkos_Setup_SYCL.hpp in macros -IF (KOKKOS_ENABLE_SYCL) - IF(KOKKOS_CXX_STANDARD LESS 17) - MESSAGE(FATAL_ERROR "SYCL backend requires C++17 or newer!") - ENDIF() - LIST(APPEND DEVICE_SETUP_LIST SYCL) -ENDIF() +if(KOKKOS_ENABLE_SYCL) + if(KOKKOS_CXX_STANDARD LESS 17) + message(FATAL_ERROR "SYCL backend requires C++17 or newer!") + endif() + list(APPEND DEVICE_SETUP_LIST SYCL) +endif() diff --git a/lib/kokkos/cmake/kokkos_enable_options.cmake b/lib/kokkos/cmake/kokkos_enable_options.cmake index 53764b0c68..a5d6fdfe4e 100644 --- a/lib/kokkos/cmake/kokkos_enable_options.cmake +++ b/lib/kokkos/cmake/kokkos_enable_options.cmake @@ -1,198 +1,236 @@ ########################## NOTES ############################################### # List the options for configuring kokkos using CMake method of doing it. -# These options then get mapped onto KOKKOS_SETTINGS environment variable by -# kokkos_settings.cmake. It is separate to allow other packages to override -# these variables (e.g., TriBITS). ########################## AVAILABLE OPTIONS ################################### # Use lists for documentation, verification, and programming convenience - -FUNCTION(KOKKOS_ENABLE_OPTION SUFFIX DEFAULT DOCSTRING) - KOKKOS_OPTION(ENABLE_${SUFFIX} ${DEFAULT} BOOL ${DOCSTRING}) - STRING(TOUPPER ${SUFFIX} UC_NAME) - IF (KOKKOS_ENABLE_${UC_NAME} AND NOT "Kokkos_ENABLE_${UC_NAME}" IN_LIST Kokkos_OPTIONS_NOT_TO_EXPORT) - LIST(APPEND KOKKOS_ENABLED_OPTIONS ${UC_NAME}) +function(KOKKOS_ENABLE_OPTION SUFFIX DEFAULT DOCSTRING) + kokkos_option(ENABLE_${SUFFIX} ${DEFAULT} BOOL ${DOCSTRING}) + string(TOUPPER ${SUFFIX} UC_NAME) + if(KOKKOS_ENABLE_${UC_NAME} AND NOT "Kokkos_ENABLE_${UC_NAME}" IN_LIST Kokkos_OPTIONS_NOT_TO_EXPORT) + list(APPEND KOKKOS_ENABLED_OPTIONS ${UC_NAME}) #I hate that CMake makes me do this - SET(KOKKOS_ENABLED_OPTIONS ${KOKKOS_ENABLED_OPTIONS} PARENT_SCOPE) - ENDIF() - SET(KOKKOS_ENABLE_${UC_NAME} ${KOKKOS_ENABLE_${UC_NAME}} PARENT_SCOPE) -ENDFUNCTION() + set(KOKKOS_ENABLED_OPTIONS ${KOKKOS_ENABLED_OPTIONS} PARENT_SCOPE) + endif() + set(KOKKOS_ENABLE_${UC_NAME} ${KOKKOS_ENABLE_${UC_NAME}} PARENT_SCOPE) +endfunction() # Certain defaults will depend on knowing the enabled devices -KOKKOS_CFG_DEPENDS(OPTIONS DEVICES) -KOKKOS_CFG_DEPENDS(OPTIONS COMPILER_ID) +kokkos_cfg_depends(OPTIONS DEVICES) +kokkos_cfg_depends(OPTIONS COMPILER_ID) # Put a check in just in case people are using this option -KOKKOS_DEPRECATED_LIST(OPTIONS ENABLE) +kokkos_deprecated_list(OPTIONS ENABLE) -KOKKOS_ENABLE_OPTION(CUDA_RELOCATABLE_DEVICE_CODE OFF "Whether to enable relocatable device code (RDC) for CUDA") -KOKKOS_ENABLE_OPTION(CUDA_UVM OFF "Whether to use unified memory (UM) for CUDA by default") -KOKKOS_ENABLE_OPTION(CUDA_LDG_INTRINSIC OFF "Whether to use CUDA LDG intrinsics") +kokkos_enable_option(CUDA_RELOCATABLE_DEVICE_CODE OFF "Whether to enable relocatable device code (RDC) for CUDA") +kokkos_enable_option(CUDA_UVM OFF "Whether to use unified memory (UM) for CUDA by default") +kokkos_enable_option(CUDA_LDG_INTRINSIC OFF "Whether to use CUDA LDG intrinsics") # In contrast to other CUDA-dependent, options CUDA_LAMBDA is ON by default. # That is problematic when CUDA is not enabled because this not only yields a # bogus warning, but also exports the Kokkos_ENABLE_CUDA_LAMBDA variable and -# sets it to ON. This if-clause is a crutch that delays the refactoring of the -# way we declare all options until after we get rid of TriBITS. -IF (Trilinos_ENABLE_Kokkos AND TPL_ENABLE_CUDA) - SET(CUDA_LAMBDA_DEFAULT ON) -ELSEIF (KOKKOS_ENABLE_CUDA) - SET(CUDA_LAMBDA_DEFAULT ON) -ELSE() - SET(CUDA_LAMBDA_DEFAULT OFF) -ENDIF() -KOKKOS_ENABLE_OPTION(CUDA_LAMBDA ${CUDA_LAMBDA_DEFAULT} "Whether to allow lambda expressions on the device with NVCC **DEPRECATED**") +# sets it to ON. +kokkos_enable_option( + CUDA_LAMBDA ${KOKKOS_ENABLE_CUDA} "Whether to allow lambda expressions on the device with NVCC **DEPRECATED**" +) -# May be used to disable our use of CudaMallocAsync. It had caused issues in -# the past when UCX was used as MPI communication layer. We expect it is -# resolved but we keep the option around a bit longer to be safe. -KOKKOS_ENABLE_OPTION(IMPL_CUDA_MALLOC_ASYNC ON "Whether to enable CudaMallocAsync (requires CUDA Toolkit 11.2)") -KOKKOS_ENABLE_OPTION(IMPL_NVHPC_AS_DEVICE_COMPILER OFF "Whether to allow nvc++ as Cuda device compiler") -KOKKOS_ENABLE_OPTION(IMPL_CUDA_UNIFIED_MEMORY OFF "Whether to leverage unified memory architectures for CUDA") +# As of 09/2024, cudaMallocAsync causes issues with ICP and older version of UCX +# as MPI communication layer. +kokkos_enable_option(IMPL_CUDA_MALLOC_ASYNC OFF "Whether to enable CudaMallocAsync (requires CUDA Toolkit 11.2)") +kokkos_enable_option(IMPL_NVHPC_AS_DEVICE_COMPILER OFF "Whether to allow nvc++ as Cuda device compiler") +kokkos_enable_option(IMPL_CUDA_UNIFIED_MEMORY OFF "Whether to leverage unified memory architectures for CUDA") -KOKKOS_ENABLE_OPTION(DEPRECATED_CODE_4 ON "Whether code deprecated in major release 4 is available" ) -KOKKOS_ENABLE_OPTION(DEPRECATION_WARNINGS ON "Whether to emit deprecation warnings" ) -KOKKOS_ENABLE_OPTION(HIP_RELOCATABLE_DEVICE_CODE OFF "Whether to enable relocatable device code (RDC) for HIP") -KOKKOS_ENABLE_OPTION(TESTS OFF "Whether to build the unit tests") -KOKKOS_ENABLE_OPTION(BENCHMARKS OFF "Whether to build the benchmarks") -KOKKOS_ENABLE_OPTION(EXAMPLES OFF "Whether to build the examples") -STRING(TOUPPER "${CMAKE_BUILD_TYPE}" UPPERCASE_CMAKE_BUILD_TYPE) -IF(UPPERCASE_CMAKE_BUILD_TYPE STREQUAL "DEBUG") - KOKKOS_ENABLE_OPTION(DEBUG ON "Whether to activate extra debug features - may increase compile times") - KOKKOS_ENABLE_OPTION(DEBUG_DUALVIEW_MODIFY_CHECK ON "Debug check on dual views") -ELSE() - KOKKOS_ENABLE_OPTION(DEBUG OFF "Whether to activate extra debug features - may increase compile times") - KOKKOS_ENABLE_OPTION(DEBUG_DUALVIEW_MODIFY_CHECK OFF "Debug check on dual views") -ENDIF() -UNSET(_UPPERCASE_CMAKE_BUILD_TYPE) -KOKKOS_ENABLE_OPTION(LARGE_MEM_TESTS OFF "Whether to perform extra large memory tests") -KOKKOS_ENABLE_OPTION(DEBUG_BOUNDS_CHECK OFF "Whether to use bounds checking - will increase runtime") -KOKKOS_ENABLE_OPTION(COMPILER_WARNINGS OFF "Whether to print all compiler warnings") -KOKKOS_ENABLE_OPTION(TUNING OFF "Whether to create bindings for tuning tools") -KOKKOS_ENABLE_OPTION(AGGRESSIVE_VECTORIZATION OFF "Whether to aggressively vectorize loops") -KOKKOS_ENABLE_OPTION(COMPILE_AS_CMAKE_LANGUAGE OFF "Whether to use native cmake language support") -KOKKOS_ENABLE_OPTION(HIP_MULTIPLE_KERNEL_INSTANTIATIONS OFF "Whether multiple kernels are instantiated at compile time - improve performance but increase compile time") -KOKKOS_ENABLE_OPTION(IMPL_HIP_UNIFIED_MEMORY OFF "Whether to leverage unified memory architectures for HIP") +kokkos_enable_option(DEPRECATED_CODE_4 ON "Whether code deprecated in major release 4 is available") +kokkos_enable_option(DEPRECATION_WARNINGS ON "Whether to emit deprecation warnings") +kokkos_enable_option(HIP_RELOCATABLE_DEVICE_CODE OFF "Whether to enable relocatable device code (RDC) for HIP") + +# Disabling RDC only works properly since oneAPI 2024.1.0 +if(KOKKOS_ENABLE_SYCL AND KOKKOS_CXX_COMPILER_ID STREQUAL IntelLLVM AND KOKKOS_CXX_COMPILER_VERSION VERSION_LESS + 2024.1.0 +) + set(SYCL_RDC_DEFAULT ON) +else() + set(SYCL_RDC_DEFAULT OFF) +endif() +kokkos_enable_option( + SYCL_RELOCATABLE_DEVICE_CODE ${SYCL_RDC_DEFAULT} "Whether to enable relocatable device code (RDC) for SYCL" +) +kokkos_enable_option(TESTS OFF "Whether to build the unit tests") +kokkos_enable_option(BENCHMARKS OFF "Whether to build the benchmarks") +kokkos_enable_option(EXAMPLES OFF "Whether to build the examples") +string(TOUPPER "${CMAKE_BUILD_TYPE}" UPPERCASE_CMAKE_BUILD_TYPE) +if(UPPERCASE_CMAKE_BUILD_TYPE STREQUAL "DEBUG") + kokkos_enable_option(DEBUG ON "Whether to activate extra debug features - may increase compile times") + kokkos_enable_option(DEBUG_DUALVIEW_MODIFY_CHECK ON "Debug check on dual views") +else() + kokkos_enable_option(DEBUG OFF "Whether to activate extra debug features - may increase compile times") + kokkos_enable_option(DEBUG_DUALVIEW_MODIFY_CHECK OFF "Debug check on dual views") +endif() +unset(_UPPERCASE_CMAKE_BUILD_TYPE) +kokkos_enable_option(LARGE_MEM_TESTS OFF "Whether to perform extra large memory tests") +kokkos_enable_option(DEBUG_BOUNDS_CHECK OFF "Whether to use bounds checking - will increase runtime") +kokkos_enable_option(COMPILER_WARNINGS OFF "Whether to print all compiler warnings") +kokkos_enable_option(TUNING OFF "Whether to create bindings for tuning tools") +kokkos_enable_option(AGGRESSIVE_VECTORIZATION OFF "Whether to aggressively vectorize loops") +kokkos_enable_option(COMPILE_AS_CMAKE_LANGUAGE OFF "Whether to use native cmake language support") +kokkos_enable_option( + HIP_MULTIPLE_KERNEL_INSTANTIATIONS OFF + "Whether multiple kernels are instantiated at compile time - improve performance but increase compile time" +) +kokkos_enable_option(IMPL_HIP_MALLOC_ASYNC OFF "Whether to enable hipMallocAsync") +kokkos_enable_option(OPENACC_FORCE_HOST_AS_DEVICE OFF "Whether to force to use host as a target device for OpenACC") # This option will go away eventually, but allows fallback to old implementation when needed. -KOKKOS_ENABLE_OPTION(DESUL_ATOMICS_EXTERNAL OFF "Whether to use an external desul installation") -KOKKOS_ENABLE_OPTION(ATOMICS_BYPASS OFF "**NOT RECOMMENDED** Whether to make atomics non-atomic for non-threaded MPI-only use cases") -KOKKOS_ENABLE_OPTION(IMPL_REF_COUNT_BRANCH_UNLIKELY ON "Whether to use the C++20 `[[unlikely]]` attribute in the view reference counting") +kokkos_enable_option(DESUL_ATOMICS_EXTERNAL OFF "Whether to use an external desul installation") +kokkos_enable_option( + ATOMICS_BYPASS OFF "**NOT RECOMMENDED** Whether to make atomics non-atomic for non-threaded MPI-only use cases" +) +kokkos_enable_option( + IMPL_REF_COUNT_BRANCH_UNLIKELY ON "Whether to use the C++20 `[[unlikely]]` attribute in the view reference counting" +) mark_as_advanced(Kokkos_ENABLE_IMPL_REF_COUNT_BRANCH_UNLIKELY) -KOKKOS_ENABLE_OPTION(IMPL_VIEW_OF_VIEWS_DESTRUCTOR_PRECONDITION_VIOLATION_WORKAROUND OFF "Whether to enable a workaround for invalid use of View of Views that causes program hang on destruction.") +kokkos_enable_option( + IMPL_VIEW_OF_VIEWS_DESTRUCTOR_PRECONDITION_VIOLATION_WORKAROUND OFF + "Whether to enable a workaround for invalid use of View of Views that causes program hang on destruction." +) mark_as_advanced(Kokkos_ENABLE_IMPL_VIEW_OF_VIEWS_DESTRUCTOR_PRECONDITION_VIOLATION_WORKAROUND) -KOKKOS_ENABLE_OPTION(IMPL_MDSPAN ON "Whether to enable experimental mdspan support") -KOKKOS_ENABLE_OPTION(MDSPAN_EXTERNAL OFF BOOL "Whether to use an external version of mdspan") -KOKKOS_ENABLE_OPTION(IMPL_SKIP_COMPILER_MDSPAN ON BOOL "Whether to use an internal version of mdspan even if the compiler supports mdspan") +kokkos_enable_option(IMPL_MDSPAN ON "Whether to enable experimental mdspan support") +kokkos_enable_option(MDSPAN_EXTERNAL OFF BOOL "Whether to use an external version of mdspan") +kokkos_enable_option( + IMPL_SKIP_COMPILER_MDSPAN ON BOOL "Whether to use an internal version of mdspan even if the compiler supports mdspan" +) mark_as_advanced(Kokkos_ENABLE_IMPL_MDSPAN) mark_as_advanced(Kokkos_ENABLE_MDSPAN_EXTERNAL) mark_as_advanced(Kokkos_ENABLE_IMPL_SKIP_COMPILER_MDSPAN) -IF (Trilinos_ENABLE_Kokkos) - SET(COMPLEX_ALIGN_DEFAULT OFF) -ELSE() - SET(COMPLEX_ALIGN_DEFAULT ON) -ENDIF() -KOKKOS_ENABLE_OPTION(COMPLEX_ALIGN ${COMPLEX_ALIGN_DEFAULT} "Whether to align Kokkos::complex to 2*alignof(RealType)") +kokkos_enable_option(COMPLEX_ALIGN ON "Whether to align Kokkos::complex to 2*alignof(RealType)") -IF (KOKKOS_ENABLE_TESTS) - SET(HEADER_SELF_CONTAINMENT_TESTS_DEFAULT ON) -ELSE() - SET(HEADER_SELF_CONTAINMENT_TESTS_DEFAULT OFF) -ENDIF() -KOKKOS_ENABLE_OPTION(HEADER_SELF_CONTAINMENT_TESTS ${HEADER_SELF_CONTAINMENT_TESTS_DEFAULT} "Enable header self-containment unit tests") -IF (NOT KOKKOS_ENABLE_TESTS AND KOKKOS_ENABLE_HEADER_SELF_CONTAINMENT_TESTS) - MESSAGE(WARNING "Kokkos_ENABLE_HEADER_SELF_CONTAINMENT_TESTS is ON but Kokkos_ENABLE_TESTS is OFF. Option will be ignored.") -ENDIF() +if(KOKKOS_ENABLE_TESTS) + set(HEADER_SELF_CONTAINMENT_TESTS_DEFAULT ON) +else() + set(HEADER_SELF_CONTAINMENT_TESTS_DEFAULT OFF) +endif() +kokkos_enable_option( + HEADER_SELF_CONTAINMENT_TESTS ${HEADER_SELF_CONTAINMENT_TESTS_DEFAULT} "Enable header self-containment unit tests" +) +if(NOT KOKKOS_ENABLE_TESTS AND KOKKOS_ENABLE_HEADER_SELF_CONTAINMENT_TESTS) + message( + WARNING "Kokkos_ENABLE_HEADER_SELF_CONTAINMENT_TESTS is ON but Kokkos_ENABLE_TESTS is OFF. Option will be ignored." + ) +endif() -IF (KOKKOS_ENABLE_CUDA AND (KOKKOS_CXX_COMPILER_ID STREQUAL Clang)) - SET(CUDA_CONSTEXPR_DEFAULT ON) -ELSE() - SET(CUDA_CONSTEXPR_DEFAULT OFF) -ENDIF() -KOKKOS_ENABLE_OPTION(CUDA_CONSTEXPR ${CUDA_CONSTEXPR_DEFAULT} "Whether to activate experimental relaxed constexpr functions") +if(KOKKOS_ENABLE_CUDA AND (KOKKOS_CXX_COMPILER_ID STREQUAL Clang)) + set(CUDA_CONSTEXPR_DEFAULT ON) +else() + set(CUDA_CONSTEXPR_DEFAULT OFF) +endif() +kokkos_enable_option( + CUDA_CONSTEXPR ${CUDA_CONSTEXPR_DEFAULT} "Whether to activate experimental relaxed constexpr functions" +) -IF (KOKKOS_ENABLE_HPX) - SET(HPX_ASYNC_DISPATCH_DEFAULT ON) -ELSE() - SET(HPX_ASYNC_DISPATCH_DEFAULT OFF) -ENDIF() -KOKKOS_ENABLE_OPTION(IMPL_HPX_ASYNC_DISPATCH ${HPX_ASYNC_DISPATCH_DEFAULT} "Whether HPX supports asynchronous dispatch") +if(KOKKOS_ENABLE_HPX) + set(HPX_ASYNC_DISPATCH_DEFAULT ON) +else() + set(HPX_ASYNC_DISPATCH_DEFAULT OFF) +endif() +kokkos_enable_option(IMPL_HPX_ASYNC_DISPATCH ${HPX_ASYNC_DISPATCH_DEFAULT} "Whether HPX supports asynchronous dispatch") -Kokkos_ENABLE_OPTION(UNSUPPORTED_ARCHS OFF "Whether to allow architectures in backends Kokkos doesn't optimize for") +kokkos_enable_option(UNSUPPORTED_ARCHS OFF "Whether to allow architectures in backends Kokkos doesn't optimize for") -FUNCTION(check_device_specific_options) - CMAKE_PARSE_ARGUMENTS(SOME "" "DEVICE" "OPTIONS" ${ARGN}) - IF(NOT KOKKOS_ENABLE_${SOME_DEVICE}) - FOREACH(OPTION ${SOME_OPTIONS}) - IF(NOT DEFINED CACHE{Kokkos_ENABLE_${OPTION}} OR NOT DEFINED CACHE{Kokkos_ENABLE_${SOME_DEVICE}}) - MESSAGE(FATAL_ERROR "Internal logic error: option '${OPTION}' or device '${SOME_DEVICE}' not recognized.") - ENDIF() - IF(KOKKOS_ENABLE_${OPTION}) - MESSAGE(WARNING "Kokkos_ENABLE_${OPTION} is ON but ${SOME_DEVICE} backend is not enabled. Option will be ignored.") - UNSET(KOKKOS_ENABLE_${OPTION} PARENT_SCOPE) - ENDIF() - ENDFOREACH() - ENDIF() -ENDFUNCTION() +function(check_device_specific_options) + cmake_parse_arguments(SOME "" "DEVICE" "OPTIONS" ${ARGN}) + if(NOT KOKKOS_ENABLE_${SOME_DEVICE}) + foreach(OPTION ${SOME_OPTIONS}) + if(NOT DEFINED CACHE{Kokkos_ENABLE_${OPTION}} OR NOT DEFINED CACHE{Kokkos_ENABLE_${SOME_DEVICE}}) + message(FATAL_ERROR "Internal logic error: option '${OPTION}' or device '${SOME_DEVICE}' not recognized.") + endif() + if(KOKKOS_ENABLE_${OPTION}) + message( + WARNING "Kokkos_ENABLE_${OPTION} is ON but ${SOME_DEVICE} backend is not enabled. Option will be ignored." + ) + unset(KOKKOS_ENABLE_${OPTION} PARENT_SCOPE) + endif() + endforeach() + endif() +endfunction() -CHECK_DEVICE_SPECIFIC_OPTIONS(DEVICE CUDA OPTIONS CUDA_UVM CUDA_RELOCATABLE_DEVICE_CODE CUDA_LAMBDA CUDA_CONSTEXPR CUDA_LDG_INTRINSIC IMPL_CUDA_UNIFIED_MEMORY) -CHECK_DEVICE_SPECIFIC_OPTIONS(DEVICE HIP OPTIONS HIP_RELOCATABLE_DEVICE_CODE) -CHECK_DEVICE_SPECIFIC_OPTIONS(DEVICE HPX OPTIONS IMPL_HPX_ASYNC_DISPATCH) +check_device_specific_options( + DEVICE + CUDA + OPTIONS + CUDA_UVM + CUDA_RELOCATABLE_DEVICE_CODE + CUDA_LAMBDA + CUDA_CONSTEXPR + CUDA_LDG_INTRINSIC + IMPL_CUDA_MALLOC_ASYNC + IMPL_CUDA_UNIFIED_MEMORY +) +check_device_specific_options( + DEVICE HIP OPTIONS HIP_RELOCATABLE_DEVICE_CODE HIP_MULTIPLE_KERNEL_INSTANTIATIONS IMPL_HIP_MALLOC_ASYNC +) +check_device_specific_options(DEVICE HPX OPTIONS IMPL_HPX_ASYNC_DISPATCH) +check_device_specific_options(DEVICE OPENACC OPTIONS OPENACC_FORCE_HOST_AS_DEVICE) # Needed due to change from deprecated name to new header define name -IF (KOKKOS_ENABLE_AGGRESSIVE_VECTORIZATION) - SET(KOKKOS_OPT_RANGE_AGGRESSIVE_VECTORIZATION ON) -ENDIF() +if(KOKKOS_ENABLE_AGGRESSIVE_VECTORIZATION) + set(KOKKOS_OPT_RANGE_AGGRESSIVE_VECTORIZATION ON) +endif() # Force consistency of KOKKOS_ENABLE_CUDA_RELOCATABLE_DEVICE_CODE # and CMAKE_CUDA_SEPARABLE_COMPILATION when we are compiling # using the CMake CUDA language support. # Either one being on will turn the other one on. -IF (KOKKOS_COMPILE_LANGUAGE STREQUAL CUDA) - IF (KOKKOS_ENABLE_CUDA_RELOCATABLE_DEVICE_CODE) - IF (NOT CMAKE_CUDA_SEPARABLE_COMPILATION) - MESSAGE(STATUS "Setting CMAKE_CUDA_SEPARABLE_COMPILATION=ON since Kokkos_ENABLE_CUDA_RELOCATABLE_DEVICE_CODE is true. When compiling Kokkos with CMake language CUDA, please use CMAKE_CUDA_SEPARABLE_COMPILATION to control RDC support") - SET(CMAKE_CUDA_SEPARABLE_COMPILATION ON) - ENDIF() - ELSE() - IF (CMAKE_CUDA_SEPARABLE_COMPILATION) - SET(KOKKOS_ENABLE_CUDA_RELOCATABLE_DEVICE_CODE ON) - ENDIF() - ENDIF() -ENDIF() +if(KOKKOS_COMPILE_LANGUAGE STREQUAL CUDA) + if(KOKKOS_ENABLE_CUDA_RELOCATABLE_DEVICE_CODE) + if(NOT CMAKE_CUDA_SEPARABLE_COMPILATION) + message( + STATUS + "Setting CMAKE_CUDA_SEPARABLE_COMPILATION=ON since Kokkos_ENABLE_CUDA_RELOCATABLE_DEVICE_CODE is true. When compiling Kokkos with CMake language CUDA, please use CMAKE_CUDA_SEPARABLE_COMPILATION to control RDC support" + ) + set(CMAKE_CUDA_SEPARABLE_COMPILATION ON) + endif() + else() + if(CMAKE_CUDA_SEPARABLE_COMPILATION) + set(KOKKOS_ENABLE_CUDA_RELOCATABLE_DEVICE_CODE ON) + endif() + endif() +endif() # This is known to occur with Clang 9. We would need to use nvcc as the linker # http://lists.llvm.org/pipermail/cfe-dev/2018-June/058296.html # TODO: Through great effort we can use a different linker by hacking # CMAKE_CXX_LINK_EXECUTABLE in a future release -IF (KOKKOS_ENABLE_CUDA_RELOCATABLE_DEVICE_CODE AND KOKKOS_CXX_COMPILER_ID STREQUAL Clang) - MESSAGE(FATAL_ERROR "Relocatable device code is currently not supported with Clang - must use nvcc_wrapper or turn off RDC") -ENDIF() +if(KOKKOS_ENABLE_CUDA_RELOCATABLE_DEVICE_CODE AND KOKKOS_CXX_COMPILER_ID STREQUAL Clang) + message( + FATAL_ERROR "Relocatable device code is currently not supported with Clang - must use nvcc_wrapper or turn off RDC" + ) +endif() -IF (KOKKOS_ENABLE_CUDA_RELOCATABLE_DEVICE_CODE AND BUILD_SHARED_LIBS) - MESSAGE(FATAL_ERROR "Relocatable device code requires static libraries.") -ENDIF() +if(KOKKOS_ENABLE_CUDA_RELOCATABLE_DEVICE_CODE AND BUILD_SHARED_LIBS) + message(FATAL_ERROR "Relocatable device code requires static libraries.") +endif() -IF(Kokkos_ENABLE_CUDA_LDG_INTRINSIC) - IF(KOKKOS_ENABLE_DEPRECATED_CODE_4) - MESSAGE(DEPRECATION "Setting Kokkos_ENABLE_CUDA_LDG_INTRINSIC is deprecated. LDG intrinsics are always enabled.") - ELSE() - MESSAGE(FATAL_ERROR "Kokkos_ENABLE_CUDA_LDG_INTRINSIC has been removed. LDG intrinsics are always enabled.") - ENDIF() -ENDIF() -IF(Kokkos_ENABLE_CUDA AND NOT Kokkos_ENABLE_CUDA_LAMBDA) - IF(KOKKOS_ENABLE_DEPRECATED_CODE_4) - MESSAGE(DEPRECATION "Setting Kokkos_ENABLE_CUDA_LAMBDA is deprecated. Lambda expressions in device code are always enabled. Forcing -DKokkos_ENABLE_CUDA_LAMBDA=ON") +if(Kokkos_ENABLE_CUDA_LDG_INTRINSIC) + if(KOKKOS_ENABLE_DEPRECATED_CODE_4) + message(DEPRECATION "Setting Kokkos_ENABLE_CUDA_LDG_INTRINSIC is deprecated. LDG intrinsics are always enabled.") + else() + message(FATAL_ERROR "Kokkos_ENABLE_CUDA_LDG_INTRINSIC has been removed. LDG intrinsics are always enabled.") + endif() +endif() +if(Kokkos_ENABLE_CUDA AND NOT Kokkos_ENABLE_CUDA_LAMBDA) + if(KOKKOS_ENABLE_DEPRECATED_CODE_4) + message( + DEPRECATION + "Setting Kokkos_ENABLE_CUDA_LAMBDA is deprecated. Lambda expressions in device code are always enabled. Forcing -DKokkos_ENABLE_CUDA_LAMBDA=ON" + ) set(Kokkos_ENABLE_CUDA_LAMBDA ON CACHE BOOL "Kokkos turned Cuda lambda support ON!" FORCE) set(KOKKOS_ENABLE_CUDA_LAMBDA ON) - ELSE() - MESSAGE(FATAL_ERROR "Kokkos_ENABLE_CUDA_LAMBDA has been removed. Lambda expressions in device code always enabled.") - ENDIF() -ENDIF() + else() + message(FATAL_ERROR "Kokkos_ENABLE_CUDA_LAMBDA has been removed. Lambda expressions in device code always enabled.") + endif() +endif() - -IF(DEFINED Kokkos_ENABLE_IMPL_DESUL_ATOMICS) - MESSAGE(WARNING "Kokkos_ENABLE_IMPL_DESUL_ATOMICS option has been removed. Desul atomics cannot be disabled.") -ENDIF() +if(DEFINED Kokkos_ENABLE_IMPL_DESUL_ATOMICS) + message(WARNING "Kokkos_ENABLE_IMPL_DESUL_ATOMICS option has been removed. Desul atomics cannot be disabled.") +endif() diff --git a/lib/kokkos/cmake/kokkos_functions.cmake b/lib/kokkos/cmake/kokkos_functions.cmake index d1f1e0d7a7..38eedd8362 100644 --- a/lib/kokkos/cmake/kokkos_functions.cmake +++ b/lib/kokkos/cmake/kokkos_functions.cmake @@ -5,12 +5,8 @@ # Validate options are given with correct case and define an internal # upper-case version for use within -set(Kokkos_OPTIONS_NOT_TO_EXPORT - Kokkos_ENABLE_BENCHMARKS - Kokkos_ENABLE_EXAMPLES - Kokkos_ENABLE_TESTS - Kokkos_ENABLE_HEADER_SELF_CONTAINMENT_TESTS - Kokkos_ENABLE_COMPILER_WARNINGS +set(Kokkos_OPTIONS_NOT_TO_EXPORT Kokkos_ENABLE_BENCHMARKS Kokkos_ENABLE_EXAMPLES Kokkos_ENABLE_TESTS + Kokkos_ENABLE_HEADER_SELF_CONTAINMENT_TESTS Kokkos_ENABLE_COMPILER_WARNINGS ) # @@ -22,139 +18,122 @@ set(Kokkos_OPTIONS_NOT_TO_EXPORT # It attempts to print a helpful message about updating the options for the new CMake. # Kokkos_${SUFFIX} is the name of the option (like Kokkos_ARCH) being checked. # Kokkos_${PREFIX}_X is the name of new option to be defined from a list X,Y,Z,... -FUNCTION(kokkos_deprecated_list SUFFIX PREFIX) - SET(CAMEL_NAME Kokkos_${SUFFIX}) - STRING(TOUPPER ${CAMEL_NAME} UC_NAME) +function(kokkos_deprecated_list SUFFIX PREFIX) + set(CAMEL_NAME Kokkos_${SUFFIX}) + string(TOUPPER ${CAMEL_NAME} UC_NAME) #I don't love doing it this way but better to be safe - FOREACH(opt ${KOKKOS_GIVEN_VARIABLES}) - STRING(TOUPPER ${opt} OPT_UC) - IF ("${OPT_UC}" STREQUAL "${UC_NAME}") - STRING(REPLACE "," ";" optlist "${${opt}}") - SET(ERROR_MSG "Given deprecated option list ${opt}. This must now be given as separate -D options, which assuming you spelled options correctly would be:") - FOREACH(entry ${optlist}) - STRING(TOUPPER ${entry} ENTRY_UC) - STRING(APPEND ERROR_MSG "\n -DKokkos_${PREFIX}_${ENTRY_UC}=ON") - ENDFOREACH() - STRING(APPEND ERROR_MSG "\nRemove CMakeCache.txt and re-run. For a list of valid options, refer to BUILD.md or even look at CMakeCache.txt (before deleting it).") - IF (KOKKOS_HAS_TRILINOS) - MESSAGE(WARNING ${ERROR_MSG}) - FOREACH(entry ${optlist}) - STRING(TOUPPER ${entry} ENTRY_UC) - SET(${CAMEL_NAME}_${ENTRY_UC} ON CACHE BOOL "Deprecated Trilinos translation") - ENDFOREACH() - UNSET(${opt} CACHE) - ELSE() - MESSAGE(SEND_ERROR ${ERROR_MSG}) - ENDIF() - ENDIF() - ENDFOREACH() -ENDFUNCTION() + foreach(opt ${KOKKOS_GIVEN_VARIABLES}) + string(TOUPPER ${opt} OPT_UC) + if("${OPT_UC}" STREQUAL "${UC_NAME}") + string(REPLACE "," ";" optlist "${${opt}}") + set(ERROR_MSG + "Given deprecated option list ${opt}. This must now be given as separate -D options, which assuming you spelled options correctly would be:" + ) + foreach(entry ${optlist}) + string(TOUPPER ${entry} ENTRY_UC) + string(APPEND ERROR_MSG "\n -DKokkos_${PREFIX}_${ENTRY_UC}=ON") + endforeach() + string( + APPEND + ERROR_MSG + "\nRemove CMakeCache.txt and re-run. For a list of valid options, refer to BUILD.md or even look at CMakeCache.txt (before deleting it)." + ) + message(SEND_ERROR ${ERROR_MSG}) + endif() + endforeach() +endfunction() -FUNCTION(kokkos_option CAMEL_SUFFIX DEFAULT TYPE DOCSTRING) - SET(CAMEL_NAME Kokkos_${CAMEL_SUFFIX}) - STRING(TOUPPER ${CAMEL_NAME} UC_NAME) +function(kokkos_option CAMEL_SUFFIX DEFAULT TYPE DOCSTRING) + set(CAMEL_NAME Kokkos_${CAMEL_SUFFIX}) + string(TOUPPER ${CAMEL_NAME} UC_NAME) - LIST(APPEND KOKKOS_OPTION_KEYS ${CAMEL_SUFFIX}) - SET(KOKKOS_OPTION_KEYS ${KOKKOS_OPTION_KEYS} PARENT_SCOPE) - LIST(APPEND KOKKOS_OPTION_VALUES "${DOCSTRING}") - SET(KOKKOS_OPTION_VALUES ${KOKKOS_OPTION_VALUES} PARENT_SCOPE) - LIST(APPEND KOKKOS_OPTION_TYPES ${TYPE}) - SET(KOKKOS_OPTION_TYPES ${KOKKOS_OPTION_TYPES} PARENT_SCOPE) + list(APPEND KOKKOS_OPTION_KEYS ${CAMEL_SUFFIX}) + set(KOKKOS_OPTION_KEYS ${KOKKOS_OPTION_KEYS} PARENT_SCOPE) + list(APPEND KOKKOS_OPTION_VALUES "${DOCSTRING}") + set(KOKKOS_OPTION_VALUES ${KOKKOS_OPTION_VALUES} PARENT_SCOPE) + list(APPEND KOKKOS_OPTION_TYPES ${TYPE}) + set(KOKKOS_OPTION_TYPES ${KOKKOS_OPTION_TYPES} PARENT_SCOPE) # Make sure this appears in the cache with the appropriate DOCSTRING - SET(${CAMEL_NAME} ${DEFAULT} CACHE ${TYPE} ${DOCSTRING}) - - IF (KOKKOS_HAS_TRILINOS) - IF (NOT CAMEL_NAME IN_LIST Kokkos_OPTIONS_NOT_TO_EXPORT) - TRIBITS_PKG_EXPORT_CACHE_VAR(${CAMEL_NAME}) - ENDIF() - ENDIF() + set(${CAMEL_NAME} ${DEFAULT} CACHE ${TYPE} ${DOCSTRING}) #I don't love doing it this way because it's N^2 in number options, but c'est la vie - FOREACH(opt ${KOKKOS_GIVEN_VARIABLES}) - STRING(TOUPPER ${opt} OPT_UC) - IF ("${OPT_UC}" STREQUAL "${UC_NAME}") - IF (NOT "${opt}" STREQUAL "${CAMEL_NAME}") - IF (KOKKOS_HAS_TRILINOS) - #Allow this for now if Trilinos... we need to bootstrap our way to integration - MESSAGE(WARNING "Deprecated option ${opt} found - please change spelling to ${CAMEL_NAME}") - SET(${CAMEL_NAME} "${${opt}}" CACHE ${TYPE} ${DOCSTRING} FORCE) - UNSET(${opt} CACHE) - ELSE() - MESSAGE(FATAL_ERROR "Matching option found for ${CAMEL_NAME} with the wrong case ${opt}. Please delete your CMakeCache.txt and change option to -D${CAMEL_NAME}=${${opt}}. This is now enforced to avoid hard-to-debug CMake cache inconsistencies.") - ENDIF() - ENDIF() - ENDIF() - ENDFOREACH() + foreach(opt ${KOKKOS_GIVEN_VARIABLES}) + string(TOUPPER ${opt} OPT_UC) + if("${OPT_UC}" STREQUAL "${UC_NAME}") + if(NOT "${opt}" STREQUAL "${CAMEL_NAME}") + message( + FATAL_ERROR + "Matching option found for ${CAMEL_NAME} with the wrong case ${opt}. Please delete your CMakeCache.txt and change option to -D${CAMEL_NAME}=${${opt}}. This is now enforced to avoid hard-to-debug CMake cache inconsistencies." + ) + endif() + endif() + endforeach() #okay, great, we passed the validation test - use the default - IF (DEFINED ${CAMEL_NAME}) - SET(${UC_NAME} ${${CAMEL_NAME}} PARENT_SCOPE) - ELSE() - SET(${UC_NAME} ${DEFAULT} PARENT_SCOPE) - ENDIF() -ENDFUNCTION() + if(DEFINED ${CAMEL_NAME}) + set(${UC_NAME} ${${CAMEL_NAME}} PARENT_SCOPE) + else() + set(${UC_NAME} ${DEFAULT} PARENT_SCOPE) + endif() +endfunction() -INCLUDE (CMakeDependentOption) -FUNCTION(kokkos_dependent_option CAMEL_SUFFIX DOCSTRING DEFAULT DEPENDENCY FORCE) - SET(CAMEL_NAME Kokkos_${CAMEL_SUFFIX}) - STRING(TOUPPER ${CAMEL_NAME} UC_NAME) +include(CMakeDependentOption) +function(kokkos_dependent_option CAMEL_SUFFIX DOCSTRING DEFAULT DEPENDENCY FORCE) + set(CAMEL_NAME Kokkos_${CAMEL_SUFFIX}) + string(TOUPPER ${CAMEL_NAME} UC_NAME) - LIST(APPEND KOKKOS_OPTION_KEYS ${CAMEL_SUFFIX}) - SET(KOKKOS_OPTION_KEYS ${KOKKOS_OPTION_KEYS} PARENT_SCOPE) - LIST(APPEND KOKKOS_OPTION_VALUES "${DOCSTRING}") - SET(KOKKOS_OPTION_VALUES ${KOKKOS_OPTION_VALUES} PARENT_SCOPE) - LIST(APPEND KOKKOS_OPTION_TYPES BOOL) - SET(KOKKOS_OPTION_TYPES ${KOKKOS_OPTION_TYPES} PARENT_SCOPE) + list(APPEND KOKKOS_OPTION_KEYS ${CAMEL_SUFFIX}) + set(KOKKOS_OPTION_KEYS ${KOKKOS_OPTION_KEYS} PARENT_SCOPE) + list(APPEND KOKKOS_OPTION_VALUES "${DOCSTRING}") + set(KOKKOS_OPTION_VALUES ${KOKKOS_OPTION_VALUES} PARENT_SCOPE) + list(APPEND KOKKOS_OPTION_TYPES BOOL) + set(KOKKOS_OPTION_TYPES ${KOKKOS_OPTION_TYPES} PARENT_SCOPE) - CMAKE_DEPENDENT_OPTION(${CAMEL_NAME} ${DOCSTRING} ${DEFAULT} "${DEPENDENCY}" ${FORCE}) + cmake_dependent_option(${CAMEL_NAME} ${DOCSTRING} ${DEFAULT} "${DEPENDENCY}" ${FORCE}) #I don't love doing it this way because it's N^2 in number options, but c'est la vie - FOREACH(opt ${KOKKOS_GIVEN_VARIABLES}) - STRING(TOUPPER ${opt} OPT_UC) - IF ("${OPT_UC}" STREQUAL "${UC_NAME}") - IF (NOT "${opt}" STREQUAL "${CAMEL_NAME}") - IF (KOKKOS_HAS_TRILINOS) - #Allow this for now if Trilinos... we need to bootstrap our way to integration - MESSAGE(WARNING "Deprecated option ${opt} found - please change spelling to ${CAMEL_NAME}") - SET(${CAMEL_NAME} "${${opt}}" CACHE ${TYPE} ${DOCSTRING} FORCE) - UNSET(${opt} CACHE) - ELSE() - MESSAGE(FATAL_ERROR "Matching option found for ${CAMEL_NAME} with the wrong case ${opt}. Please delete your CMakeCache.txt and change option to -D${CAMEL_NAME}=${${opt}}. This is now enforced to avoid hard-to-debug CMake cache inconsistencies.") - ENDIF() - ENDIF() - ENDIF() - ENDFOREACH() + foreach(opt ${KOKKOS_GIVEN_VARIABLES}) + string(TOUPPER ${opt} OPT_UC) + if("${OPT_UC}" STREQUAL "${UC_NAME}") + if(NOT "${opt}" STREQUAL "${CAMEL_NAME}") + message( + FATAL_ERROR + "Matching option found for ${CAMEL_NAME} with the wrong case ${opt}. Please delete your CMakeCache.txt and change option to -D${CAMEL_NAME}=${${opt}}. This is now enforced to avoid hard-to-debug CMake cache inconsistencies." + ) + endif() + endif() + endforeach() #okay, great, we passed the validation test - use the default - IF (DEFINED ${CAMEL_NAME}) - SET(${UC_NAME} ${${CAMEL_NAME}} PARENT_SCOPE) - ELSE() - SET(${UC_NAME} ${DEFAULT} PARENT_SCOPE) - ENDIF() -ENDFUNCTION() + if(DEFINED ${CAMEL_NAME}) + set(${UC_NAME} ${${CAMEL_NAME}} PARENT_SCOPE) + else() + set(${UC_NAME} ${DEFAULT} PARENT_SCOPE) + endif() +endfunction() -FUNCTION(kokkos_set_option CAMEL_SUFFIX VALUE) - LIST(FIND KOKKOS_OPTION_KEYS ${CAMEL_SUFFIX} OPTION_INDEX) - IF(OPTION_INDEX EQUAL -1) - MESSAGE(FATAL_ERROR "Couldn't set value for Kokkos_${CAMEL_SUFFIX}") - ENDIF() - SET(CAMEL_NAME Kokkos_${CAMEL_SUFFIX}) - STRING(TOUPPER ${CAMEL_NAME} UC_NAME) +function(kokkos_set_option CAMEL_SUFFIX VALUE) + list(FIND KOKKOS_OPTION_KEYS ${CAMEL_SUFFIX} OPTION_INDEX) + if(OPTION_INDEX EQUAL -1) + message(FATAL_ERROR "Couldn't set value for Kokkos_${CAMEL_SUFFIX}") + endif() + set(CAMEL_NAME Kokkos_${CAMEL_SUFFIX}) + string(TOUPPER ${CAMEL_NAME} UC_NAME) - LIST(GET KOKKOS_OPTION_VALUES ${OPTION_INDEX} DOCSTRING) - LIST(GET KOKKOS_OPTION_TYPES ${OPTION_INDEX} TYPE) - SET(${CAMEL_NAME} ${VALUE} CACHE ${TYPE} ${DOCSTRING} FORCE) - MESSAGE(STATUS "Setting ${CAMEL_NAME}=${VALUE}") - SET(${UC_NAME} ${VALUE} PARENT_SCOPE) -ENDFUNCTION() + list(GET KOKKOS_OPTION_VALUES ${OPTION_INDEX} DOCSTRING) + list(GET KOKKOS_OPTION_TYPES ${OPTION_INDEX} TYPE) + set(${CAMEL_NAME} ${VALUE} CACHE ${TYPE} ${DOCSTRING} FORCE) + message(STATUS "Setting ${CAMEL_NAME}=${VALUE}") + set(${UC_NAME} ${VALUE} PARENT_SCOPE) +endfunction() -FUNCTION(kokkos_append_config_line LINE) - GLOBAL_APPEND(KOKKOS_TPL_EXPORTS "${LINE}") -ENDFUNCTION() +function(kokkos_append_config_line LINE) + global_append(KOKKOS_TPL_EXPORTS "${LINE}") +endfunction() -MACRO(kokkos_export_cmake_tpl NAME) +macro(kokkos_export_cmake_tpl NAME) cmake_parse_arguments(KOKKOS_EXTRA_ARG "REQUIRED" "" "COMPONENTS" ${ARGN}) #CMake TPLs are located with a call to find_package @@ -163,91 +142,88 @@ MACRO(kokkos_export_cmake_tpl NAME) #If Kokkos was configured to find the TPL through a _DIR variable #make sure thar DIR variable is available to downstream packages - IF (DEFINED ${NAME}_DIR) + if(DEFINED ${NAME}_DIR) #The downstream project may override the TPL location that Kokkos used #Check if the downstream project chose its own TPL location #If not, make the Kokkos found location available - KOKKOS_APPEND_CONFIG_LINE("IF(NOT DEFINED ${NAME}_DIR)") - KOKKOS_APPEND_CONFIG_LINE(" SET(${NAME}_DIR ${${NAME}_DIR})") - KOKKOS_APPEND_CONFIG_LINE("ENDIF()") - ENDIF() + kokkos_append_config_line("IF(NOT DEFINED ${NAME}_DIR)") + kokkos_append_config_line(" SET(${NAME}_DIR ${${NAME}_DIR})") + kokkos_append_config_line("ENDIF()") + endif() - IF (DEFINED ${NAME}_ROOT) + if(DEFINED ${NAME}_ROOT) #The downstream project may override the TPL location that Kokkos used #Check if the downstream project chose its own TPL location #If not, make the Kokkos found location available - KOKKOS_APPEND_CONFIG_LINE("IF(NOT DEFINED ${NAME}_ROOT)") - KOKKOS_APPEND_CONFIG_LINE(" SET(${NAME}_ROOT ${${NAME}_ROOT})") - KOKKOS_APPEND_CONFIG_LINE("ENDIF()") - ENDIF() - SET(KOKKOS_CONFIG_STRING "FIND_DEPENDENCY(${NAME}") + kokkos_append_config_line("IF(NOT DEFINED ${NAME}_ROOT)") + kokkos_append_config_line(" SET(${NAME}_ROOT ${${NAME}_ROOT})") + kokkos_append_config_line("ENDIF()") + endif() + set(KOKKOS_CONFIG_STRING "FIND_DEPENDENCY(${NAME}") - IF(KOKKOS_EXTRA_ARG_REQUIRED) - STRING(APPEND KOKKOS_CONFIG_STRING " REQUIRED") - ENDIF() - IF(KOKKOS_EXTRA_ARG_COMPONENTS) - STRING(APPEND KOKKOS_CONFIG_STRING " COMPONENTS ${KOKKOS_EXTRA_ARG_COMPONENTS}") - ENDIF() - STRING(APPEND KOKKOS_CONFIG_STRING ")") - KOKKOS_APPEND_CONFIG_LINE(${KOKKOS_CONFIG_STRING}) -ENDMACRO() + if(KOKKOS_EXTRA_ARG_REQUIRED) + string(APPEND KOKKOS_CONFIG_STRING " REQUIRED") + endif() + if(KOKKOS_EXTRA_ARG_COMPONENTS) + string(APPEND KOKKOS_CONFIG_STRING " COMPONENTS ${KOKKOS_EXTRA_ARG_COMPONENTS}") + endif() + string(APPEND KOKKOS_CONFIG_STRING ")") + kokkos_append_config_line(${KOKKOS_CONFIG_STRING}) +endmacro() -MACRO(kokkos_export_imported_tpl NAME) - IF (NOT KOKKOS_HAS_TRILINOS) - GET_TARGET_PROPERTY(LIB_IMPORTED ${NAME} IMPORTED) - IF (NOT LIB_IMPORTED) - # This is not an imported target - # This an interface library that we created - INSTALL( - TARGETS ${NAME} - EXPORT KokkosTargets - RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} - LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} - ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR} - ) - ELSE() - #make sure this also gets "exported" in the config file - KOKKOS_APPEND_CONFIG_LINE("IF(NOT TARGET ${NAME})") +macro(kokkos_export_imported_tpl NAME) + get_target_property(LIB_IMPORTED ${NAME} IMPORTED) + if(NOT LIB_IMPORTED) + # This is not an imported target + # This an interface library that we created + install( + TARGETS ${NAME} + EXPORT KokkosTargets + RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} + LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} + ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR} + ) + else() + #make sure this also gets "exported" in the config file + kokkos_append_config_line("IF(NOT TARGET ${NAME})") - GET_TARGET_PROPERTY(LIB_TYPE ${NAME} TYPE) - IF (${LIB_TYPE} STREQUAL "INTERFACE_LIBRARY") - KOKKOS_APPEND_CONFIG_LINE("ADD_LIBRARY(${NAME} INTERFACE IMPORTED)") - KOKKOS_APPEND_CONFIG_LINE("SET_TARGET_PROPERTIES(${NAME} PROPERTIES") - ELSE() - KOKKOS_APPEND_CONFIG_LINE("ADD_LIBRARY(${NAME} UNKNOWN IMPORTED)") - KOKKOS_APPEND_CONFIG_LINE("SET_TARGET_PROPERTIES(${NAME} PROPERTIES") - GET_TARGET_PROPERTY(TPL_LIBRARY ${NAME} IMPORTED_LOCATION) - IF(TPL_LIBRARY) - KOKKOS_APPEND_CONFIG_LINE("IMPORTED_LOCATION \"${TPL_LIBRARY}\"") - ENDIF() - ENDIF() + get_target_property(LIB_TYPE ${NAME} TYPE) + if(${LIB_TYPE} STREQUAL "INTERFACE_LIBRARY") + kokkos_append_config_line("ADD_LIBRARY(${NAME} INTERFACE IMPORTED)") + kokkos_append_config_line("SET_TARGET_PROPERTIES(${NAME} PROPERTIES") + else() + kokkos_append_config_line("ADD_LIBRARY(${NAME} UNKNOWN IMPORTED)") + kokkos_append_config_line("SET_TARGET_PROPERTIES(${NAME} PROPERTIES") + get_target_property(TPL_LIBRARY ${NAME} IMPORTED_LOCATION) + if(TPL_LIBRARY) + kokkos_append_config_line("IMPORTED_LOCATION \"${TPL_LIBRARY}\"") + endif() + endif() - GET_TARGET_PROPERTY(TPL_INCLUDES ${NAME} INTERFACE_INCLUDE_DIRECTORIES) - IF(TPL_INCLUDES) - KOKKOS_APPEND_CONFIG_LINE("INTERFACE_INCLUDE_DIRECTORIES \"${TPL_INCLUDES}\"") - ENDIF() + get_target_property(TPL_INCLUDES ${NAME} INTERFACE_INCLUDE_DIRECTORIES) + if(TPL_INCLUDES) + kokkos_append_config_line("INTERFACE_INCLUDE_DIRECTORIES \"${TPL_INCLUDES}\"") + endif() - GET_TARGET_PROPERTY(TPL_COMPILE_OPTIONS ${NAME} INTERFACE_COMPILE_OPTIONS) - IF(TPL_COMPILE_OPTIONS) - KOKKOS_APPEND_CONFIG_LINE("INTERFACE_COMPILE_OPTIONS ${TPL_COMPILE_OPTIONS}") - ENDIF() + get_target_property(TPL_COMPILE_OPTIONS ${NAME} INTERFACE_COMPILE_OPTIONS) + if(TPL_COMPILE_OPTIONS) + kokkos_append_config_line("INTERFACE_COMPILE_OPTIONS ${TPL_COMPILE_OPTIONS}") + endif() - SET(TPL_LINK_OPTIONS) - GET_TARGET_PROPERTY(TPL_LINK_OPTIONS ${NAME} INTERFACE_LINK_OPTIONS) - IF(TPL_LINK_OPTIONS) - KOKKOS_APPEND_CONFIG_LINE("INTERFACE_LINK_OPTIONS ${TPL_LINK_OPTIONS}") - ENDIF() - - GET_TARGET_PROPERTY(TPL_LINK_LIBRARIES ${NAME} INTERFACE_LINK_LIBRARIES) - IF(TPL_LINK_LIBRARIES) - KOKKOS_APPEND_CONFIG_LINE("INTERFACE_LINK_LIBRARIES \"${TPL_LINK_LIBRARIES}\"") - ENDIF() - KOKKOS_APPEND_CONFIG_LINE(")") - KOKKOS_APPEND_CONFIG_LINE("ENDIF()") - ENDIF() - ENDIF() -ENDMACRO() + set(TPL_LINK_OPTIONS) + get_target_property(TPL_LINK_OPTIONS ${NAME} INTERFACE_LINK_OPTIONS) + if(TPL_LINK_OPTIONS) + kokkos_append_config_line("INTERFACE_LINK_OPTIONS ${TPL_LINK_OPTIONS}") + endif() + get_target_property(TPL_LINK_LIBRARIES ${NAME} INTERFACE_LINK_LIBRARIES) + if(TPL_LINK_LIBRARIES) + kokkos_append_config_line("INTERFACE_LINK_LIBRARIES \"${TPL_LINK_LIBRARIES}\"") + endif() + kokkos_append_config_line(")") + kokkos_append_config_line("ENDIF()") + endif() +endmacro() # # @MACRO: KOKKOS_IMPORT_TPL() @@ -271,57 +247,43 @@ ENDMACRO() # # If specified, this TPL will build an INTERFACE library rather than an # IMPORTED target -IF (KOKKOS_HAS_TRILINOS) -MACRO(kokkos_import_tpl NAME) - #do nothing -ENDMACRO() -ELSE() -MACRO(kokkos_import_tpl NAME) - CMAKE_PARSE_ARGUMENTS(TPL - "NO_EXPORT;INTERFACE" - "" - "" - ${ARGN}) - IF (TPL_INTERFACE) - SET(TPL_IMPORTED_NAME ${NAME}) - ELSE() - SET(TPL_IMPORTED_NAME Kokkos::${NAME}) - ENDIF() +macro(kokkos_import_tpl NAME) + cmake_parse_arguments(TPL "NO_EXPORT;INTERFACE" "" "" ${ARGN}) + if(TPL_INTERFACE) + set(TPL_IMPORTED_NAME ${NAME}) + else() + set(TPL_IMPORTED_NAME Kokkos::${NAME}) + endif() - IF (KOKKOS_ENABLE_${NAME}) + if(KOKKOS_ENABLE_${NAME}) #Tack on a TPL here to make sure we avoid using anyone else's find - FIND_PACKAGE(TPL${NAME} REQUIRED MODULE) - IF(NOT TARGET ${TPL_IMPORTED_NAME}) - MESSAGE(FATAL_ERROR "Find module succeeded for ${NAME}, but did not produce valid target ${TPL_IMPORTED_NAME}") - ENDIF() - IF(NOT TPL_NO_EXPORT) - GET_TARGET_PROPERTY(TPL_ORIGINAL_NAME ${TPL_IMPORTED_NAME} ALIASED_TARGET) - IF (NOT TPL_ORIGINAL_NAME) - SET(TPL_ORIGINAL_NAME ${TPL_IMPORTED_NAME}) - ENDIF() - KOKKOS_EXPORT_IMPORTED_TPL(${TPL_ORIGINAL_NAME}) - ENDIF() - LIST(APPEND KOKKOS_ENABLED_TPLS ${NAME}) - ENDIF() -ENDMACRO(kokkos_import_tpl) -ENDIF() + find_package(TPL${NAME} REQUIRED MODULE) + if(NOT TARGET ${TPL_IMPORTED_NAME}) + message(FATAL_ERROR "Find module succeeded for ${NAME}, but did not produce valid target ${TPL_IMPORTED_NAME}") + endif() + if(NOT TPL_NO_EXPORT) + get_target_property(TPL_ORIGINAL_NAME ${TPL_IMPORTED_NAME} ALIASED_TARGET) + if(NOT TPL_ORIGINAL_NAME) + set(TPL_ORIGINAL_NAME ${TPL_IMPORTED_NAME}) + endif() + kokkos_export_imported_tpl(${TPL_ORIGINAL_NAME}) + endif() + list(APPEND KOKKOS_ENABLED_TPLS ${NAME}) + endif() +endmacro(kokkos_import_tpl) -MACRO(kokkos_import_cmake_tpl MODULE_NAME) +macro(kokkos_import_cmake_tpl MODULE_NAME) kokkos_import_tpl(${MODULE_NAME} ${ARGN} NO_EXPORT) - CMAKE_PARSE_ARGUMENTS(TPL - "NO_EXPORT" - "OPTION_NAME" - "" - ${ARGN}) + cmake_parse_arguments(TPL "NO_EXPORT" "OPTION_NAME" "" ${ARGN}) - IF (NOT TPL_OPTION_NAME) - SET(TPL_OPTION_NAME ${MODULE_NAME}) - ENDIF() + if(NOT TPL_OPTION_NAME) + set(TPL_OPTION_NAME ${MODULE_NAME}) + endif() - IF (NOT TPL_NO_EXPORT) - KOKKOS_EXPORT_CMAKE_TPL(${MODULE_NAME}) - ENDIF() -ENDMACRO() + if(NOT TPL_NO_EXPORT) + kokkos_export_cmake_tpl(${MODULE_NAME}) + endif() +endmacro() # # @MACRO: KOKKOS_CREATE_IMPORTED_TPL() @@ -368,68 +330,57 @@ ENDMACRO() # # If specified, this gives a list of linker flags that must be used # for using this library. -MACRO(kokkos_create_imported_tpl NAME) - CMAKE_PARSE_ARGUMENTS(TPL - "INTERFACE" - "LIBRARY" - "LINK_LIBRARIES;INCLUDES;COMPILE_DEFINITIONS;COMPILE_OPTIONS;LINK_OPTIONS" - ${ARGN}) +macro(kokkos_create_imported_tpl NAME) + cmake_parse_arguments( + TPL "INTERFACE" "LIBRARY" "LINK_LIBRARIES;INCLUDES;COMPILE_DEFINITIONS;COMPILE_OPTIONS;LINK_OPTIONS" ${ARGN} + ) - - IF (KOKKOS_HAS_TRILINOS) - #TODO: we need to set a bunch of cache variables here - ELSEIF (TPL_INTERFACE) - ADD_LIBRARY(${NAME} INTERFACE) + if(TPL_INTERFACE) + add_library(${NAME} INTERFACE) #Give this an importy-looking name - ADD_LIBRARY(Kokkos::${NAME} ALIAS ${NAME}) - IF (TPL_LIBRARY) - MESSAGE(SEND_ERROR "TPL Interface library ${NAME} should not have an IMPORTED_LOCATION") - ENDIF() + add_library(Kokkos::${NAME} ALIAS ${NAME}) + if(TPL_LIBRARY) + message(SEND_ERROR "TPL Interface library ${NAME} should not have an IMPORTED_LOCATION") + endif() #Things have to go in quoted in case we have multiple list entries - IF(TPL_LINK_LIBRARIES) - TARGET_LINK_LIBRARIES(${NAME} INTERFACE ${TPL_LINK_LIBRARIES}) - ENDIF() - IF(TPL_INCLUDES) - TARGET_INCLUDE_DIRECTORIES(${NAME} INTERFACE ${TPL_INCLUDES}) - ENDIF() - IF(TPL_COMPILE_DEFINITIONS) - TARGET_COMPILE_DEFINITIONS(${NAME} INTERFACE ${TPL_COMPILE_DEFINITIONS}) - ENDIF() - IF(TPL_COMPILE_OPTIONS) - TARGET_COMPILE_OPTIONS(${NAME} INTERFACE ${TPL_COMPILE_OPTIONS}) - ENDIF() - IF(TPL_LINK_OPTIONS) - TARGET_LINK_LIBRARIES(${NAME} INTERFACE ${TPL_LINK_OPTIONS}) - ENDIF() - ELSE() - ADD_LIBRARY(${NAME} UNKNOWN IMPORTED) - IF(TPL_LIBRARY) - SET_TARGET_PROPERTIES(${NAME} PROPERTIES - IMPORTED_LOCATION ${TPL_LIBRARY}) - ENDIF() + if(TPL_LINK_LIBRARIES) + target_link_libraries(${NAME} INTERFACE ${TPL_LINK_LIBRARIES}) + endif() + if(TPL_INCLUDES) + target_include_directories(${NAME} INTERFACE ${TPL_INCLUDES}) + endif() + if(TPL_COMPILE_DEFINITIONS) + target_compile_definitions(${NAME} INTERFACE ${TPL_COMPILE_DEFINITIONS}) + endif() + if(TPL_COMPILE_OPTIONS) + target_compile_options(${NAME} INTERFACE ${TPL_COMPILE_OPTIONS}) + endif() + if(TPL_LINK_OPTIONS) + target_link_libraries(${NAME} INTERFACE ${TPL_LINK_OPTIONS}) + endif() + else() + add_library(${NAME} UNKNOWN IMPORTED) + if(TPL_LIBRARY) + set_target_properties(${NAME} PROPERTIES IMPORTED_LOCATION ${TPL_LIBRARY}) + endif() #Things have to go in quoted in case we have multiple list entries - IF(TPL_LINK_LIBRARIES) - SET_TARGET_PROPERTIES(${NAME} PROPERTIES - INTERFACE_LINK_LIBRARIES "${TPL_LINK_LIBRARIES}") - ENDIF() - IF(TPL_INCLUDES) - SET_TARGET_PROPERTIES(${NAME} PROPERTIES - INTERFACE_INCLUDE_DIRECTORIES "${TPL_INCLUDES}") - ENDIF() - IF(TPL_COMPILE_DEFINITIONS) - SET_TARGET_PROPERTIES(${NAME} PROPERTIES - INTERFACE_COMPILE_DEFINITIONS "${TPL_COMPILE_DEFINITIONS}") - ENDIF() - IF(TPL_COMPILE_OPTIONS) - SET_TARGET_PROPERTIES(${NAME} PROPERTIES - INTERFACE_COMPILE_OPTIONS "${TPL_COMPILE_OPTIONS}") - ENDIF() - IF(TPL_LINK_OPTIONS) - SET_TARGET_PROPERTIES(${NAME} PROPERTIES - INTERFACE_LINK_LIBRARIES "${TPL_LINK_OPTIONS}") - ENDIF() - ENDIF() -ENDMACRO() + if(TPL_LINK_LIBRARIES) + set_target_properties(${NAME} PROPERTIES INTERFACE_LINK_LIBRARIES "${TPL_LINK_LIBRARIES}") + endif() + if(TPL_INCLUDES) + set_target_properties(${NAME} PROPERTIES INTERFACE_INCLUDE_DIRECTORIES "${TPL_INCLUDES}") + endif() + if(TPL_COMPILE_DEFINITIONS) + set_target_properties(${NAME} PROPERTIES INTERFACE_COMPILE_DEFINITIONS "${TPL_COMPILE_DEFINITIONS}") + endif() + if(TPL_COMPILE_OPTIONS) + set_target_properties(${NAME} PROPERTIES INTERFACE_COMPILE_OPTIONS "${TPL_COMPILE_OPTIONS}") + endif() + if(TPL_LINK_OPTIONS) + set_target_properties(${NAME} PROPERTIES INTERFACE_LINK_LIBRARIES "${TPL_LINK_OPTIONS}") + endif() + endif() +endmacro() # # @MACRO: KOKKOS_FIND_HEADER @@ -479,37 +430,32 @@ ENDMACRO() # # Custom paths to search for the header # -MACRO(kokkos_find_header VAR_NAME HEADER TPL_NAME) - CMAKE_PARSE_ARGUMENTS(TPL - "ALLOW_SYSTEM_PATH_FALLBACK" - "" - "PATHS" - ${ARGN}) +macro(kokkos_find_header VAR_NAME HEADER TPL_NAME) + cmake_parse_arguments(TPL "ALLOW_SYSTEM_PATH_FALLBACK" "" "PATHS" ${ARGN}) - SET(${VAR_NAME} "${VARNAME}-NOTFOUND") - SET(HAVE_CUSTOM_PATHS FALSE) + set(${VAR_NAME} "${VARNAME}-NOTFOUND") + set(HAVE_CUSTOM_PATHS FALSE) - IF(DEFINED ${TPL_NAME}_ROOT OR - DEFINED ENV{${TPL_NAME}_ROOT} OR - DEFINED KOKKOS_${TPL_NAME}_DIR OR - TPL_PATHS) - FIND_PATH(${VAR_NAME} ${HEADER} - PATHS - ${${TPL_NAME}_ROOT} - $ENV{${TPL_NAME}_ROOT} - ${KOKKOS_${TPL_NAME}_DIR} - ${TPL_PATHS} + if(DEFINED ${TPL_NAME}_ROOT + OR DEFINED ENV{${TPL_NAME}_ROOT} + OR DEFINED KOKKOS_${TPL_NAME}_DIR + OR TPL_PATHS + ) + find_path( + ${VAR_NAME} ${HEADER} + PATHS ${${TPL_NAME}_ROOT} $ENV{${TPL_NAME}_ROOT} ${KOKKOS_${TPL_NAME}_DIR} ${TPL_PATHS} PATH_SUFFIXES include - NO_DEFAULT_PATH) - SET(HAVE_CUSTOM_PATHS TRUE) - ENDIF() + NO_DEFAULT_PATH + ) + set(HAVE_CUSTOM_PATHS TRUE) + endif() - IF(NOT HAVE_CUSTOM_PATHS OR TPL_ALLOW_SYSTEM_PATH_FALLBACK) + if(NOT HAVE_CUSTOM_PATHS OR TPL_ALLOW_SYSTEM_PATH_FALLBACK) #No-op if ${VAR_NAME} set by previous call - FIND_PATH(${VAR_NAME} ${HEADER}) - ENDIF() + find_path(${VAR_NAME} ${HEADER}) + endif() -ENDMACRO() +endmacro() # # @MACRO: KOKKOS_FIND_LIBRARY @@ -565,42 +511,36 @@ ENDMACRO() # Suffixes appended to PATHS when attempting to locate # the library. Defaults to {lib, lib64}. # -MACRO(kokkos_find_library VAR_NAME LIB TPL_NAME) - CMAKE_PARSE_ARGUMENTS(TPL - "ALLOW_SYSTEM_PATH_FALLBACK" - "" - "PATHS;SUFFIXES" - ${ARGN}) +macro(kokkos_find_library VAR_NAME LIB TPL_NAME) + cmake_parse_arguments(TPL "ALLOW_SYSTEM_PATH_FALLBACK" "" "PATHS;SUFFIXES" ${ARGN}) - IF(NOT TPL_SUFFIXES) - SET(TPL_SUFFIXES lib lib64) - ENDIF() + if(NOT TPL_SUFFIXES) + set(TPL_SUFFIXES lib lib64) + endif() - SET(${VAR_NAME} "${VARNAME}-NOTFOUND") - SET(HAVE_CUSTOM_PATHS FALSE) + set(${VAR_NAME} "${VARNAME}-NOTFOUND") + set(HAVE_CUSTOM_PATHS FALSE) - IF(DEFINED ${TPL_NAME}_ROOT OR - DEFINED ENV{${TPL_NAME}_ROOT} OR - DEFINED KOKKOS_${TPL_NAME}_DIR OR - TPL_PATHS) - FIND_LIBRARY(${VAR_NAME} ${LIB} - PATHS - ${${TPL_NAME}_ROOT} - $ENV{${TPL_NAME}_ROOT} - ${KOKKOS_${TPL_NAME}_DIR} - ${TPL_PATHS} - PATH_SUFFIXES - ${TPL_SUFFIXES} - NO_DEFAULT_PATH) - SET(HAVE_CUSTOM_PATHS TRUE) - ENDIF() + if(DEFINED ${TPL_NAME}_ROOT + OR DEFINED ENV{${TPL_NAME}_ROOT} + OR DEFINED KOKKOS_${TPL_NAME}_DIR + OR TPL_PATHS + ) + find_library( + ${VAR_NAME} ${LIB} + PATHS ${${TPL_NAME}_ROOT} $ENV{${TPL_NAME}_ROOT} ${KOKKOS_${TPL_NAME}_DIR} ${TPL_PATHS} + PATH_SUFFIXES ${TPL_SUFFIXES} + NO_DEFAULT_PATH + ) + set(HAVE_CUSTOM_PATHS TRUE) + endif() - IF(NOT HAVE_CUSTOM_PATHS OR TPL_ALLOW_SYSTEM_PATH_FALLBACK) + if(NOT HAVE_CUSTOM_PATHS OR TPL_ALLOW_SYSTEM_PATH_FALLBACK) #No-op if ${VAR_NAME} set by previous call - FIND_LIBRARY(${VAR_NAME} ${LIB} PATH_SUFFIXES ${TPL_SUFFIXES}) - ENDIF() + find_library(${VAR_NAME} ${LIB} PATH_SUFFIXES ${TPL_SUFFIXES}) + endif() -ENDMACRO() +endmacro() # # @MACRO: KOKKOS_FIND_IMPORTED @@ -683,111 +623,127 @@ ENDMACRO() # If specified, this gives a list of paths to search for the headers # If not given, _ROOT/include and _ROOT/include will be searched. # -MACRO(kokkos_find_imported NAME) - CMAKE_PARSE_ARGUMENTS(TPL - "INTERFACE;ALLOW_SYSTEM_PATH_FALLBACK" - "IMPORTED_NAME;MODULE_NAME;LIBRARY;HEADER" - "LIBRARIES;LIBRARY_PATHS;LIBRARY_SUFFIXES;HEADERS;HEADER_PATHS" - ${ARGN}) +macro(kokkos_find_imported NAME) + cmake_parse_arguments( + TPL "INTERFACE;ALLOW_SYSTEM_PATH_FALLBACK" "IMPORTED_NAME;MODULE_NAME;LIBRARY;HEADER" + "LIBRARIES;LIBRARY_PATHS;LIBRARY_SUFFIXES;HEADERS;HEADER_PATHS" ${ARGN} + ) - IF(NOT TPL_MODULE_NAME) - SET(TPL_MODULE_NAME TPL${NAME}) - ENDIF() + if(NOT TPL_MODULE_NAME) + set(TPL_MODULE_NAME TPL${NAME}) + endif() - IF (TPL_ALLOW_SYSTEM_PATH_FALLBACK) - SET(ALLOW_PATH_FALLBACK_OPT ALLOW_SYSTEM_PATH_FALLBACK) - ELSE() - SET(ALLOW_PATH_FALLBACK_OPT) - ENDIF() + if(TPL_ALLOW_SYSTEM_PATH_FALLBACK) + set(ALLOW_PATH_FALLBACK_OPT ALLOW_SYSTEM_PATH_FALLBACK) + else() + set(ALLOW_PATH_FALLBACK_OPT) + endif() - IF (NOT TPL_IMPORTED_NAME) - IF (TPL_INTERFACE) - SET(TPL_IMPORTED_NAME ${NAME}) - ELSE() - SET(TPL_IMPORTED_NAME Kokkos::${NAME}) - ENDIF() - ENDIF() + if(NOT TPL_IMPORTED_NAME) + if(TPL_INTERFACE) + set(TPL_IMPORTED_NAME ${NAME}) + else() + set(TPL_IMPORTED_NAME Kokkos::${NAME}) + endif() + endif() - IF (NOT TPL_LIBRARY_SUFFIXES) - SET(TPL_LIBRARY_SUFFIXES lib) - IF(KOKKOS_IMPL_32BIT) - LIST(APPEND TPL_LIBRARY_SUFFIXES lib32) - ELSE() - LIST(APPEND TPL_LIBRARY_SUFFIXES lib64) - ENDIF() - ENDIF() + if(NOT TPL_LIBRARY_SUFFIXES) + set(TPL_LIBRARY_SUFFIXES lib) + if(KOKKOS_IMPL_32BIT) + list(APPEND TPL_LIBRARY_SUFFIXES lib32) + else() + list(APPEND TPL_LIBRARY_SUFFIXES lib64) + endif() + endif() - SET(${NAME}_INCLUDE_DIRS) - IF (TPL_HEADER) - KOKKOS_FIND_HEADER(${NAME}_INCLUDE_DIRS ${TPL_HEADER} ${NAME} ${ALLOW_PATH_FALLBACK_OPT} PATHS ${TPL_HEADER_PATHS}) - ENDIF() + set(${NAME}_INCLUDE_DIRS) + if(TPL_HEADER) + kokkos_find_header(${NAME}_INCLUDE_DIRS ${TPL_HEADER} ${NAME} ${ALLOW_PATH_FALLBACK_OPT} PATHS ${TPL_HEADER_PATHS}) + endif() - FOREACH(HEADER ${TPL_HEADERS}) - KOKKOS_FIND_HEADER(HEADER_FIND_TEMP ${HEADER} ${NAME} ${ALLOW_PATH_FALLBACK_OPT} PATHS ${TPL_HEADER_PATHS}) - IF(HEADER_FIND_TEMP) - LIST(APPEND ${NAME}_INCLUDE_DIRS ${HEADER_FIND_TEMP}) - ENDIF() - ENDFOREACH() + foreach(HEADER ${TPL_HEADERS}) + kokkos_find_header(HEADER_FIND_TEMP ${HEADER} ${NAME} ${ALLOW_PATH_FALLBACK_OPT} PATHS ${TPL_HEADER_PATHS}) + if(HEADER_FIND_TEMP) + list(APPEND ${NAME}_INCLUDE_DIRS ${HEADER_FIND_TEMP}) + endif() + endforeach() - SET(${NAME}_LIBRARY) - IF(TPL_LIBRARY) - KOKKOS_FIND_LIBRARY(${NAME}_LIBRARY ${TPL_LIBRARY} ${NAME} + set(${NAME}_LIBRARY) + if(TPL_LIBRARY) + kokkos_find_library( + ${NAME}_LIBRARY + ${TPL_LIBRARY} + ${NAME} ${ALLOW_PATH_FALLBACK_OPT} - PATHS ${TPL_LIBRARY_PATHS} - SUFFIXES ${TPL_LIBRARY_SUFFIXES}) - ENDIF() + PATHS + ${TPL_LIBRARY_PATHS} + SUFFIXES + ${TPL_LIBRARY_SUFFIXES} + ) + endif() - SET(${NAME}_FOUND_LIBRARIES) - FOREACH(LIB ${TPL_LIBRARIES}) - KOKKOS_FIND_LIBRARY(${LIB}_LOCATION ${LIB} ${NAME} + set(${NAME}_FOUND_LIBRARIES) + foreach(LIB ${TPL_LIBRARIES}) + kokkos_find_library( + ${LIB}_LOCATION + ${LIB} + ${NAME} ${ALLOW_PATH_FALLBACK_OPT} - PATHS ${TPL_LIBRARY_PATHS} - SUFFIXES ${TPL_LIBRARY_SUFFIXES}) - IF(${LIB}_LOCATION) - LIST(APPEND ${NAME}_FOUND_LIBRARIES ${${LIB}_LOCATION}) - ELSE() - SET(${NAME}_FOUND_LIBRARIES ${${LIB}_LOCATION}) - BREAK() - ENDIF() - ENDFOREACH() + PATHS + ${TPL_LIBRARY_PATHS} + SUFFIXES + ${TPL_LIBRARY_SUFFIXES} + ) + if(${LIB}_LOCATION) + list(APPEND ${NAME}_FOUND_LIBRARIES ${${LIB}_LOCATION}) + else() + set(${NAME}_FOUND_LIBRARIES ${${LIB}_LOCATION}) + break() + endif() + endforeach() - INCLUDE(FindPackageHandleStandardArgs) + include(FindPackageHandleStandardArgs) #Collect all the variables we need to be valid for #find_package to have succeeded - SET(TPL_VARS_NEEDED) - IF (TPL_LIBRARY) - LIST(APPEND TPL_VARS_NEEDED ${NAME}_LIBRARY) - ENDIF() - IF(TPL_HEADER) - LIST(APPEND TPL_VARS_NEEDED ${NAME}_INCLUDE_DIRS) - ENDIF() - IF(TPL_LIBRARIES) - LIST(APPEND TPL_VARS_NEEDED ${NAME}_FOUND_LIBRARIES) - ENDIF() - FIND_PACKAGE_HANDLE_STANDARD_ARGS(${TPL_MODULE_NAME} REQUIRED_VARS ${TPL_VARS_NEEDED}) + set(TPL_VARS_NEEDED) + if(TPL_LIBRARY) + list(APPEND TPL_VARS_NEEDED ${NAME}_LIBRARY) + endif() + if(TPL_HEADER) + list(APPEND TPL_VARS_NEEDED ${NAME}_INCLUDE_DIRS) + endif() + if(TPL_LIBRARIES) + list(APPEND TPL_VARS_NEEDED ${NAME}_FOUND_LIBRARIES) + endif() + find_package_handle_standard_args(${TPL_MODULE_NAME} REQUIRED_VARS ${TPL_VARS_NEEDED}) - MARK_AS_ADVANCED(${NAME}_INCLUDE_DIRS ${NAME}_FOUND_LIBRARIES ${NAME}_LIBRARY) + mark_as_advanced(${NAME}_INCLUDE_DIRS ${NAME}_FOUND_LIBRARIES ${NAME}_LIBRARY) #this is so much fun on a Cray system #/usr/include should never be added as a -isystem include #this freaks out the compiler include search order - IF (KOKKOS_IS_CRAYPE) - LIST(REMOVE_ITEM ${NAME}_INCLUDE_DIRS "/usr/include") - ENDIF() + if(KOKKOS_IS_CRAYPE) + list(REMOVE_ITEM ${NAME}_INCLUDE_DIRS "/usr/include") + endif() - IF (${TPL_MODULE_NAME}_FOUND) - SET(IMPORT_TYPE) - IF (TPL_INTERFACE) - SET(IMPORT_TYPE "INTERFACE") - SET(${NAME}_FOUND_LIBRARIES ${TPL_LIBRARIES}) - ENDIF() - KOKKOS_CREATE_IMPORTED_TPL(${TPL_IMPORTED_NAME} + if(${TPL_MODULE_NAME}_FOUND) + set(IMPORT_TYPE) + if(TPL_INTERFACE) + set(IMPORT_TYPE "INTERFACE") + set(${NAME}_FOUND_LIBRARIES ${TPL_LIBRARIES}) + endif() + kokkos_create_imported_tpl( + ${TPL_IMPORTED_NAME} ${IMPORT_TYPE} - INCLUDES "${${NAME}_INCLUDE_DIRS}" - LIBRARY "${${NAME}_LIBRARY}" - LINK_LIBRARIES "${${NAME}_FOUND_LIBRARIES}") - ENDIF() -ENDMACRO(kokkos_find_imported) + INCLUDES + "${${NAME}_INCLUDE_DIRS}" + LIBRARY + "${${NAME}_LIBRARY}" + LINK_LIBRARIES + "${${NAME}_FOUND_LIBRARIES}" + ) + endif() +endmacro(kokkos_find_imported) # # @MACRO: KOKKOS_LINK_TPL() @@ -817,109 +773,114 @@ ENDMACRO(kokkos_find_imported) # If specified, this gives the exact name of the target to link against # target_link_libraries( ) # -FUNCTION(kokkos_link_tpl TARGET) - CMAKE_PARSE_ARGUMENTS(TPL - "PUBLIC;PRIVATE;INTERFACE" - "IMPORTED_NAME" - "" - ${ARGN}) +function(kokkos_link_tpl TARGET) + cmake_parse_arguments(TPL "PUBLIC;PRIVATE;INTERFACE" "IMPORTED_NAME" "" ${ARGN}) #the name of the TPL - SET(TPL ${TPL_UNPARSED_ARGUMENTS}) - IF (KOKKOS_HAS_TRILINOS) - #Do nothing, they will have already been linked - ELSE() - IF (NOT TPL_IMPORTED_NAME) - SET(TPL_IMPORTED_NAME Kokkos::${TPL}) - ENDIF() - IF (KOKKOS_ENABLE_${TPL}) - IF (TPL_PUBLIC) - TARGET_LINK_LIBRARIES(${TARGET} PUBLIC ${TPL_IMPORTED_NAME}) - ELSEIF (TPL_PRIVATE) - TARGET_LINK_LIBRARIES(${TARGET} PRIVATE ${TPL_IMPORTED_NAME}) - ELSEIF (TPL_INTERFACE) - TARGET_LINK_LIBRARIES(${TARGET} INTERFACE ${TPL_IMPORTED_NAME}) - ELSE() - TARGET_LINK_LIBRARIES(${TARGET} ${TPL_IMPORTED_NAME}) - ENDIF() - ENDIF() - ENDIF() -ENDFUNCTION() + set(TPL ${TPL_UNPARSED_ARGUMENTS}) + if(NOT TPL_IMPORTED_NAME) + set(TPL_IMPORTED_NAME Kokkos::${TPL}) + endif() + if(KOKKOS_ENABLE_${TPL}) + if(TPL_PUBLIC) + target_link_libraries(${TARGET} PUBLIC ${TPL_IMPORTED_NAME}) + elseif(TPL_PRIVATE) + target_link_libraries(${TARGET} PRIVATE ${TPL_IMPORTED_NAME}) + elseif(TPL_INTERFACE) + target_link_libraries(${TARGET} INTERFACE ${TPL_IMPORTED_NAME}) + else() + target_link_libraries(${TARGET} ${TPL_IMPORTED_NAME}) + endif() + endif() +endfunction() -FUNCTION(COMPILER_SPECIFIC_OPTIONS_HELPER) - SET(COMPILERS NVIDIA NVHPC DEFAULT Cray Intel Clang AppleClang IntelLLVM GNU HIPCC Fujitsu MSVC) - CMAKE_PARSE_ARGUMENTS( - PARSE - "LINK_OPTIONS;COMPILE_OPTIONS;COMPILE_DEFINITIONS;LINK_LIBRARIES" - "COMPILER_ID" - "${COMPILERS}" - ${ARGN}) - IF(PARSE_UNPARSED_ARGUMENTS) - MESSAGE(SEND_ERROR "'${PARSE_UNPARSED_ARGUMENTS}' argument(s) not recognized when providing compiler specific options") - ENDIF() +function(COMPILER_SPECIFIC_OPTIONS_HELPER) + set(COMPILERS + NVIDIA + NVHPC + DEFAULT + Cray + Intel + Clang + AppleClang + IntelLLVM + GNU + HIPCC + Fujitsu + MSVC + CrayClang + ) + cmake_parse_arguments( + PARSE "LINK_OPTIONS;COMPILE_OPTIONS;COMPILE_DEFINITIONS;LINK_LIBRARIES" "COMPILER_ID" "${COMPILERS}" ${ARGN} + ) + if(PARSE_UNPARSED_ARGUMENTS) + message( + SEND_ERROR "'${PARSE_UNPARSED_ARGUMENTS}' argument(s) not recognized when providing compiler specific options" + ) + endif() - IF(PARSE_COMPILER_ID) - SET(COMPILER ${${PARSE_COMPILER_ID}}) - ELSE() - SET(COMPILER ${KOKKOS_CXX_COMPILER_ID}) - ENDIF() + if(PARSE_COMPILER_ID) + set(COMPILER ${${PARSE_COMPILER_ID}}) + else() + set(COMPILER ${KOKKOS_CXX_COMPILER_ID}) + endif() - SET(COMPILER_SPECIFIC_FLAGS_TMP ${PARSE_DEFAULT}) - FOREACH(COMP ${COMPILERS}) - IF (COMPILER STREQUAL "${COMP}") - IF (PARSE_${COMPILER}) - IF ("${PARSE_${COMPILER}}" STREQUAL "NO-VALUE-SPECIFIED") - SET(COMPILER_SPECIFIC_FLAGS_TMP "") - ELSE() - SET(COMPILER_SPECIFIC_FLAGS_TMP ${PARSE_${COMPILER}}) - ENDIF() - ENDIF() - ENDIF() - ENDFOREACH() + set(COMPILER_SPECIFIC_FLAGS_TMP ${PARSE_DEFAULT}) + foreach(COMP ${COMPILERS}) + if(COMPILER STREQUAL "${COMP}") + if(PARSE_${COMPILER}) + if("${PARSE_${COMPILER}}" STREQUAL "NO-VALUE-SPECIFIED") + set(COMPILER_SPECIFIC_FLAGS_TMP "") + else() + set(COMPILER_SPECIFIC_FLAGS_TMP ${PARSE_${COMPILER}}) + endif() + endif() + endif() + endforeach() - IF (PARSE_COMPILE_OPTIONS) + if(PARSE_COMPILE_OPTIONS) # The funky logic here is for future handling of argument deduplication # If we naively pass multiple -Xcompiler flags to target_compile_options # -Xcompiler will get deduplicated and break the build - IF ("-Xcompiler" IN_LIST COMPILER_SPECIFIC_FLAGS_TMP) - LIST(REMOVE_ITEM COMPILER_SPECIFIC_FLAGS_TMP "-Xcompiler") - GLOBAL_APPEND(KOKKOS_XCOMPILER_OPTIONS ${COMPILER_SPECIFIC_FLAGS_TMP}) - ELSE() - GLOBAL_APPEND(KOKKOS_COMPILE_OPTIONS ${COMPILER_SPECIFIC_FLAGS_TMP}) - ENDIF() - ENDIF() + if("-Xcompiler" IN_LIST COMPILER_SPECIFIC_FLAGS_TMP) + list(REMOVE_ITEM COMPILER_SPECIFIC_FLAGS_TMP "-Xcompiler") + global_append(KOKKOS_XCOMPILER_OPTIONS ${COMPILER_SPECIFIC_FLAGS_TMP}) + else() + global_append(KOKKOS_COMPILE_OPTIONS ${COMPILER_SPECIFIC_FLAGS_TMP}) + endif() + endif() - IF (PARSE_LINK_OPTIONS) - GLOBAL_APPEND(KOKKOS_LINK_OPTIONS ${COMPILER_SPECIFIC_FLAGS_TMP}) - ENDIF() + if(PARSE_LINK_OPTIONS) + global_append(KOKKOS_LINK_OPTIONS ${COMPILER_SPECIFIC_FLAGS_TMP}) + endif() - IF (PARSE_COMPILE_DEFINITIONS) - GLOBAL_APPEND(KOKKOS_COMPILE_DEFINITIONS ${COMPILER_SPECIFIC_FLAGS_TMP}) - ENDIF() + if(PARSE_COMPILE_DEFINITIONS) + global_append(KOKKOS_COMPILE_DEFINITIONS ${COMPILER_SPECIFIC_FLAGS_TMP}) + endif() - IF (PARSE_LINK_LIBRARIES) - GLOBAL_APPEND(KOKKOS_LINK_LIBRARIES ${COMPILER_SPECIFIC_FLAGS_TMP}) - ENDIF() -ENDFUNCTION(COMPILER_SPECIFIC_OPTIONS_HELPER) + if(PARSE_LINK_LIBRARIES) + global_append(KOKKOS_LINK_LIBRARIES ${COMPILER_SPECIFIC_FLAGS_TMP}) + endif() +endfunction(COMPILER_SPECIFIC_OPTIONS_HELPER) -FUNCTION(COMPILER_SPECIFIC_FLAGS) - COMPILER_SPECIFIC_OPTIONS_HELPER(${ARGN} COMPILE_OPTIONS LINK_OPTIONS) -ENDFUNCTION(COMPILER_SPECIFIC_FLAGS) +function(COMPILER_SPECIFIC_FLAGS) + compiler_specific_options_helper(${ARGN} COMPILE_OPTIONS LINK_OPTIONS) +endfunction(COMPILER_SPECIFIC_FLAGS) -FUNCTION(COMPILER_SPECIFIC_OPTIONS) - COMPILER_SPECIFIC_OPTIONS_HELPER(${ARGN} COMPILE_OPTIONS) -ENDFUNCTION(COMPILER_SPECIFIC_OPTIONS) +function(COMPILER_SPECIFIC_OPTIONS) + compiler_specific_options_helper(${ARGN} COMPILE_OPTIONS) +endfunction(COMPILER_SPECIFIC_OPTIONS) -FUNCTION(COMPILER_SPECIFIC_LINK_OPTIONS) - COMPILER_SPECIFIC_OPTIONS_HELPER(${ARGN} LINK_OPTIONS) -ENDFUNCTION(COMPILER_SPECIFIC_LINK_OPTIONS) +function(COMPILER_SPECIFIC_LINK_OPTIONS) + compiler_specific_options_helper(${ARGN} LINK_OPTIONS) +endfunction(COMPILER_SPECIFIC_LINK_OPTIONS) -FUNCTION(COMPILER_SPECIFIC_DEFS) - COMPILER_SPECIFIC_OPTIONS_HELPER(${ARGN} COMPILE_DEFINITIONS) -ENDFUNCTION(COMPILER_SPECIFIC_DEFS) +function(COMPILER_SPECIFIC_DEFS) + compiler_specific_options_helper(${ARGN} COMPILE_DEFINITIONS) +endfunction(COMPILER_SPECIFIC_DEFS) -FUNCTION(COMPILER_SPECIFIC_LIBS) - COMPILER_SPECIFIC_OPTIONS_HELPER(${ARGN} LINK_LIBRARIES) -ENDFUNCTION(COMPILER_SPECIFIC_LIBS) +function(COMPILER_SPECIFIC_LIBS) + compiler_specific_options_helper(${ARGN} LINK_LIBRARIES) +endfunction(COMPILER_SPECIFIC_LIBS) # Given a list of the form # key1;value1;key2;value2,... # Create a list of all keys in a variable named ${KEY_LIST_NAME} @@ -927,41 +888,42 @@ ENDFUNCTION(COMPILER_SPECIFIC_LIBS) # kokkos_key_value_map(ARCH ALL_ARCHES key1;value1;key2;value2) # would produce a list variable ALL_ARCHES=key1;key2 # and individual variables ARCHkey1=value1 and ARCHkey2=value2 -MACRO(KOKKOS_KEY_VALUE_MAP VAR_PREFIX KEY_LIST_NAME) - SET(PARSE_KEY ON) - SET(${KEY_LIST_NAME}) - FOREACH(ENTRY ${ARGN}) - IF(PARSE_KEY) - SET(CURRENT_KEY ${ENTRY}) - SET(PARSE_KEY OFF) - LIST(APPEND ${KEY_LIST_NAME} ${CURRENT_KEY}) - ELSE() - SET(${VAR_PREFIX}${CURRENT_KEY} ${ENTRY}) - SET(PARSE_KEY ON) - ENDIF() - ENDFOREACH() -ENDMACRO() +macro(KOKKOS_KEY_VALUE_MAP VAR_PREFIX KEY_LIST_NAME) + set(PARSE_KEY ON) + set(${KEY_LIST_NAME}) + foreach(ENTRY ${ARGN}) + if(PARSE_KEY) + set(CURRENT_KEY ${ENTRY}) + set(PARSE_KEY OFF) + list(APPEND ${KEY_LIST_NAME} ${CURRENT_KEY}) + else() + set(${VAR_PREFIX}${CURRENT_KEY} ${ENTRY}) + set(PARSE_KEY ON) + endif() + endforeach() +endmacro() -FUNCTION(KOKKOS_CHECK_DEPRECATED_OPTIONS) - KOKKOS_KEY_VALUE_MAP(DEPRECATED_MSG_ DEPRECATED_LIST ${ARGN}) - FOREACH(OPTION_SUFFIX ${DEPRECATED_LIST}) - SET(OPTION_NAME Kokkos_${OPTION_SUFFIX}) - SET(OPTION_MESSAGE ${DEPRECATED_MSG_${OPTION_SUFFIX}}) - IF(DEFINED ${OPTION_NAME}) # This variable has been given by the user as on or off - MESSAGE(SEND_ERROR "Removed option ${OPTION_NAME} has been given with value ${${OPTION_NAME}}. ${OPT_MESSAGE}") - ENDIF() - ENDFOREACH() -ENDFUNCTION() +function(KOKKOS_CHECK_DEPRECATED_OPTIONS) + kokkos_key_value_map(DEPRECATED_MSG_ DEPRECATED_LIST ${ARGN}) + foreach(OPTION_SUFFIX ${DEPRECATED_LIST}) + set(OPTION_NAME Kokkos_${OPTION_SUFFIX}) + set(OPTION_MESSAGE ${DEPRECATED_MSG_${OPTION_SUFFIX}}) + if(DEFINED ${OPTION_NAME}) # This variable has been given by the user as on or off + message(SEND_ERROR "Removed option ${OPTION_NAME} has been given with value ${${OPTION_NAME}}. ${OPT_MESSAGE}") + endif() + endforeach() +endfunction() # this function checks whether the current CXX compiler supports building CUDA -FUNCTION(kokkos_cxx_compiler_cuda_test _VAR) - # don't run this test every time - IF(DEFINED ${_VAR}) - RETURN() - ENDIF() +function(kokkos_cxx_compiler_cuda_test _VAR) + # don't run this test every time + if(DEFINED ${_VAR}) + return() + endif() - FILE(WRITE ${PROJECT_BINARY_DIR}/compile_tests/compiles_cuda.cpp -" + file( + WRITE ${PROJECT_BINARY_DIR}/compile_tests/compiles_cuda.cpp + " #include #include @@ -985,14 +947,13 @@ int main() cudaDeviceSynchronize(); return EXIT_SUCCESS; } -") +" + ) - TRY_COMPILE(_RET - ${PROJECT_BINARY_DIR}/compile_tests - SOURCES ${PROJECT_BINARY_DIR}/compile_tests/compiles_cuda.cpp) + try_compile(_RET ${PROJECT_BINARY_DIR}/compile_tests SOURCES ${PROJECT_BINARY_DIR}/compile_tests/compiles_cuda.cpp) - SET(${_VAR} ${_RET} CACHE STRING "CXX compiler supports building CUDA") -ENDFUNCTION() + set(${_VAR} ${_RET} CACHE STRING "CXX compiler supports building CUDA") +endfunction() # this function is provided to easily select which files use nvcc_wrapper: # @@ -1005,58 +966,77 @@ ENDFUNCTION() # NOTE: this is VERY DIFFERENT than the version in KokkosConfigCommon.cmake.in. # This version explicitly uses nvcc_wrapper. # -FUNCTION(kokkos_compilation) - # check whether the compiler already supports building CUDA - KOKKOS_CXX_COMPILER_CUDA_TEST(Kokkos_CXX_COMPILER_COMPILES_CUDA) - # if CUDA compile test has already been performed, just return - IF(Kokkos_CXX_COMPILER_COMPILES_CUDA) - RETURN() - ENDIF() +function(kokkos_compilation) + # check whether the compiler already supports building CUDA + kokkos_cxx_compiler_cuda_test(Kokkos_CXX_COMPILER_COMPILES_CUDA) + # if CUDA compile test has already been performed, just return + if(Kokkos_CXX_COMPILER_COMPILES_CUDA) + return() + endif() - CMAKE_PARSE_ARGUMENTS(COMP "GLOBAL;PROJECT" "" "DIRECTORY;TARGET;SOURCE" ${ARGN}) + cmake_parse_arguments(COMP "GLOBAL;PROJECT" "" "DIRECTORY;TARGET;SOURCE" ${ARGN}) - # find kokkos_launch_compiler - FIND_PROGRAM(Kokkos_COMPILE_LAUNCHER - NAMES kokkos_launch_compiler - HINTS ${PROJECT_SOURCE_DIR} - PATHS ${PROJECT_SOURCE_DIR} - PATH_SUFFIXES bin) + # find kokkos_launch_compiler + find_program( + Kokkos_COMPILE_LAUNCHER + NAMES kokkos_launch_compiler + HINTS ${PROJECT_SOURCE_DIR} + PATHS ${PROJECT_SOURCE_DIR} + PATH_SUFFIXES bin + ) - IF(NOT Kokkos_COMPILE_LAUNCHER) - MESSAGE(FATAL_ERROR "Kokkos could not find 'kokkos_launch_compiler'. Please set '-DKokkos_COMPILE_LAUNCHER=/path/to/launcher'") - ENDIF() + if(NOT Kokkos_COMPILE_LAUNCHER) + message( + FATAL_ERROR + "Kokkos could not find 'kokkos_launch_compiler'. Please set '-DKokkos_COMPILE_LAUNCHER=/path/to/launcher'" + ) + endif() - # find nvcc_wrapper - FIND_PROGRAM(Kokkos_NVCC_WRAPPER - NAMES nvcc_wrapper - HINTS ${PROJECT_SOURCE_DIR} - PATHS ${PROJECT_SOURCE_DIR} - PATH_SUFFIXES bin) + # find nvcc_wrapper + find_program( + Kokkos_NVCC_WRAPPER + NAMES nvcc_wrapper + HINTS ${PROJECT_SOURCE_DIR} + PATHS ${PROJECT_SOURCE_DIR} + PATH_SUFFIXES bin + ) - IF(NOT Kokkos_COMPILE_LAUNCHER) - MESSAGE(FATAL_ERROR "Kokkos could not find 'nvcc_wrapper'. Please set '-DKokkos_COMPILE_LAUNCHER=/path/to/nvcc_wrapper'") - ENDIF() + if(NOT Kokkos_COMPILE_LAUNCHER) + message( + FATAL_ERROR "Kokkos could not find 'nvcc_wrapper'. Please set '-DKokkos_COMPILE_LAUNCHER=/path/to/nvcc_wrapper'" + ) + endif() - IF(COMP_GLOBAL) - # if global, don't bother setting others - SET_PROPERTY(GLOBAL PROPERTY RULE_LAUNCH_COMPILE "${Kokkos_COMPILE_LAUNCHER} ${Kokkos_NVCC_WRAPPER} ${CMAKE_CXX_COMPILER}") - SET_PROPERTY(GLOBAL PROPERTY RULE_LAUNCH_LINK "${Kokkos_COMPILE_LAUNCHER} ${Kokkos_NVCC_WRAPPER} ${CMAKE_CXX_COMPILER}") - ELSE() - FOREACH(_TYPE PROJECT DIRECTORY TARGET SOURCE) - # make project/subproject scoping easy, e.g. KokkosCompilation(PROJECT) after project(...) - IF("${_TYPE}" STREQUAL "PROJECT" AND COMP_${_TYPE}) - LIST(APPEND COMP_DIRECTORY ${PROJECT_SOURCE_DIR}) - UNSET(COMP_${_TYPE}) - ENDIF() - # set the properties if defined - IF(COMP_${_TYPE}) - # MESSAGE(STATUS "Using nvcc_wrapper :: ${_TYPE} :: ${COMP_${_TYPE}}") - SET_PROPERTY(${_TYPE} ${COMP_${_TYPE}} PROPERTY RULE_LAUNCH_COMPILE "${Kokkos_COMPILE_LAUNCHER} ${Kokkos_NVCC_WRAPPER} ${CMAKE_CXX_COMPILER}") - SET_PROPERTY(${_TYPE} ${COMP_${_TYPE}} PROPERTY RULE_LAUNCH_LINK "${Kokkos_COMPILE_LAUNCHER} ${Kokkos_NVCC_WRAPPER} ${CMAKE_CXX_COMPILER}") - ENDIF() - ENDFOREACH() - ENDIF() -ENDFUNCTION() + if(COMP_GLOBAL) + # if global, don't bother setting others + set_property( + GLOBAL PROPERTY RULE_LAUNCH_COMPILE "${Kokkos_COMPILE_LAUNCHER} ${Kokkos_NVCC_WRAPPER} ${CMAKE_CXX_COMPILER}" + ) + set_property( + GLOBAL PROPERTY RULE_LAUNCH_LINK "${Kokkos_COMPILE_LAUNCHER} ${Kokkos_NVCC_WRAPPER} ${CMAKE_CXX_COMPILER}" + ) + else() + foreach(_TYPE PROJECT DIRECTORY TARGET SOURCE) + # make project/subproject scoping easy, e.g. KokkosCompilation(PROJECT) after project(...) + if("${_TYPE}" STREQUAL "PROJECT" AND COMP_${_TYPE}) + list(APPEND COMP_DIRECTORY ${PROJECT_SOURCE_DIR}) + unset(COMP_${_TYPE}) + endif() + # set the properties if defined + if(COMP_${_TYPE}) + # MESSAGE(STATUS "Using nvcc_wrapper :: ${_TYPE} :: ${COMP_${_TYPE}}") + set_property( + ${_TYPE} ${COMP_${_TYPE}} PROPERTY RULE_LAUNCH_COMPILE + "${Kokkos_COMPILE_LAUNCHER} ${Kokkos_NVCC_WRAPPER} ${CMAKE_CXX_COMPILER}" + ) + set_property( + ${_TYPE} ${COMP_${_TYPE}} PROPERTY RULE_LAUNCH_LINK + "${Kokkos_COMPILE_LAUNCHER} ${Kokkos_NVCC_WRAPPER} ${CMAKE_CXX_COMPILER}" + ) + endif() + endforeach() + endif() +endfunction() ## KOKKOS_CONFIG_HEADER - parse the data list which is a list of backend names ## and create output config header file...used for ## creating dynamic include files based on enabled backends @@ -1066,14 +1046,15 @@ ENDFUNCTION() ## HEADER_GUARD TEXT used with include header guard ## HEADER_PREFIX prefix used with include (i.e. fwd, decl, setup) ## DATA_LIST list of backends to include in generated file -FUNCTION(KOKKOS_CONFIG_HEADER SRC_FILE TARGET_FILE HEADER_GUARD HEADER_PREFIX DATA_LIST) - SET(HEADER_GUARD_TAG "${HEADER_GUARD}_HPP_") - CONFIGURE_FILE(cmake/${SRC_FILE} ${PROJECT_BINARY_DIR}/temp/${TARGET_FILE}.work COPYONLY) - FOREACH( BACKEND_NAME ${DATA_LIST} ) - SET(INCLUDE_NEXT_FILE "#include <${HEADER_PREFIX}_${BACKEND_NAME}.hpp> -\@INCLUDE_NEXT_FILE\@") - CONFIGURE_FILE(${PROJECT_BINARY_DIR}/temp/${TARGET_FILE}.work ${PROJECT_BINARY_DIR}/temp/${TARGET_FILE}.work @ONLY) - ENDFOREACH() - SET(INCLUDE_NEXT_FILE "" ) - CONFIGURE_FILE(${PROJECT_BINARY_DIR}/temp/${TARGET_FILE}.work ${TARGET_FILE} @ONLY) -ENDFUNCTION() +function(KOKKOS_CONFIG_HEADER SRC_FILE TARGET_FILE HEADER_GUARD HEADER_PREFIX DATA_LIST) + set(HEADER_GUARD_TAG "${HEADER_GUARD}_HPP_") + configure_file(cmake/${SRC_FILE} ${PROJECT_BINARY_DIR}/temp/${TARGET_FILE}.work COPYONLY) + foreach(BACKEND_NAME ${DATA_LIST}) + set(INCLUDE_NEXT_FILE "#include <${HEADER_PREFIX}_${BACKEND_NAME}.hpp> +\@INCLUDE_NEXT_FILE\@" + ) + configure_file(${PROJECT_BINARY_DIR}/temp/${TARGET_FILE}.work ${PROJECT_BINARY_DIR}/temp/${TARGET_FILE}.work @ONLY) + endforeach() + set(INCLUDE_NEXT_FILE "") + configure_file(${PROJECT_BINARY_DIR}/temp/${TARGET_FILE}.work ${TARGET_FILE} @ONLY) +endfunction() diff --git a/lib/kokkos/cmake/kokkos_install.cmake b/lib/kokkos/cmake/kokkos_install.cmake index f818dfa244..3ae7570ffe 100644 --- a/lib/kokkos/cmake/kokkos_install.cmake +++ b/lib/kokkos/cmake/kokkos_install.cmake @@ -1,57 +1,51 @@ -INCLUDE(CMakePackageConfigHelpers) -IF (NOT KOKKOS_HAS_TRILINOS AND NOT Kokkos_INSTALL_TESTING) - INCLUDE(GNUInstallDirs) +include(CMakePackageConfigHelpers) +if(NOT Kokkos_INSTALL_TESTING) + include(GNUInstallDirs) #Set all the variables needed for KokkosConfig.cmake - GET_PROPERTY(KOKKOS_PROP_LIBS GLOBAL PROPERTY KOKKOS_LIBRARIES_NAMES) - SET(KOKKOS_LIBRARIES ${KOKKOS_PROP_LIBS}) + get_property(KOKKOS_PROP_LIBS GLOBAL PROPERTY KOKKOS_LIBRARIES_NAMES) + set(KOKKOS_LIBRARIES ${KOKKOS_PROP_LIBS}) - INCLUDE(CMakePackageConfigHelpers) - CONFIGURE_PACKAGE_CONFIG_FILE( - cmake/KokkosConfig.cmake.in - "${Kokkos_BINARY_DIR}/KokkosConfig.cmake" - INSTALL_DESTINATION ${CMAKE_INSTALL_FULL_LIBDIR}/cmake) + include(CMakePackageConfigHelpers) + configure_package_config_file( + cmake/KokkosConfig.cmake.in "${Kokkos_BINARY_DIR}/KokkosConfig.cmake" + INSTALL_DESTINATION ${CMAKE_INSTALL_FULL_LIBDIR}/cmake + ) - CONFIGURE_PACKAGE_CONFIG_FILE( - cmake/KokkosConfigCommon.cmake.in - "${Kokkos_BINARY_DIR}/KokkosConfigCommon.cmake" - INSTALL_DESTINATION ${CMAKE_INSTALL_FULL_LIBDIR}/cmake) + configure_package_config_file( + cmake/KokkosConfigCommon.cmake.in "${Kokkos_BINARY_DIR}/KokkosConfigCommon.cmake" + INSTALL_DESTINATION ${CMAKE_INSTALL_FULL_LIBDIR}/cmake + ) - WRITE_BASIC_PACKAGE_VERSION_FILE("${Kokkos_BINARY_DIR}/KokkosConfigVersion.cmake" - VERSION "${Kokkos_VERSION}" - COMPATIBILITY AnyNewerVersion) + write_basic_package_version_file( + "${Kokkos_BINARY_DIR}/KokkosConfigVersion.cmake" VERSION "${Kokkos_VERSION}" COMPATIBILITY AnyNewerVersion + ) # Install the KokkosConfig*.cmake files - install(FILES - "${Kokkos_BINARY_DIR}/KokkosConfig.cmake" - "${Kokkos_BINARY_DIR}/KokkosConfigCommon.cmake" - "${Kokkos_BINARY_DIR}/KokkosConfigVersion.cmake" - DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/Kokkos) + install(FILES "${Kokkos_BINARY_DIR}/KokkosConfig.cmake" "${Kokkos_BINARY_DIR}/KokkosConfigCommon.cmake" + "${Kokkos_BINARY_DIR}/KokkosConfigVersion.cmake" DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/Kokkos + ) install(EXPORT KokkosTargets NAMESPACE Kokkos:: DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/Kokkos) export(EXPORT KokkosTargets NAMESPACE Kokkos:: FILE ${Kokkos_BINARY_DIR}/KokkosTargets.cmake) # Required to be a TriBITS-compliant external package file(MAKE_DIRECTORY ${CMAKE_BINARY_DIR}/cmake_packages/Kokkos) - file(COPY ${Kokkos_BINARY_DIR}/KokkosConfig.cmake - ${Kokkos_BINARY_DIR}/KokkosConfigCommon.cmake - ${Kokkos_BINARY_DIR}/KokkosConfigVersion.cmake - DESTINATION ${CMAKE_BINARY_DIR}/cmake_packages/Kokkos) - export(EXPORT KokkosTargets NAMESPACE Kokkos:: FILE ${CMAKE_BINARY_DIR}/cmake_packages/Kokkos/KokkosTargets.cmake) -ELSE() - CONFIGURE_FILE(cmake/KokkosConfigCommon.cmake.in ${Kokkos_BINARY_DIR}/KokkosConfigCommon.cmake @ONLY) - file(READ ${Kokkos_BINARY_DIR}/KokkosConfigCommon.cmake KOKKOS_CONFIG_COMMON) - file(APPEND "${CMAKE_CURRENT_BINARY_DIR}/CMakeFiles/KokkosConfig_install.cmake" "${KOKKOS_CONFIG_COMMON}") - CONFIGURE_FILE(cmake/KokkosTrilinosConfig.cmake.in ${Kokkos_BINARY_DIR}/KokkosTrilinosConfig.cmake @ONLY) - file(READ ${Kokkos_BINARY_DIR}/KokkosTrilinosConfig.cmake KOKKOS_TRILINOS_CONFIG) - file(APPEND "${CMAKE_CURRENT_BINARY_DIR}/CMakeFiles/KokkosConfig_install.cmake" "${KOKKOS_TRILINOS_CONFIG}") + file(COPY ${Kokkos_BINARY_DIR}/KokkosConfig.cmake ${Kokkos_BINARY_DIR}/KokkosConfigCommon.cmake + ${Kokkos_BINARY_DIR}/KokkosConfigVersion.cmake DESTINATION ${CMAKE_BINARY_DIR}/cmake_packages/Kokkos + ) + file(WRITE ${CMAKE_BINARY_DIR}/cmake_packages/Kokkos/KokkosTargets.cmake + "include(${Kokkos_BINARY_DIR}/KokkosTargets.cmake)" + ) +else() + configure_file(cmake/KokkosConfigCommon.cmake.in ${Kokkos_BINARY_DIR}/KokkosConfigCommon.cmake @ONLY) - WRITE_BASIC_PACKAGE_VERSION_FILE("${CMAKE_CURRENT_BINARY_DIR}/KokkosConfigVersion.cmake" - VERSION "${Kokkos_VERSION}" - COMPATIBILITY AnyNewerVersion) + write_basic_package_version_file( + "${CMAKE_CURRENT_BINARY_DIR}/KokkosConfigVersion.cmake" VERSION "${Kokkos_VERSION}" COMPATIBILITY AnyNewerVersion + ) install(FILES ${CMAKE_CURRENT_BINARY_DIR}/KokkosConfigVersion.cmake - DESTINATION "${${PROJECT_NAME}_INSTALL_LIB_DIR}/cmake/${PACKAGE_NAME}") -ENDIF() - -INSTALL(FILES ${CMAKE_CURRENT_BINARY_DIR}/KokkosCore_config.h DESTINATION ${KOKKOS_HEADER_DIR}) + DESTINATION "${${PROJECT_NAME}_INSTALL_LIB_DIR}/cmake/Kokkos" + ) +endif() +install(FILES ${CMAKE_CURRENT_BINARY_DIR}/KokkosCore_config.h DESTINATION ${KOKKOS_HEADER_DIR}) diff --git a/lib/kokkos/cmake/kokkos_pick_cxx_std.cmake b/lib/kokkos/cmake/kokkos_pick_cxx_std.cmake index ae14a10d53..0d31e6d131 100644 --- a/lib/kokkos/cmake/kokkos_pick_cxx_std.cmake +++ b/lib/kokkos/cmake/kokkos_pick_cxx_std.cmake @@ -1,20 +1,28 @@ # From CMake 3.10 documentation #This can run at any time -KOKKOS_OPTION(CXX_STANDARD "" STRING "[[DEPRECATED - USE CMAKE_CXX_STANDARD INSTEAD]] The C++ standard for Kokkos to use: 17 or 20. If empty, this will default to CMAKE_CXX_STANDARD. If both CMAKE_CXX_STANDARD and Kokkos_CXX_STANDARD are empty, this will default to 17") +kokkos_option( + CXX_STANDARD + "" + STRING + "[[DEPRECATED - USE CMAKE_CXX_STANDARD INSTEAD]] The C++ standard for Kokkos to use: 17 or 20. If empty, this will default to CMAKE_CXX_STANDARD. If both CMAKE_CXX_STANDARD and Kokkos_CXX_STANDARD are empty, this will default to 17" +) # Set CXX standard flags -SET(KOKKOS_ENABLE_CXX17 OFF) -SET(KOKKOS_ENABLE_CXX20 OFF) -SET(KOKKOS_ENABLE_CXX23 OFF) -SET(KOKKOS_ENABLE_CXX26 OFF) -IF (KOKKOS_CXX_STANDARD) - MESSAGE(FATAL_ERROR "Setting the variable Kokkos_CXX_STANDARD in configuration is deprecated - set CMAKE_CXX_STANDARD directly instead") -ENDIF() +set(KOKKOS_ENABLE_CXX17 OFF) +set(KOKKOS_ENABLE_CXX20 OFF) +set(KOKKOS_ENABLE_CXX23 OFF) +set(KOKKOS_ENABLE_CXX26 OFF) +if(KOKKOS_CXX_STANDARD) + message( + FATAL_ERROR + "Setting the variable Kokkos_CXX_STANDARD in configuration is deprecated - set CMAKE_CXX_STANDARD directly instead" + ) +endif() -IF (NOT CMAKE_CXX_STANDARD) - SET(KOKKOS_CXX_STANDARD "17") -ELSE() - SET(KOKKOS_CXX_STANDARD ${CMAKE_CXX_STANDARD}) -ENDIF() -MESSAGE(STATUS "Setting default Kokkos CXX standard to ${KOKKOS_CXX_STANDARD}") +if(NOT CMAKE_CXX_STANDARD) + set(KOKKOS_CXX_STANDARD "17") +else() + set(KOKKOS_CXX_STANDARD ${CMAKE_CXX_STANDARD}) +endif() +message(STATUS "Setting default Kokkos CXX standard to ${KOKKOS_CXX_STANDARD}") diff --git a/lib/kokkos/cmake/kokkos_test_cxx_std.cmake b/lib/kokkos/cmake/kokkos_test_cxx_std.cmake index 5b45674e05..a84e714064 100644 --- a/lib/kokkos/cmake/kokkos_test_cxx_std.cmake +++ b/lib/kokkos/cmake/kokkos_test_cxx_std.cmake @@ -1,101 +1,112 @@ -KOKKOS_CFG_DEPENDS(CXX_STD COMPILER_ID) +kokkos_cfg_depends(CXX_STD COMPILER_ID) -FUNCTION(kokkos_set_cxx_standard_feature standard) - SET(EXTENSION_NAME CMAKE_CXX${standard}_EXTENSION_COMPILE_OPTION) - SET(STANDARD_NAME CMAKE_CXX${standard}_STANDARD_COMPILE_OPTION) - SET(FEATURE_NAME cxx_std_${standard}) +function(kokkos_set_cxx_standard_feature standard) + set(EXTENSION_NAME CMAKE_CXX${standard}_EXTENSION_COMPILE_OPTION) + set(STANDARD_NAME CMAKE_CXX${standard}_STANDARD_COMPILE_OPTION) + set(FEATURE_NAME cxx_std_${standard}) #CMake's way of telling us that the standard (or extension) #flags are supported is the extension/standard variables - IF (NOT DEFINED CMAKE_CXX_EXTENSIONS) - IF(KOKKOS_DONT_ALLOW_EXTENSIONS) - GLOBAL_SET(KOKKOS_USE_CXX_EXTENSIONS OFF) - ELSE() - GLOBAL_SET(KOKKOS_USE_CXX_EXTENSIONS ON) - ENDIF() - ELSEIF(CMAKE_CXX_EXTENSIONS) - IF(KOKKOS_DONT_ALLOW_EXTENSIONS) - MESSAGE(FATAL_ERROR "The chosen configuration does not support CXX extensions flags: ${KOKKOS_DONT_ALLOW_EXTENSIONS}. Must set CMAKE_CXX_EXTENSIONS=OFF to continue") - ELSE() - GLOBAL_SET(KOKKOS_USE_CXX_EXTENSIONS ON) - ENDIF() - ELSE() - #For trilinos, we need to make sure downstream projects - GLOBAL_SET(KOKKOS_USE_CXX_EXTENSIONS OFF) - ENDIF() + if(NOT DEFINED CMAKE_CXX_EXTENSIONS) + if(KOKKOS_DONT_ALLOW_EXTENSIONS) + global_set(KOKKOS_USE_CXX_EXTENSIONS OFF) + else() + global_set(KOKKOS_USE_CXX_EXTENSIONS ON) + endif() + elseif(CMAKE_CXX_EXTENSIONS) + if(KOKKOS_DONT_ALLOW_EXTENSIONS) + message( + FATAL_ERROR + "The chosen configuration does not support CXX extensions flags: ${KOKKOS_DONT_ALLOW_EXTENSIONS}. Must set CMAKE_CXX_EXTENSIONS=OFF to continue" + ) + else() + global_set(KOKKOS_USE_CXX_EXTENSIONS ON) + endif() + endif() - IF (KOKKOS_USE_CXX_EXTENSIONS AND ${EXTENSION_NAME}) - MESSAGE(STATUS "Using ${${EXTENSION_NAME}} for C++${standard} extensions as feature") - GLOBAL_SET(KOKKOS_CXX_STANDARD_FEATURE ${FEATURE_NAME}) - ELSEIF(NOT KOKKOS_USE_CXX_EXTENSIONS AND ${STANDARD_NAME}) - MESSAGE(STATUS "Using ${${STANDARD_NAME}} for C++${standard} standard as feature") - IF (KOKKOS_CXX_COMPILER_ID STREQUAL NVIDIA AND (KOKKOS_CXX_HOST_COMPILER_ID STREQUAL GNU OR KOKKOS_CXX_HOST_COMPILER_ID STREQUAL Clang)) - IF(${KOKKOS_CXX_COMPILER_VERSION} VERSION_LESS 12.0.0) - SET(SUPPORTED_NVCC_FLAGS "-std=c++17") - ELSE() - SET(SUPPORTED_NVCC_FLAGS "-std=c++17" "-std=c++20") - ENDIF() - IF (NOT ${${STANDARD_NAME}} IN_LIST SUPPORTED_NVCC_FLAGS) - MESSAGE(FATAL_ERROR "CMake wants to use ${${STANDARD_NAME}} which is not supported by NVCC. Using a more recent host compiler or a more recent CMake version might help.") - ENDIF() - ENDIF() - GLOBAL_SET(KOKKOS_CXX_STANDARD_FEATURE ${FEATURE_NAME}) - ELSEIF (CMAKE_CXX_COMPILER_ID STREQUAL "MSVC" OR "x${CMAKE_CXX_SIMULATE_ID}" STREQUAL "xMSVC") + if(KOKKOS_USE_CXX_EXTENSIONS AND ${EXTENSION_NAME}) + message(STATUS "Using ${${EXTENSION_NAME}} for C++${standard} extensions as feature") + global_set(KOKKOS_CXX_STANDARD_FEATURE ${FEATURE_NAME}) + elseif(NOT KOKKOS_USE_CXX_EXTENSIONS AND ${STANDARD_NAME}) + message(STATUS "Using ${${STANDARD_NAME}} for C++${standard} standard as feature") + if(KOKKOS_CXX_COMPILER_ID STREQUAL NVIDIA AND (KOKKOS_CXX_HOST_COMPILER_ID STREQUAL GNU + OR KOKKOS_CXX_HOST_COMPILER_ID STREQUAL Clang) + ) + if(${KOKKOS_CXX_COMPILER_VERSION} VERSION_LESS 12.0.0) + set(SUPPORTED_NVCC_FLAGS "-std=c++17") + else() + set(SUPPORTED_NVCC_FLAGS "-std=c++17" "-std=c++20") + endif() + if(NOT ${${STANDARD_NAME}} IN_LIST SUPPORTED_NVCC_FLAGS) + message( + FATAL_ERROR + "CMake wants to use ${${STANDARD_NAME}} which is not supported by NVCC. Using a more recent host compiler or a more recent CMake version might help." + ) + endif() + endif() + global_set(KOKKOS_CXX_STANDARD_FEATURE ${FEATURE_NAME}) + elseif(CMAKE_CXX_COMPILER_ID STREQUAL "MSVC" OR "x${CMAKE_CXX_SIMULATE_ID}" STREQUAL "xMSVC") #MSVC doesn't need a command line flag, that doesn't mean it has no support - MESSAGE(STATUS "Using no flag for C++${standard} standard as feature") - GLOBAL_SET(KOKKOS_CXX_STANDARD_FEATURE ${FEATURE_NAME}) - ELSEIF((KOKKOS_CXX_COMPILER_ID STREQUAL "NVIDIA") AND WIN32) - MESSAGE(STATUS "Using no flag for C++${standard} standard as feature") - GLOBAL_SET(KOKKOS_CXX_STANDARD_FEATURE "") - ELSEIF((KOKKOS_CXX_COMPILER_ID STREQUAL "Fujitsu")) - MESSAGE(STATUS "Using no flag for C++${standard} standard as feature") - GLOBAL_SET(KOKKOS_CXX_STANDARD_FEATURE "") - ELSE() + message(STATUS "Using no flag for C++${standard} standard as feature") + global_set(KOKKOS_CXX_STANDARD_FEATURE ${FEATURE_NAME}) + elseif((KOKKOS_CXX_COMPILER_ID STREQUAL "NVIDIA") AND WIN32) + message(STATUS "Using no flag for C++${standard} standard as feature") + global_set(KOKKOS_CXX_STANDARD_FEATURE "") + elseif(KOKKOS_CXX_COMPILER_ID STREQUAL "Fujitsu") + message(STATUS "Using no flag for C++${standard} standard as feature") + global_set(KOKKOS_CXX_STANDARD_FEATURE "") + else() #nope, we can't do anything here - MESSAGE(WARNING "C++${standard} is not supported as a compiler feature. We will choose custom flags for now, but this behavior has been deprecated. Please open an issue at https://github.com/kokkos/kokkos/issues reporting that ${KOKKOS_CXX_COMPILER_ID} ${KOKKOS_CXX_COMPILER_VERSION} failed for ${KOKKOS_CXX_STANDARD}, preferably including your CMake command.") - GLOBAL_SET(KOKKOS_CXX_STANDARD_FEATURE "") - ENDIF() + message( + WARNING + "C++${standard} is not supported as a compiler feature. We will choose custom flags for now, but this behavior has been deprecated. Please open an issue at https://github.com/kokkos/kokkos/issues reporting that ${KOKKOS_CXX_COMPILER_ID} ${KOKKOS_CXX_COMPILER_VERSION} failed for ${KOKKOS_CXX_STANDARD}, preferably including your CMake command." + ) + global_set(KOKKOS_CXX_STANDARD_FEATURE "") + endif() - IF((NOT WIN32) AND (NOT ("${KOKKOS_CXX_COMPILER_ID}" STREQUAL "Fujitsu"))) - IF(NOT ${FEATURE_NAME} IN_LIST CMAKE_CXX_COMPILE_FEATURES) - MESSAGE(FATAL_ERROR "Compiler ${KOKKOS_CXX_COMPILER_ID} should support ${FEATURE_NAME}, but CMake reports feature not supported") - ENDIF() - ENDIF() -ENDFUNCTION() + if((NOT WIN32) AND (NOT ("${KOKKOS_CXX_COMPILER_ID}" STREQUAL "Fujitsu"))) + if(NOT ${FEATURE_NAME} IN_LIST CMAKE_CXX_COMPILE_FEATURES) + message( + FATAL_ERROR + "Compiler ${KOKKOS_CXX_COMPILER_ID} should support ${FEATURE_NAME}, but CMake reports feature not supported" + ) + endif() + endif() +endfunction() -IF(KOKKOS_CXX_STANDARD STREQUAL "17") +if(KOKKOS_CXX_STANDARD STREQUAL "17") kokkos_set_cxx_standard_feature(17) - SET(KOKKOS_CXX_INTERMEDIATE_STANDARD "1Z") - SET(KOKKOS_ENABLE_CXX17 ON) -ELSEIF(KOKKOS_CXX_STANDARD STREQUAL "20") + set(KOKKOS_CXX_INTERMEDIATE_STANDARD "1Z") + set(KOKKOS_ENABLE_CXX17 ON) +elseif(KOKKOS_CXX_STANDARD STREQUAL "20") kokkos_set_cxx_standard_feature(20) - SET(KOKKOS_CXX_INTERMEDIATE_STANDARD "2A") - SET(KOKKOS_ENABLE_CXX20 ON) -ELSEIF(KOKKOS_CXX_STANDARD STREQUAL "23") + set(KOKKOS_CXX_INTERMEDIATE_STANDARD "2A") + set(KOKKOS_ENABLE_CXX20 ON) +elseif(KOKKOS_CXX_STANDARD STREQUAL "23") kokkos_set_cxx_standard_feature(23) - SET(KOKKOS_CXX_INTERMEDIATE_STANDARD "2B") - SET(KOKKOS_ENABLE_CXX23 ON) -ELSEIF(KOKKOS_CXX_STANDARD STREQUAL "26") + set(KOKKOS_CXX_INTERMEDIATE_STANDARD "2B") + set(KOKKOS_ENABLE_CXX23 ON) +elseif(KOKKOS_CXX_STANDARD STREQUAL "26") kokkos_set_cxx_standard_feature(26) - SET(KOKKOS_CXX_INTERMEDIATE_STANDARD "2C") - SET(KOKKOS_ENABLE_CXX26 ON) -ELSE() - MESSAGE(FATAL_ERROR "Kokkos requires C++17 or newer but requested ${KOKKOS_CXX_STANDARD}!") -ENDIF() + set(KOKKOS_CXX_INTERMEDIATE_STANDARD "2C") + set(KOKKOS_ENABLE_CXX26 ON) +else() + message(FATAL_ERROR "Kokkos requires C++17 or newer but requested ${KOKKOS_CXX_STANDARD}!") +endif() # Enforce that we can compile a simple C++17 program -TRY_COMPILE(CAN_COMPILE_CPP17 - ${KOKKOS_TOP_BUILD_DIR}/corner_cases - ${KOKKOS_SOURCE_DIR}/cmake/compile_tests/cplusplus17.cpp - OUTPUT_VARIABLE ERROR_MESSAGE - CXX_STANDARD 17 +try_compile( + CAN_COMPILE_CPP17 ${KOKKOS_TOP_BUILD_DIR}/corner_cases ${KOKKOS_SOURCE_DIR}/cmake/compile_tests/cplusplus17.cpp + OUTPUT_VARIABLE ERROR_MESSAGE CXX_STANDARD 17 ) -if (NOT CAN_COMPILE_CPP17) - UNSET(CAN_COMPILE_CPP17 CACHE) #make sure CMake always re-runs this - MESSAGE(FATAL_ERROR "C++${KOKKOS_CXX_STANDARD}-compliant compiler detected, but unable to compile C++17 or later program. Verify that ${CMAKE_CXX_COMPILER_ID}:${CMAKE_CXX_COMPILER_VERSION} is set up correctly (e.g., check that correct library headers are being used).\nFailing output:\n ${ERROR_MESSAGE}") -ENDIF() -UNSET(CAN_COMPILE_CPP17 CACHE) #make sure CMake always re-runs this - +if(NOT CAN_COMPILE_CPP17) + unset(CAN_COMPILE_CPP17 CACHE) #make sure CMake always re-runs this + message( + FATAL_ERROR + "C++${KOKKOS_CXX_STANDARD}-compliant compiler detected, but unable to compile C++17 or later program. Verify that ${CMAKE_CXX_COMPILER_ID}:${CMAKE_CXX_COMPILER_VERSION} is set up correctly (e.g., check that correct library headers are being used).\nFailing output:\n ${ERROR_MESSAGE}" + ) +endif() +unset(CAN_COMPILE_CPP17 CACHE) #make sure CMake always re-runs this # Enforce that extensions are turned off for nvcc_wrapper. # For compiling CUDA code using nvcc_wrapper, we will use the host compiler's @@ -105,66 +116,70 @@ UNSET(CAN_COMPILE_CPP17 CACHE) #make sure CMake always re-runs this # that we can only use host compilers for CUDA builds that use those flags. # It also means that extensions (gnu++17) can't be turned on for CUDA builds. -IF(KOKKOS_CXX_COMPILER_ID STREQUAL NVIDIA) - IF(NOT DEFINED CMAKE_CXX_EXTENSIONS) - SET(CMAKE_CXX_EXTENSIONS OFF) - ELSEIF(CMAKE_CXX_EXTENSIONS) - MESSAGE(FATAL_ERROR "NVCC doesn't support C++ extensions. Set -DCMAKE_CXX_EXTENSIONS=OFF") - ENDIF() -ENDIF() +if(KOKKOS_CXX_COMPILER_ID STREQUAL NVIDIA) + if(NOT DEFINED CMAKE_CXX_EXTENSIONS) + set(CMAKE_CXX_EXTENSIONS OFF) + elseif(CMAKE_CXX_EXTENSIONS) + message(FATAL_ERROR "NVCC doesn't support C++ extensions. Set -DCMAKE_CXX_EXTENSIONS=OFF") + endif() +endif() -IF(KOKKOS_ENABLE_CUDA) +if(KOKKOS_ENABLE_CUDA) # ENFORCE that the compiler can compile CUDA code. - IF(KOKKOS_CXX_COMPILER_ID STREQUAL Clang) - IF(KOKKOS_CXX_COMPILER_VERSION VERSION_LESS 4.0.0) - MESSAGE(FATAL_ERROR "Compiling CUDA code directly with Clang requires version 4.0.0 or higher.") - ENDIF() - IF(NOT DEFINED CMAKE_CXX_EXTENSIONS) - SET(CMAKE_CXX_EXTENSIONS OFF) - ELSEIF(CMAKE_CXX_EXTENSIONS) - MESSAGE(FATAL_ERROR "Compiling CUDA code with clang doesn't support C++ extensions. Set -DCMAKE_CXX_EXTENSIONS=OFF") - ENDIF() - ELSEIF(NOT KOKKOS_CXX_COMPILER_ID STREQUAL NVIDIA) - MESSAGE(FATAL_ERROR "Invalid compiler for CUDA. The compiler must be nvcc_wrapper or Clang or use kokkos_launch_compiler, but compiler ID was ${KOKKOS_CXX_COMPILER_ID}") - ENDIF() -ENDIF() + if(KOKKOS_CXX_COMPILER_ID STREQUAL Clang) + if(KOKKOS_CXX_COMPILER_VERSION VERSION_LESS 4.0.0) + message(FATAL_ERROR "Compiling CUDA code directly with Clang requires version 4.0.0 or higher.") + endif() + if(NOT DEFINED CMAKE_CXX_EXTENSIONS) + set(CMAKE_CXX_EXTENSIONS OFF) + elseif(CMAKE_CXX_EXTENSIONS) + message( + FATAL_ERROR "Compiling CUDA code with clang doesn't support C++ extensions. Set -DCMAKE_CXX_EXTENSIONS=OFF" + ) + endif() + elseif(NOT KOKKOS_CXX_COMPILER_ID STREQUAL NVIDIA) + message( + FATAL_ERROR + "Invalid compiler for CUDA. The compiler must be nvcc_wrapper or Clang or use kokkos_launch_compiler, but compiler ID was ${KOKKOS_CXX_COMPILER_ID}" + ) + endif() +endif() -IF (NOT KOKKOS_CXX_STANDARD_FEATURE) +if(NOT KOKKOS_CXX_STANDARD_FEATURE) #we need to pick the C++ flags ourselves - UNSET(CMAKE_CXX_STANDARD) - UNSET(CMAKE_CXX_STANDARD CACHE) - IF(KOKKOS_CXX_COMPILER_ID STREQUAL Cray) - INCLUDE(${KOKKOS_SRC_PATH}/cmake/cray.cmake) + unset(CMAKE_CXX_STANDARD) + unset(CMAKE_CXX_STANDARD CACHE) + if(KOKKOS_CXX_COMPILER_ID STREQUAL Cray) + include(${KOKKOS_SRC_PATH}/cmake/cray.cmake) kokkos_set_cray_flags(${KOKKOS_CXX_STANDARD} ${KOKKOS_CXX_INTERMEDIATE_STANDARD}) - ELSEIF(KOKKOS_CXX_COMPILER_ID STREQUAL NVHPC) - INCLUDE(${KOKKOS_SRC_PATH}/cmake/pgi.cmake) + elseif(KOKKOS_CXX_COMPILER_ID STREQUAL NVHPC) + include(${KOKKOS_SRC_PATH}/cmake/pgi.cmake) kokkos_set_pgi_flags(${KOKKOS_CXX_STANDARD} ${KOKKOS_CXX_INTERMEDIATE_STANDARD}) - ELSEIF(KOKKOS_CXX_COMPILER_ID STREQUAL Intel) - INCLUDE(${KOKKOS_SRC_PATH}/cmake/intel.cmake) + elseif(KOKKOS_CXX_COMPILER_ID STREQUAL Intel) + include(${KOKKOS_SRC_PATH}/cmake/intel.cmake) kokkos_set_intel_flags(${KOKKOS_CXX_STANDARD} ${KOKKOS_CXX_INTERMEDIATE_STANDARD}) - ELSEIF((KOKKOS_CXX_COMPILER_ID STREQUAL "MSVC") OR ((KOKKOS_CXX_COMPILER_ID STREQUAL "NVIDIA") AND WIN32)) - INCLUDE(${KOKKOS_SRC_PATH}/cmake/msvc.cmake) + elseif((KOKKOS_CXX_COMPILER_ID STREQUAL "MSVC") OR ((KOKKOS_CXX_COMPILER_ID STREQUAL "NVIDIA") AND WIN32)) + include(${KOKKOS_SRC_PATH}/cmake/msvc.cmake) kokkos_set_msvc_flags(${KOKKOS_CXX_STANDARD} ${KOKKOS_CXX_INTERMEDIATE_STANDARD}) - ELSE() - INCLUDE(${KOKKOS_SRC_PATH}/cmake/gnu.cmake) + else() + include(${KOKKOS_SRC_PATH}/cmake/gnu.cmake) kokkos_set_gnu_flags(${KOKKOS_CXX_STANDARD} ${KOKKOS_CXX_INTERMEDIATE_STANDARD}) - ENDIF() + endif() #check that the compiler accepts the C++ standard flag - INCLUDE(CheckCXXCompilerFlag) - IF (DEFINED CXX_STD_FLAGS_ACCEPTED) - UNSET(CXX_STD_FLAGS_ACCEPTED CACHE) - ENDIF() - CHECK_CXX_COMPILER_FLAG("${KOKKOS_CXX_STANDARD_FLAG}" CXX_STD_FLAGS_ACCEPTED) - IF (NOT CXX_STD_FLAGS_ACCEPTED) - CHECK_CXX_COMPILER_FLAG("${KOKKOS_CXX_INTERMEDIATE_STANDARD_FLAG}" CXX_INT_STD_FLAGS_ACCEPTED) - IF (NOT CXX_INT_STD_FLAGS_ACCEPTED) - MESSAGE(FATAL_ERROR "${KOKKOS_CXX_COMPILER_ID} did not accept ${KOKKOS_CXX_STANDARD_FLAG} or ${KOKKOS_CXX_INTERMEDIATE_STANDARD_FLAG}. You likely need to reduce the level of the C++ standard from ${KOKKOS_CXX_STANDARD}") - ENDIF() - SET(KOKKOS_CXX_STANDARD_FLAG ${KOKKOS_CXX_INTERMEDIATE_STANDARD_FLAG}) - ENDIF() - MESSAGE(STATUS "Compiler features not supported, but ${KOKKOS_CXX_COMPILER_ID} accepts ${KOKKOS_CXX_STANDARD_FLAG}") -ENDIF() - - - - + include(CheckCXXCompilerFlag) + if(DEFINED CXX_STD_FLAGS_ACCEPTED) + unset(CXX_STD_FLAGS_ACCEPTED CACHE) + endif() + check_cxx_compiler_flag("${KOKKOS_CXX_STANDARD_FLAG}" CXX_STD_FLAGS_ACCEPTED) + if(NOT CXX_STD_FLAGS_ACCEPTED) + check_cxx_compiler_flag("${KOKKOS_CXX_INTERMEDIATE_STANDARD_FLAG}" CXX_INT_STD_FLAGS_ACCEPTED) + if(NOT CXX_INT_STD_FLAGS_ACCEPTED) + message( + FATAL_ERROR + "${KOKKOS_CXX_COMPILER_ID} did not accept ${KOKKOS_CXX_STANDARD_FLAG} or ${KOKKOS_CXX_INTERMEDIATE_STANDARD_FLAG}. You likely need to reduce the level of the C++ standard from ${KOKKOS_CXX_STANDARD}" + ) + endif() + set(KOKKOS_CXX_STANDARD_FLAG ${KOKKOS_CXX_INTERMEDIATE_STANDARD_FLAG}) + endif() + message(STATUS "Compiler features not supported, but ${KOKKOS_CXX_COMPILER_ID} accepts ${KOKKOS_CXX_STANDARD_FLAG}") +endif() diff --git a/lib/kokkos/cmake/kokkos_tpls.cmake b/lib/kokkos/cmake/kokkos_tpls.cmake index cda9e0d600..f43aff4d1f 100644 --- a/lib/kokkos/cmake/kokkos_tpls.cmake +++ b/lib/kokkos/cmake/kokkos_tpls.cmake @@ -1,126 +1,120 @@ -KOKKOS_CFG_DEPENDS(TPLS OPTIONS) -KOKKOS_CFG_DEPENDS(TPLS DEVICES) -KOKKOS_CFG_DEPENDS(TPLS COMPILER_ID) +kokkos_cfg_depends(TPLS OPTIONS) +kokkos_cfg_depends(TPLS DEVICES) +kokkos_cfg_depends(TPLS COMPILER_ID) -FUNCTION(KOKKOS_TPL_OPTION PKG DEFAULT) - CMAKE_PARSE_ARGUMENTS(PARSED - "" - "TRIBITS" - "" - ${ARGN}) +function(KOKKOS_TPL_OPTION PKG DEFAULT) + cmake_parse_arguments(PARSED "" "TRIBITS" "" ${ARGN}) - IF (PARSED_TRIBITS) + if(PARSED_TRIBITS) #this is also a TPL option you can activate with Tribits - IF (NOT "${TPL_ENABLE_${PARSED_TRIBITS}}" STREQUAL "") + if(NOT "${TPL_ENABLE_${PARSED_TRIBITS}}" STREQUAL "") #Tribits brought its own default that should take precedence - SET(DEFAULT ${TPL_ENABLE_${PARSED_TRIBITS}}) - ENDIF() - ENDIF() + set(DEFAULT ${TPL_ENABLE_${PARSED_TRIBITS}}) + endif() + endif() - KOKKOS_ENABLE_OPTION(${PKG} ${DEFAULT} "Whether to enable the ${PKG} library") - KOKKOS_OPTION(${PKG}_DIR "" PATH "Location of ${PKG} library") - SET(KOKKOS_ENABLE_${PKG} ${KOKKOS_ENABLE_${PKG}} PARENT_SCOPE) - SET(KOKKOS_${PKG}_DIR ${KOKKOS_${PKG}_DIR} PARENT_SCOPE) + kokkos_enable_option(${PKG} ${DEFAULT} "Whether to enable the ${PKG} library") + kokkos_option(${PKG}_DIR "" PATH "Location of ${PKG} library") + set(KOKKOS_ENABLE_${PKG} ${KOKKOS_ENABLE_${PKG}} PARENT_SCOPE) + set(KOKKOS_${PKG}_DIR ${KOKKOS_${PKG}_DIR} PARENT_SCOPE) +endfunction() - IF (KOKKOS_HAS_TRILINOS - AND KOKKOS_ENABLE_${PKG} - AND NOT PARSED_TRIBITS) - #this TPL was enabled, but it is not valid to use inside of TriBITS - MESSAGE(FATAL_ERROR "Enabled TPL ${PKG} inside TriBITS build, " - "but this can only be enabled in a standalone build") - ENDIF() -ENDFUNCTION() - -KOKKOS_TPL_OPTION(HWLOC Off TRIBITS HWLOC) -KOKKOS_TPL_OPTION(CUDA ${Kokkos_ENABLE_CUDA} TRIBITS CUDA) -IF(KOKKOS_ENABLE_HIP AND NOT KOKKOS_CXX_COMPILER_ID STREQUAL HIPCC AND NOT - KOKKOS_HAS_TRILINOS) - SET(ROCM_DEFAULT ON) -ELSE() - SET(ROCM_DEFAULT OFF) -ENDIF() -IF(KOKKOS_ENABLE_HIP AND NOT KOKKOS_HAS_TRILINOS) - SET(ROCTHRUST_DEFAULT ON) -ELSE() - SET(ROCTHRUST_DEFAULT OFF) -ENDIF() -KOKKOS_TPL_OPTION(ROCM ${ROCM_DEFAULT}) -KOKKOS_TPL_OPTION(ROCTHRUST ${ROCTHRUST_DEFAULT}) - -IF(KOKKOS_ENABLE_SYCL AND NOT KOKKOS_HAS_TRILINOS) - SET(ONEDPL_DEFAULT ON) -ELSE() - SET(ONEDPL_DEFAULT OFF) -ENDIF() -KOKKOS_TPL_OPTION(ONEDPL ${ONEDPL_DEFAULT}) - -IF (WIN32) - SET(LIBDL_DEFAULT Off) -ELSE() - SET(LIBDL_DEFAULT On) -ENDIF() -KOKKOS_TPL_OPTION(LIBDL ${LIBDL_DEFAULT} TRIBITS DLlib) - -IF(Trilinos_ENABLE_Kokkos AND TPL_ENABLE_HPX) -SET(HPX_DEFAULT ON) -ELSE() -SET(HPX_DEFAULT OFF) -ENDIF() -KOKKOS_TPL_OPTION(HPX ${HPX_DEFAULT}) - -KOKKOS_TPL_OPTION(THREADS ${Kokkos_ENABLE_THREADS} TRIBITS Pthread) - -IF(Trilinos_ENABLE_Kokkos AND TPL_ENABLE_quadmath) - SET(LIBQUADMATH_DEFAULT ON) -ELSE() - SET(LIBQUADMATH_DEFAULT OFF) -ENDIF() -KOKKOS_TPL_OPTION(LIBQUADMATH ${LIBQUADMATH_DEFAULT} TRIBITS quadmath) - -#Make sure we use our local FindKokkosCuda.cmake -KOKKOS_IMPORT_TPL(HPX INTERFACE) -KOKKOS_IMPORT_TPL(CUDA INTERFACE) -KOKKOS_IMPORT_TPL(HWLOC) -KOKKOS_IMPORT_TPL(LIBDL) -IF (NOT WIN32) - KOKKOS_IMPORT_TPL(THREADS INTERFACE) -ENDIF() -IF (NOT KOKKOS_ENABLE_COMPILE_AS_CMAKE_LANGUAGE) - KOKKOS_IMPORT_TPL(ROCM INTERFACE) -ENDIF() -KOKKOS_IMPORT_TPL(ONEDPL INTERFACE) -KOKKOS_IMPORT_TPL(LIBQUADMATH) -KOKKOS_IMPORT_TPL(ROCTHRUST) - -IF (Kokkos_ENABLE_DESUL_ATOMICS_EXTERNAL) - find_package(desul REQUIRED COMPONENTS atomics) - KOKKOS_EXPORT_CMAKE_TPL(desul REQUIRED COMPONENTS atomics) -ENDIF() - -if (Kokkos_ENABLE_IMPL_MDSPAN AND Kokkos_ENABLE_MDSPAN_EXTERNAL) - find_package(mdspan REQUIRED) - KOKKOS_EXPORT_CMAKE_TPL(mdspan REQUIRED) +kokkos_tpl_option(HWLOC Off TRIBITS HWLOC) +kokkos_tpl_option(CUDA ${Kokkos_ENABLE_CUDA} TRIBITS CUDA) +if(KOKKOS_ENABLE_HIP AND NOT KOKKOS_CXX_COMPILER_ID STREQUAL HIPCC) + set(ROCM_DEFAULT ON) +else() + set(ROCM_DEFAULT OFF) +endif() +if(KOKKOS_ENABLE_HIP) + set(ROCTHRUST_DEFAULT ON) +else() + set(ROCTHRUST_DEFAULT OFF) +endif() +kokkos_tpl_option(ROCM ${ROCM_DEFAULT}) +kokkos_tpl_option(ROCTHRUST ${ROCTHRUST_DEFAULT}) +if(Kokkos_ENABLE_ROCTHRUST) + include(CheckCXXSourceCompiles) + check_cxx_source_compiles( + " + #include + int main() { + static_assert(_GLIBCXX_RELEASE < 9); + return 0; + } + " + Kokkos_ENABLE_IMPL_SKIP_NO_RTTI_FLAG + ) endif() -IF (Kokkos_ENABLE_OPENMP) - find_package(OpenMP REQUIRED COMPONENTS CXX) - # FIXME_TRILINOS Trilinos doesn't allow for Kokkos to use find_dependency - # so we just append the flags here instead of linking with the OpenMP target. - IF(KOKKOS_HAS_TRILINOS) - COMPILER_SPECIFIC_FLAGS(DEFAULT ${OpenMP_CXX_FLAGS}) - ELSE() - KOKKOS_EXPORT_CMAKE_TPL(OpenMP REQUIRED COMPONENTS CXX) - ENDIF() - IF(Kokkos_ENABLE_HIP AND KOKKOS_COMPILE_LANGUAGE STREQUAL HIP) - GLOBAL_APPEND(KOKKOS_AMDGPU_OPTIONS ${OpenMP_CXX_FLAGS}) - ENDIF() - IF(Kokkos_ENABLE_CUDA AND KOKKOS_COMPILE_LANGUAGE STREQUAL CUDA) - GLOBAL_APPEND(KOKKOS_CUDA_OPTIONS -Xcompiler ${OpenMP_CXX_FLAGS}) - ENDIF() -ENDIF() +if(KOKKOS_ENABLE_SYCL) + set(ONEDPL_DEFAULT ON) +else() + set(ONEDPL_DEFAULT OFF) +endif() +kokkos_tpl_option(ONEDPL ${ONEDPL_DEFAULT}) + +if(WIN32) + set(LIBDL_DEFAULT Off) +else() + set(LIBDL_DEFAULT On) +endif() +kokkos_tpl_option(LIBDL ${LIBDL_DEFAULT} TRIBITS DLlib) + +if(Trilinos_ENABLE_Kokkos AND TPL_ENABLE_HPX) + set(HPX_DEFAULT ON) +else() + set(HPX_DEFAULT OFF) +endif() +kokkos_tpl_option(HPX ${HPX_DEFAULT}) + +kokkos_tpl_option(THREADS ${Kokkos_ENABLE_THREADS} TRIBITS Pthread) + +if(Trilinos_ENABLE_Kokkos AND TPL_ENABLE_quadmath) + set(LIBQUADMATH_DEFAULT ON) +else() + set(LIBQUADMATH_DEFAULT OFF) +endif() +kokkos_tpl_option(LIBQUADMATH ${LIBQUADMATH_DEFAULT} TRIBITS quadmath) + +#Make sure we use our local FindKokkosCuda.cmake +kokkos_import_tpl(HPX INTERFACE) +kokkos_import_tpl(CUDA INTERFACE) +kokkos_import_tpl(HWLOC) +kokkos_import_tpl(LIBDL) +if(NOT WIN32) + kokkos_import_tpl(THREADS INTERFACE) +endif() +if(NOT KOKKOS_ENABLE_COMPILE_AS_CMAKE_LANGUAGE) + kokkos_import_tpl(ROCM INTERFACE) +endif() +kokkos_import_tpl(ONEDPL INTERFACE) +kokkos_import_tpl(LIBQUADMATH) +kokkos_import_tpl(ROCTHRUST) + +if(Kokkos_ENABLE_DESUL_ATOMICS_EXTERNAL) + find_package(desul REQUIRED COMPONENTS atomics) + kokkos_export_cmake_tpl(desul REQUIRED COMPONENTS atomics) +endif() + +if(Kokkos_ENABLE_IMPL_MDSPAN AND Kokkos_ENABLE_MDSPAN_EXTERNAL) + find_package(mdspan REQUIRED) + kokkos_export_cmake_tpl(mdspan REQUIRED) +endif() + +if(Kokkos_ENABLE_OPENMP) + find_package(OpenMP 3.0 REQUIRED COMPONENTS CXX) + kokkos_export_cmake_tpl(OpenMP REQUIRED COMPONENTS CXX) + if(Kokkos_ENABLE_HIP AND KOKKOS_COMPILE_LANGUAGE STREQUAL HIP) + global_append(KOKKOS_AMDGPU_OPTIONS ${OpenMP_CXX_FLAGS}) + endif() + if(Kokkos_ENABLE_CUDA AND KOKKOS_COMPILE_LANGUAGE STREQUAL CUDA) + global_append(KOKKOS_CUDA_OPTIONS -Xcompiler ${OpenMP_CXX_FLAGS}) + endif() +endif() #Convert list to newlines (which CMake doesn't always like in cache variables) -STRING(REPLACE ";" "\n" KOKKOS_TPL_EXPORT_TEMP "${KOKKOS_TPL_EXPORTS}") +string(REPLACE ";" "\n" KOKKOS_TPL_EXPORT_TEMP "${KOKKOS_TPL_EXPORTS}") #Convert to a regular variable -UNSET(KOKKOS_TPL_EXPORTS CACHE) -SET(KOKKOS_TPL_EXPORTS ${KOKKOS_TPL_EXPORT_TEMP}) +unset(KOKKOS_TPL_EXPORTS CACHE) +set(KOKKOS_TPL_EXPORTS ${KOKKOS_TPL_EXPORT_TEMP}) diff --git a/lib/kokkos/cmake/kokkos_tribits.cmake b/lib/kokkos/cmake/kokkos_tribits.cmake index 6da543a2c8..2fda803b11 100644 --- a/lib/kokkos/cmake/kokkos_tribits.cmake +++ b/lib/kokkos/cmake/kokkos_tribits.cmake @@ -1,82 +1,47 @@ #These are tribits wrappers only ever called by Kokkos itself -INCLUDE(CMakeParseArguments) -INCLUDE(CTest) -INCLUDE(GNUInstallDirs) +include(CMakeParseArguments) +include(CTest) +include(GNUInstallDirs) -MESSAGE(STATUS "The project name is: ${PROJECT_NAME}") +message(STATUS "The project name is: ${PROJECT_NAME}") -IF(GTest_FOUND) - SET(KOKKOS_GTEST_LIB GTest::gtest) - MESSAGE(STATUS "Using gtest found in ${GTest_DIR}") -ELSE() # fallback to internal gtest - SET(KOKKOS_GTEST_LIB kokkos_gtest) - MESSAGE(STATUS "Using internal gtest for testing") -ENDIF() +if(GTest_FOUND) + set(KOKKOS_GTEST_LIB GTest::gtest) + message(STATUS "Using gtest found in ${GTest_DIR}") +else() # fallback to internal gtest + set(KOKKOS_GTEST_LIB kokkos_gtest) + message(STATUS "Using internal gtest for testing") +endif() -FUNCTION(VERIFY_EMPTY CONTEXT) +function(VERIFY_EMPTY CONTEXT) if(${ARGN}) - MESSAGE(FATAL_ERROR "Kokkos does not support all of Tribits. Unhandled arguments in ${CONTEXT}:\n${ARGN}") + message(FATAL_ERROR "Kokkos does not support all of Tribits. Unhandled arguments in ${CONTEXT}:\n${ARGN}") endif() -ENDFUNCTION() +endfunction() -#Leave this here for now - but only do for tribits -#This breaks the standalone CMake -IF (KOKKOS_HAS_TRILINOS) - IF(NOT DEFINED ${PROJECT_NAME}_ENABLE_OpenMP) - SET(${PROJECT_NAME}_ENABLE_OpenMP OFF) - ENDIF() +macro(KOKKOS_PROCESS_SUBPACKAGES) + add_subdirectory(core) + add_subdirectory(containers) + add_subdirectory(algorithms) + add_subdirectory(simd) + add_subdirectory(example) + add_subdirectory(benchmarks) +endmacro() - IF(NOT DEFINED ${PROJECT_NAME}_ENABLE_HPX) - SET(${PROJECT_NAME}_ENABLE_HPX OFF) - ENDIF() +macro(KOKKOS_INTERNAL_ADD_LIBRARY_INSTALL LIBRARY_NAME) + kokkos_lib_type(${LIBRARY_NAME} INCTYPE) + target_include_directories(${LIBRARY_NAME} ${INCTYPE} $) - IF(NOT DEFINED ${PROJECT_NAME}_ENABLE_DEBUG) - SET(${PROJECT_NAME}_ENABLE_DEBUG OFF) - ENDIF() - - IF(NOT DEFINED ${PROJECT_NAME}_ENABLE_TESTS) - SET(${PROJECT_NAME}_ENABLE_TESTS OFF) - ENDIF() - - IF(NOT DEFINED TPL_ENABLE_Pthread) - SET(TPL_ENABLE_Pthread OFF) - ENDIF() -ENDIF() - -MACRO(KOKKOS_PROCESS_SUBPACKAGES) - ADD_SUBDIRECTORY(core) - ADD_SUBDIRECTORY(containers) - ADD_SUBDIRECTORY(algorithms) - ADD_SUBDIRECTORY(simd) - if (NOT KOKKOS_HAS_TRILINOS) - ADD_SUBDIRECTORY(example) - ADD_SUBDIRECTORY(benchmarks) - endif() -ENDMACRO() - -MACRO(KOKKOS_PACKAGE_DEF) - if (KOKKOS_HAS_TRILINOS) - TRIBITS_PACKAGE_DEF() - else() - #do nothing - endif() -ENDMACRO() - -MACRO(KOKKOS_INTERNAL_ADD_LIBRARY_INSTALL LIBRARY_NAME) - KOKKOS_LIB_TYPE(${LIBRARY_NAME} INCTYPE) - TARGET_INCLUDE_DIRECTORIES(${LIBRARY_NAME} ${INCTYPE} $) - - INSTALL( + install( TARGETS ${LIBRARY_NAME} EXPORT ${PROJECT_NAME} RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} - ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR} - COMPONENT ${PACKAGE_NAME} + ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR} COMPONENT ${PACKAGE_NAME} ) - INSTALL( + install( TARGETS ${LIBRARY_NAME} EXPORT KokkosTargets RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} @@ -84,157 +49,131 @@ MACRO(KOKKOS_INTERNAL_ADD_LIBRARY_INSTALL LIBRARY_NAME) ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR} ) - VERIFY_EMPTY(KOKKOS_ADD_LIBRARY ${PARSE_UNPARSED_ARGUMENTS}) -ENDMACRO() + verify_empty(KOKKOS_ADD_LIBRARY ${PARSE_UNPARSED_ARGUMENTS}) +endmacro() -FUNCTION(KOKKOS_ADD_EXECUTABLE ROOT_NAME) - if (KOKKOS_HAS_TRILINOS) - TRIBITS_ADD_EXECUTABLE(${ROOT_NAME} ${ARGN}) - else() - CMAKE_PARSE_ARGUMENTS(PARSE - "TESTONLY" - "" - "SOURCES;TESTONLYLIBS" - ${ARGN}) +function(KOKKOS_ADD_EXECUTABLE ROOT_NAME) + cmake_parse_arguments(PARSE "TESTONLY" "" "SOURCES;TESTONLYLIBS" ${ARGN}) - SET_SOURCE_FILES_PROPERTIES(${PARSE_SOURCES} PROPERTIES LANGUAGE ${KOKKOS_COMPILE_LANGUAGE}) + set_source_files_properties(${PARSE_SOURCES} PROPERTIES LANGUAGE ${KOKKOS_COMPILE_LANGUAGE}) - SET(EXE_NAME ${PACKAGE_NAME}_${ROOT_NAME}) - ADD_EXECUTABLE(${EXE_NAME} ${PARSE_SOURCES}) - IF (PARSE_TESTONLYLIBS) - TARGET_LINK_LIBRARIES(${EXE_NAME} PRIVATE ${PARSE_TESTONLYLIBS}) - ENDIF() - VERIFY_EMPTY(KOKKOS_ADD_EXECUTABLE ${PARSE_UNPARSED_ARGUMENTS}) - #All executables must link to all the kokkos targets - #This is just private linkage because exe is final - TARGET_LINK_LIBRARIES(${EXE_NAME} PRIVATE Kokkos::kokkos) + set(EXE_NAME ${PACKAGE_NAME}_${ROOT_NAME}) + add_executable(${EXE_NAME} ${PARSE_SOURCES}) + if(PARSE_TESTONLYLIBS) + target_link_libraries(${EXE_NAME} PRIVATE ${PARSE_TESTONLYLIBS}) endif() -ENDFUNCTION() + verify_empty(KOKKOS_ADD_EXECUTABLE ${PARSE_UNPARSED_ARGUMENTS}) + #All executables must link to all the kokkos targets + #This is just private linkage because exe is final + target_link_libraries(${EXE_NAME} PRIVATE Kokkos::kokkos) +endfunction() -FUNCTION(KOKKOS_ADD_EXECUTABLE_AND_TEST ROOT_NAME) - CMAKE_PARSE_ARGUMENTS(PARSE - "" - "" - "SOURCES;CATEGORIES;ARGS" - ${ARGN}) - VERIFY_EMPTY(KOKKOS_ADD_EXECUTABLE_AND_TEST ${PARSE_UNPARSED_ARGUMENTS}) +function(KOKKOS_ADD_EXECUTABLE_AND_TEST ROOT_NAME) + cmake_parse_arguments(PARSE "" "" "SOURCES;CATEGORIES;ARGS" ${ARGN}) + verify_empty(KOKKOS_ADD_EXECUTABLE_AND_TEST ${PARSE_UNPARSED_ARGUMENTS}) - IF (KOKKOS_HAS_TRILINOS) - IF(DEFINED PARSE_ARGS) - STRING(REPLACE ";" " " PARSE_ARGS "${PARSE_ARGS}") - ENDIF() - TRIBITS_ADD_EXECUTABLE_AND_TEST( - ${ROOT_NAME} - SOURCES ${PARSE_SOURCES} - TESTONLYLIBS ${KOKKOS_GTEST_LIB} - NUM_MPI_PROCS 1 - COMM serial mpi - ARGS ${PARSE_ARGS} - CATEGORIES ${PARSE_CATEGORIES} - SOURCES ${PARSE_SOURCES} - FAIL_REGULAR_EXPRESSION " FAILED " - ARGS ${PARSE_ARGS} - ) - ELSE() - KOKKOS_ADD_TEST_EXECUTABLE(${ROOT_NAME} - SOURCES ${PARSE_SOURCES} - ) - IF (PARSE_ARGS) - SET(TEST_NUMBER 0) - FOREACH (ARG_STR ${PARSE_ARGS}) - # This is passed as a single string blob to match TriBITS behavior - # We need this to be turned into a list - STRING(REPLACE " " ";" ARG_STR_LIST ${ARG_STR}) - LIST(APPEND TEST_NAME "${ROOT_NAME}${TEST_NUMBER}") - MATH(EXPR TEST_NUMBER "${TEST_NUMBER} + 1") - KOKKOS_ADD_TEST(NAME ${TEST_NAME} - EXE ${ROOT_NAME} - FAIL_REGULAR_EXPRESSION " FAILED " - ARGS ${ARG_STR_LIST} - ) - ENDFOREACH() - ELSE() - KOKKOS_ADD_TEST(NAME ${ROOT_NAME} - EXE ${ROOT_NAME} - FAIL_REGULAR_EXPRESSION " FAILED " - ) - ENDIF() - ENDIF() - # We noticed problems with -fvisibility=hidden for inline static variables - # if Kokkos was built as shared library. - IF(BUILD_SHARED_LIBS) - SET_PROPERTY(TARGET ${PACKAGE_NAME}_${ROOT_NAME} PROPERTY VISIBILITY_INLINES_HIDDEN ON) - SET_PROPERTY(TARGET ${PACKAGE_NAME}_${ROOT_NAME} PROPERTY CXX_VISIBILITY_PRESET hidden) - ENDIF() -ENDFUNCTION() + kokkos_add_test_executable(${ROOT_NAME} SOURCES ${PARSE_SOURCES}) + if(PARSE_ARGS) + set(TEST_NUMBER 0) + foreach(ARG_STR ${PARSE_ARGS}) + # This is passed as a single string blob to match TriBITS behavior + # We need this to be turned into a list + string(REPLACE " " ";" ARG_STR_LIST ${ARG_STR}) + list(APPEND TEST_NAME "${ROOT_NAME}${TEST_NUMBER}") + math(EXPR TEST_NUMBER "${TEST_NUMBER} + 1") + kokkos_add_test( + NAME + ${TEST_NAME} + EXE + ${ROOT_NAME} + FAIL_REGULAR_EXPRESSION + " FAILED " + ARGS + ${ARG_STR_LIST} + ) + endforeach() + else() + kokkos_add_test(NAME ${ROOT_NAME} EXE ${ROOT_NAME} FAIL_REGULAR_EXPRESSION " FAILED ") + endif() + # We noticed problems with -fvisibility=hidden for inline static variables + # if Kokkos was built as shared library. + if(BUILD_SHARED_LIBS AND NOT ${TEST_NAME}_DISABLE) + set_property(TARGET ${EXE_NAME} PROPERTY VISIBILITY_INLINES_HIDDEN ON) + set_property(TARGET ${EXE_NAME} PROPERTY CXX_VISIBILITY_PRESET hidden) + endif() + if(NOT + (Kokkos_INSTALL_TESTING + OR Kokkos_ENABLE_SYCL + OR Kokkos_ENABLE_HPX + OR Kokkos_ENABLE_IMPL_SKIP_NO_RTTI_FLAG + OR (KOKKOS_CXX_COMPILER_ID STREQUAL "Intel" AND KOKKOS_CXX_COMPILER_VERSION VERSION_LESS 2021.2.0) + OR (KOKKOS_CXX_COMPILER_ID STREQUAL "NVIDIA" AND KOKKOS_CXX_COMPILER_VERSION VERSION_LESS 11.3.0) + OR (KOKKOS_CXX_COMPILER_ID STREQUAL "NVIDIA" AND KOKKOS_CXX_HOST_COMPILER_ID STREQUAL "MSVC")) + ) + if(MSVC) + target_compile_options(${PACKAGE_NAME}_${ROOT_NAME} PRIVATE "/GR-") + else() + target_compile_options(${PACKAGE_NAME}_${ROOT_NAME} PRIVATE "-fno-rtti") + endif() + endif() +endfunction() -FUNCTION(KOKKOS_SET_EXE_PROPERTY ROOT_NAME) - SET(TARGET_NAME ${PACKAGE_NAME}_${ROOT_NAME}) - IF (NOT TARGET ${TARGET_NAME}) - MESSAGE(SEND_ERROR "No target ${TARGET_NAME} exists - cannot set target properties") - ENDIF() - SET_PROPERTY(TARGET ${TARGET_NAME} PROPERTY ${ARGN}) -ENDFUNCTION() +function(KOKKOS_SET_EXE_PROPERTY ROOT_NAME) + set(TARGET_NAME ${PACKAGE_NAME}_${ROOT_NAME}) + if(NOT TARGET ${TARGET_NAME}) + message(SEND_ERROR "No target ${TARGET_NAME} exists - cannot set target properties") + endif() + set_property(TARGET ${TARGET_NAME} PROPERTY ${ARGN}) +endfunction() -MACRO(KOKKOS_SETUP_BUILD_ENVIRONMENT) +macro(KOKKOS_SETUP_BUILD_ENVIRONMENT) # This is needed for both regular build and install tests - INCLUDE(${KOKKOS_SRC_PATH}/cmake/kokkos_compiler_id.cmake) + include(${KOKKOS_SRC_PATH}/cmake/kokkos_compiler_id.cmake) #set an internal option, if not already set - SET(Kokkos_INSTALL_TESTING OFF CACHE INTERNAL "Whether to build tests and examples against installation") - IF (Kokkos_INSTALL_TESTING) - SET(KOKKOS_ENABLE_TESTS ON) - SET(KOKKOS_ENABLE_BENCHMARKS ON) - SET(KOKKOS_ENABLE_EXAMPLES ON) + set(Kokkos_INSTALL_TESTING OFF CACHE INTERNAL "Whether to build tests and examples against installation") + if(Kokkos_INSTALL_TESTING) + set(KOKKOS_ENABLE_TESTS ON) + set(KOKKOS_ENABLE_BENCHMARKS ON) + set(KOKKOS_ENABLE_EXAMPLES ON) # This looks a little weird, but what we are doing # is to NOT build Kokkos but instead look for an # installed Kokkos - then build examples and tests # against that installed Kokkos - FIND_PACKAGE(Kokkos REQUIRED) + find_package(Kokkos REQUIRED) # Just grab the configuration from the installation - FOREACH(DEV ${Kokkos_DEVICES}) - SET(KOKKOS_ENABLE_${DEV} ON) - ENDFOREACH() - FOREACH(OPT ${Kokkos_OPTIONS}) - SET(KOKKOS_ENABLE_${OPT} ON) - ENDFOREACH() - FOREACH(TPL ${Kokkos_TPLS}) - SET(KOKKOS_ENABLE_${TPL} ON) - ENDFOREACH() - FOREACH(ARCH ${Kokkos_ARCH}) - SET(KOKKOS_ARCH_${ARCH} ON) - ENDFOREACH() - ELSE() - INCLUDE(${KOKKOS_SRC_PATH}/cmake/kokkos_enable_devices.cmake) - INCLUDE(${KOKKOS_SRC_PATH}/cmake/kokkos_enable_options.cmake) - INCLUDE(${KOKKOS_SRC_PATH}/cmake/kokkos_test_cxx_std.cmake) - INCLUDE(${KOKKOS_SRC_PATH}/cmake/kokkos_arch.cmake) - IF (NOT KOKKOS_HAS_TRILINOS) - SET(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${Kokkos_SOURCE_DIR}/cmake/Modules/") - ENDIF() - INCLUDE(${KOKKOS_SRC_PATH}/cmake/kokkos_tpls.cmake) - INCLUDE(${KOKKOS_SRC_PATH}/cmake/kokkos_corner_cases.cmake) - ENDIF() -ENDMACRO() - -MACRO(KOKKOS_ADD_TEST_EXECUTABLE ROOT_NAME) - CMAKE_PARSE_ARGUMENTS(PARSE - "" - "" - "SOURCES" - ${ARGN}) - KOKKOS_ADD_EXECUTABLE(${ROOT_NAME} - SOURCES ${PARSE_SOURCES} - ${PARSE_UNPARSED_ARGUMENTS} - TESTONLYLIBS ${KOKKOS_GTEST_LIB} - ) - SET(EXE_NAME ${PACKAGE_NAME}_${ROOT_NAME}) -ENDMACRO() - -MACRO(KOKKOS_PACKAGE_POSTPROCESS) - if (KOKKOS_HAS_TRILINOS) - TRIBITS_PACKAGE_POSTPROCESS() + foreach(DEV ${Kokkos_DEVICES}) + set(KOKKOS_ENABLE_${DEV} ON) + endforeach() + foreach(OPT ${Kokkos_OPTIONS}) + set(KOKKOS_ENABLE_${OPT} ON) + endforeach() + foreach(TPL ${Kokkos_TPLS}) + set(KOKKOS_ENABLE_${TPL} ON) + endforeach() + foreach(ARCH ${Kokkos_ARCH}) + set(KOKKOS_ARCH_${ARCH} ON) + endforeach() + else() + include(${KOKKOS_SRC_PATH}/cmake/kokkos_enable_devices.cmake) + include(${KOKKOS_SRC_PATH}/cmake/kokkos_enable_options.cmake) + include(${KOKKOS_SRC_PATH}/cmake/kokkos_test_cxx_std.cmake) + include(${KOKKOS_SRC_PATH}/cmake/kokkos_arch.cmake) + set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${Kokkos_SOURCE_DIR}/cmake/Modules/") + include(${KOKKOS_SRC_PATH}/cmake/kokkos_tpls.cmake) + include(${KOKKOS_SRC_PATH}/cmake/kokkos_corner_cases.cmake) endif() -ENDMACRO() +endmacro() + +macro(KOKKOS_ADD_TEST_EXECUTABLE ROOT_NAME) + cmake_parse_arguments(PARSE "" "" "SOURCES" ${ARGN}) + # Don't do anything if the user disabled the test + if(NOT ${PACKAGE_NAME}_${ROOT_NAME}_DISABLE) + kokkos_add_executable( + ${ROOT_NAME} SOURCES ${PARSE_SOURCES} ${PARSE_UNPARSED_ARGUMENTS} TESTONLYLIBS ${KOKKOS_GTEST_LIB} + ) + set(EXE_NAME ${PACKAGE_NAME}_${ROOT_NAME}) + endif() +endmacro() ## KOKKOS_CONFIGURE_CORE Configure/Generate header files for core content based ## on enabled backends. @@ -242,265 +181,214 @@ ENDMACRO() ## KOKKOS_SETUP is included in Kokkos_Macros.hpp and include prefix includes/defines ## KOKKOS_DECLARE is the declaration set ## KOKKOS_POST_INCLUDE is included at the end of Kokkos_Core.hpp -MACRO(KOKKOS_CONFIGURE_CORE) - MESSAGE(STATUS "Kokkos Backends: ${KOKKOS_ENABLED_DEVICES}") - KOKKOS_CONFIG_HEADER( KokkosCore_Config_HeaderSet.in KokkosCore_Config_FwdBackend.hpp "KOKKOS_FWD" "fwd/Kokkos_Fwd" "${KOKKOS_ENABLED_DEVICES}") - KOKKOS_CONFIG_HEADER( KokkosCore_Config_HeaderSet.in KokkosCore_Config_SetupBackend.hpp "KOKKOS_SETUP" "setup/Kokkos_Setup" "${DEVICE_SETUP_LIST}") - KOKKOS_CONFIG_HEADER( KokkosCore_Config_HeaderSet.in KokkosCore_Config_DeclareBackend.hpp "KOKKOS_DECLARE" "decl/Kokkos_Declare" "${KOKKOS_ENABLED_DEVICES}") - CONFIGURE_FILE(cmake/KokkosCore_config.h.in KokkosCore_config.h @ONLY) -ENDMACRO() +macro(KOKKOS_CONFIGURE_CORE) + message(STATUS "Kokkos Backends: ${KOKKOS_ENABLED_DEVICES}") + kokkos_config_header( + KokkosCore_Config_HeaderSet.in KokkosCore_Config_FwdBackend.hpp "KOKKOS_FWD" "fwd/Kokkos_Fwd" + "${KOKKOS_ENABLED_DEVICES}" + ) + kokkos_config_header( + KokkosCore_Config_HeaderSet.in KokkosCore_Config_SetupBackend.hpp "KOKKOS_SETUP" "setup/Kokkos_Setup" + "${DEVICE_SETUP_LIST}" + ) + kokkos_config_header( + KokkosCore_Config_HeaderSet.in KokkosCore_Config_DeclareBackend.hpp "KOKKOS_DECLARE" "decl/Kokkos_Declare" + "${KOKKOS_ENABLED_DEVICES}" + ) + configure_file(cmake/KokkosCore_config.h.in KokkosCore_config.h @ONLY) +endmacro() ## KOKKOS_INSTALL_ADDITIONAL_FILES - instruct cmake to install files in target destination. ## Includes generated header files, scripts such as nvcc_wrapper and hpcbind, ## as well as other files provided through plugins. -MACRO(KOKKOS_INSTALL_ADDITIONAL_FILES) +macro(KOKKOS_INSTALL_ADDITIONAL_FILES) # kokkos_launch_compiler is used by Kokkos to prefix compiler commands so that they forward to original kokkos compiler # if nvcc_wrapper was not used as CMAKE_CXX_COMPILER, configure the original compiler into kokkos_launch_compiler - IF(NOT "${CMAKE_CXX_COMPILER}" MATCHES "nvcc_wrapper") - SET(NVCC_WRAPPER_DEFAULT_COMPILER "${CMAKE_CXX_COMPILER}") - ELSE() - IF(NOT "$ENV{NVCC_WRAPPER_DEFAULT_COMPILER}" STREQUAL "") - SET(NVCC_WRAPPER_DEFAULT_COMPILER "$ENV{NVCC_WRAPPER_DEFAULT_COMPILER}") - ENDIF() - ENDIF() + if(NOT "${CMAKE_CXX_COMPILER}" MATCHES "nvcc_wrapper") + set(NVCC_WRAPPER_DEFAULT_COMPILER "${CMAKE_CXX_COMPILER}") + else() + if(NOT "$ENV{NVCC_WRAPPER_DEFAULT_COMPILER}" STREQUAL "") + set(NVCC_WRAPPER_DEFAULT_COMPILER "$ENV{NVCC_WRAPPER_DEFAULT_COMPILER}") + endif() + endif() - CONFIGURE_FILE(${CMAKE_CURRENT_SOURCE_DIR}/bin/kokkos_launch_compiler - ${PROJECT_BINARY_DIR}/temp/kokkos_launch_compiler - @ONLY) + configure_file( + ${CMAKE_CURRENT_SOURCE_DIR}/bin/kokkos_launch_compiler ${PROJECT_BINARY_DIR}/temp/kokkos_launch_compiler @ONLY + ) - INSTALL(PROGRAMS - "${CMAKE_CURRENT_SOURCE_DIR}/bin/nvcc_wrapper" - "${CMAKE_CURRENT_SOURCE_DIR}/bin/hpcbind" - "${PROJECT_BINARY_DIR}/temp/kokkos_launch_compiler" - DESTINATION ${CMAKE_INSTALL_BINDIR}) - INSTALL(FILES - "${CMAKE_CURRENT_BINARY_DIR}/KokkosCore_config.h" + install(PROGRAMS "${CMAKE_CURRENT_SOURCE_DIR}/bin/nvcc_wrapper" "${CMAKE_CURRENT_SOURCE_DIR}/bin/hpcbind" + "${PROJECT_BINARY_DIR}/temp/kokkos_launch_compiler" DESTINATION ${CMAKE_INSTALL_BINDIR} + ) + install( + FILES "${CMAKE_CURRENT_BINARY_DIR}/KokkosCore_config.h" "${CMAKE_CURRENT_BINARY_DIR}/KokkosCore_Config_FwdBackend.hpp" "${CMAKE_CURRENT_BINARY_DIR}/KokkosCore_Config_SetupBackend.hpp" "${CMAKE_CURRENT_BINARY_DIR}/KokkosCore_Config_DeclareBackend.hpp" - DESTINATION ${KOKKOS_HEADER_DIR}) -ENDMACRO() + DESTINATION ${KOKKOS_HEADER_DIR} + ) +endmacro() +function(KOKKOS_SET_LIBRARY_PROPERTIES LIBRARY_NAME) + cmake_parse_arguments(PARSE "PLAIN_STYLE" "" "" ${ARGN}) -FUNCTION(KOKKOS_SET_LIBRARY_PROPERTIES LIBRARY_NAME) - CMAKE_PARSE_ARGUMENTS(PARSE - "PLAIN_STYLE" - "" - "" - ${ARGN}) - - IF((NOT KOKKOS_ENABLE_COMPILE_AS_CMAKE_LANGUAGE) AND (${CMAKE_VERSION} VERSION_GREATER_EQUAL "3.18")) + if((NOT KOKKOS_ENABLE_COMPILE_AS_CMAKE_LANGUAGE) AND (${CMAKE_VERSION} VERSION_GREATER_EQUAL "3.18")) #I can use link options #check for CXX linkage using the simple 3.18 way - TARGET_LINK_OPTIONS( - ${LIBRARY_NAME} PUBLIC - $<$:${KOKKOS_LINK_OPTIONS}> - ) - ELSE() + target_link_options(${LIBRARY_NAME} PUBLIC $<$:${KOKKOS_LINK_OPTIONS}>) + else() #I can use link options #just assume CXX linkage - TARGET_LINK_OPTIONS( - ${LIBRARY_NAME} PUBLIC ${KOKKOS_LINK_OPTIONS} - ) - ENDIF() + target_link_options(${LIBRARY_NAME} PUBLIC ${KOKKOS_LINK_OPTIONS}) + endif() - TARGET_COMPILE_OPTIONS( - ${LIBRARY_NAME} PUBLIC - $<$:${KOKKOS_COMPILE_OPTIONS}> + target_compile_options( + ${LIBRARY_NAME} PUBLIC $<$:${KOKKOS_COMPILE_OPTIONS}> ) - TARGET_COMPILE_DEFINITIONS( - ${LIBRARY_NAME} PUBLIC - $<$:${KOKKOS_COMPILE_DEFINITIONS}> + target_compile_definitions( + ${LIBRARY_NAME} PUBLIC $<$:${KOKKOS_COMPILE_DEFINITIONS}> ) - TARGET_LINK_LIBRARIES( - ${LIBRARY_NAME} PUBLIC ${KOKKOS_LINK_LIBRARIES} - ) + target_link_libraries(${LIBRARY_NAME} PUBLIC ${KOKKOS_LINK_LIBRARIES}) - IF (KOKKOS_ENABLE_CUDA) - TARGET_COMPILE_OPTIONS( - ${LIBRARY_NAME} - PUBLIC $<$:${KOKKOS_CUDA_OPTIONS}> + if(KOKKOS_ENABLE_CUDA) + target_compile_options( + ${LIBRARY_NAME} PUBLIC $<$:${KOKKOS_CUDA_OPTIONS}> ) - SET(NODEDUP_CUDAFE_OPTIONS) - FOREACH(OPT ${KOKKOS_CUDAFE_OPTIONS}) - LIST(APPEND NODEDUP_CUDAFE_OPTIONS -Xcudafe ${OPT}) - ENDFOREACH() - TARGET_COMPILE_OPTIONS( - ${LIBRARY_NAME} - PUBLIC $<$:${NODEDUP_CUDAFE_OPTIONS}> + set(NODEDUP_CUDAFE_OPTIONS) + foreach(OPT ${KOKKOS_CUDAFE_OPTIONS}) + list(APPEND NODEDUP_CUDAFE_OPTIONS -Xcudafe ${OPT}) + endforeach() + target_compile_options( + ${LIBRARY_NAME} PUBLIC $<$:${NODEDUP_CUDAFE_OPTIONS}> ) - ENDIF() + endif() - IF (KOKKOS_ENABLE_HIP) - TARGET_COMPILE_OPTIONS( - ${LIBRARY_NAME} - PUBLIC $<$:${KOKKOS_AMDGPU_OPTIONS}> + if(KOKKOS_ENABLE_HIP) + target_compile_options( + ${LIBRARY_NAME} PUBLIC $<$:${KOKKOS_AMDGPU_OPTIONS}> ) - ENDIF() + endif() - LIST(LENGTH KOKKOS_XCOMPILER_OPTIONS XOPT_LENGTH) - IF (XOPT_LENGTH GREATER 1) - MESSAGE(FATAL_ERROR "CMake deduplication does not allow multiple -Xcompiler flags (${KOKKOS_XCOMPILER_OPTIONS}): will require Kokkos to upgrade to minimum 3.12") - ENDIF() - IF(KOKKOS_XCOMPILER_OPTIONS) - SET(NODEDUP_XCOMPILER_OPTIONS) - FOREACH(OPT ${KOKKOS_XCOMPILER_OPTIONS}) + list(LENGTH KOKKOS_XCOMPILER_OPTIONS XOPT_LENGTH) + if(XOPT_LENGTH GREATER 1) + message( + FATAL_ERROR + "CMake deduplication does not allow multiple -Xcompiler flags (${KOKKOS_XCOMPILER_OPTIONS}): will require Kokkos to upgrade to minimum 3.12" + ) + endif() + if(KOKKOS_XCOMPILER_OPTIONS) + set(NODEDUP_XCOMPILER_OPTIONS) + foreach(OPT ${KOKKOS_XCOMPILER_OPTIONS}) #I have to do this for now because we can't guarantee 3.12 support #I really should do this with the shell option - LIST(APPEND NODEDUP_XCOMPILER_OPTIONS -Xcompiler) - LIST(APPEND NODEDUP_XCOMPILER_OPTIONS ${OPT}) - ENDFOREACH() - TARGET_COMPILE_OPTIONS( - ${LIBRARY_NAME} - PUBLIC $<$:${NODEDUP_XCOMPILER_OPTIONS}> + list(APPEND NODEDUP_XCOMPILER_OPTIONS -Xcompiler) + list(APPEND NODEDUP_XCOMPILER_OPTIONS ${OPT}) + endforeach() + target_compile_options( + ${LIBRARY_NAME} PUBLIC $<$:${NODEDUP_XCOMPILER_OPTIONS}> ) - ENDIF() + endif() - IF (KOKKOS_CXX_STANDARD_FEATURE) + if(KOKKOS_CXX_STANDARD_FEATURE) #GREAT! I can do this the right way - TARGET_COMPILE_FEATURES(${LIBRARY_NAME} PUBLIC ${KOKKOS_CXX_STANDARD_FEATURE}) - IF (NOT KOKKOS_USE_CXX_EXTENSIONS) - SET_TARGET_PROPERTIES(${LIBRARY_NAME} PROPERTIES CXX_EXTENSIONS OFF) - ENDIF() - ELSE() + target_compile_features(${LIBRARY_NAME} PUBLIC ${KOKKOS_CXX_STANDARD_FEATURE}) + if(NOT KOKKOS_USE_CXX_EXTENSIONS) + set_target_properties(${LIBRARY_NAME} PROPERTIES CXX_EXTENSIONS OFF) + endif() + else() #OH, well, no choice but the wrong way - TARGET_COMPILE_OPTIONS(${LIBRARY_NAME} PUBLIC ${KOKKOS_CXX_STANDARD_FLAG}) - ENDIF() -ENDFUNCTION() + target_compile_options(${LIBRARY_NAME} PUBLIC ${KOKKOS_CXX_STANDARD_FLAG}) + endif() +endfunction() +function(KOKKOS_INTERNAL_ADD_LIBRARY LIBRARY_NAME) + cmake_parse_arguments(PARSE "STATIC;SHARED" "" "HEADERS;SOURCES" ${ARGN}) -FUNCTION(KOKKOS_INTERNAL_ADD_LIBRARY LIBRARY_NAME) - CMAKE_PARSE_ARGUMENTS(PARSE - "STATIC;SHARED" - "" - "HEADERS;SOURCES" - ${ARGN}) - - IF(PARSE_HEADERS) - LIST(REMOVE_DUPLICATES PARSE_HEADERS) - ENDIF() - IF(PARSE_SOURCES) - LIST(REMOVE_DUPLICATES PARSE_SOURCES) - ENDIF() - FOREACH(source ${PARSE_SOURCES}) + if(PARSE_HEADERS) + list(REMOVE_DUPLICATES PARSE_HEADERS) + endif() + if(PARSE_SOURCES) + list(REMOVE_DUPLICATES PARSE_SOURCES) + endif() + foreach(source ${PARSE_SOURCES}) set_source_files_properties(${source} PROPERTIES LANGUAGE ${KOKKOS_COMPILE_LANGUAGE}) - ENDFOREACH() + endforeach() - IF(PARSE_STATIC) - SET(LINK_TYPE STATIC) - ENDIF() + if(PARSE_STATIC) + set(LINK_TYPE STATIC) + endif() - IF(PARSE_SHARED) - SET(LINK_TYPE SHARED) - ENDIF() + if(PARSE_SHARED) + set(LINK_TYPE SHARED) + endif() # MSVC and other platforms want to have # the headers included as source files # for better dependency detection - ADD_LIBRARY( - ${LIBRARY_NAME} - ${LINK_TYPE} - ${PARSE_HEADERS} - ${PARSE_SOURCES} - ) + add_library(${LIBRARY_NAME} ${LINK_TYPE} ${PARSE_HEADERS} ${PARSE_SOURCES}) - IF(PARSE_SHARED OR BUILD_SHARED_LIBS) - SET_TARGET_PROPERTIES(${LIBRARY_NAME} PROPERTIES - VERSION ${Kokkos_VERSION} - SOVERSION ${Kokkos_VERSION_MAJOR}.${Kokkos_VERSION_MINOR} + if(PARSE_SHARED OR BUILD_SHARED_LIBS) + set_target_properties( + ${LIBRARY_NAME} PROPERTIES VERSION ${Kokkos_VERSION} SOVERSION ${Kokkos_VERSION_MAJOR}.${Kokkos_VERSION_MINOR} ) - ENDIF() + endif() - KOKKOS_INTERNAL_ADD_LIBRARY_INSTALL(${LIBRARY_NAME}) + kokkos_internal_add_library_install(${LIBRARY_NAME}) #In case we are building in-tree, add an alias name #that matches the install Kokkos:: name - ADD_LIBRARY(Kokkos::${LIBRARY_NAME} ALIAS ${LIBRARY_NAME}) -ENDFUNCTION() + add_library(Kokkos::${LIBRARY_NAME} ALIAS ${LIBRARY_NAME}) +endfunction() -FUNCTION(KOKKOS_ADD_LIBRARY LIBRARY_NAME) - CMAKE_PARSE_ARGUMENTS(PARSE - "ADD_BUILD_OPTIONS" - "" - "HEADERS" - ${ARGN} - ) - IF (KOKKOS_HAS_TRILINOS) - # We do not pass headers to trilinos. They would get installed - # to the default include folder, but we want headers installed - # preserving the directory structure, e.g. impl - # If headers got installed in both locations, it breaks some - # downstream packages - TRIBITS_ADD_LIBRARY(${LIBRARY_NAME} ${PARSE_UNPARSED_ARGUMENTS} - ADDED_LIB_TARGET_NAME_OUT ${LIBRARY_NAME}_TARGET_NAME ) - IF (PARSE_ADD_BUILD_OPTIONS) - KOKKOS_SET_LIBRARY_PROPERTIES(${${LIBRARY_NAME}_TARGET_NAME}) - ENDIF() - ELSE() - # Forward the headers, we want to know about all headers - # to make sure they appear correctly in IDEs - KOKKOS_INTERNAL_ADD_LIBRARY( - ${LIBRARY_NAME} ${PARSE_UNPARSED_ARGUMENTS} HEADERS ${PARSE_HEADERS}) - IF (PARSE_ADD_BUILD_OPTIONS) - KOKKOS_SET_LIBRARY_PROPERTIES(${LIBRARY_NAME}) - ENDIF() - ENDIF() -ENDFUNCTION() - - -FUNCTION(KOKKOS_ADD_INTERFACE_LIBRARY NAME) - IF (KOKKOS_HAS_TRILINOS) - TRIBITS_ADD_LIBRARY(${NAME} ${ARGN}) - ELSE() - ADD_LIBRARY(${NAME} INTERFACE) - KOKKOS_INTERNAL_ADD_LIBRARY_INSTALL(${NAME}) - ENDIF() -ENDFUNCTION() - - -FUNCTION(KOKKOS_LIB_INCLUDE_DIRECTORIES TARGET) - KOKKOS_LIB_TYPE(${TARGET} INCTYPE) - FOREACH(DIR ${ARGN}) - TARGET_INCLUDE_DIRECTORIES(${TARGET} ${INCTYPE} $) - ENDFOREACH() -ENDFUNCTION() - -FUNCTION(KOKKOS_LIB_COMPILE_OPTIONS TARGET) - KOKKOS_LIB_TYPE(${TARGET} INCTYPE) - KOKKOS_TARGET_COMPILE_OPTIONS(${${PROJECT_NAME}_LIBRARY_NAME_PREFIX}${TARGET} ${INCTYPE} ${ARGN}) -ENDFUNCTION() - -MACRO(KOKKOS_ADD_TEST_DIRECTORIES) - IF (KOKKOS_HAS_TRILINOS) - TRIBITS_ADD_TEST_DIRECTORIES(${ARGN}) - ELSE() - IF(KOKKOS_ENABLE_TESTS) - FOREACH(TEST_DIR ${ARGN}) - ADD_SUBDIRECTORY(${TEST_DIR}) - ENDFOREACH() - ENDIF() - ENDIF() -ENDMACRO() - -MACRO(KOKKOS_ADD_EXAMPLE_DIRECTORIES) - if (KOKKOS_HAS_TRILINOS) - TRIBITS_ADD_EXAMPLE_DIRECTORIES(${ARGN}) - else() - IF(KOKKOS_ENABLE_EXAMPLES) - FOREACH(EXAMPLE_DIR ${ARGN}) - ADD_SUBDIRECTORY(${EXAMPLE_DIR}) - ENDFOREACH() - ENDIF() +function(KOKKOS_ADD_LIBRARY LIBRARY_NAME) + cmake_parse_arguments(PARSE "ADD_BUILD_OPTIONS" "" "HEADERS" ${ARGN}) + # Forward the headers, we want to know about all headers + # to make sure they appear correctly in IDEs + kokkos_internal_add_library(${LIBRARY_NAME} ${PARSE_UNPARSED_ARGUMENTS} HEADERS ${PARSE_HEADERS}) + if(PARSE_ADD_BUILD_OPTIONS) + kokkos_set_library_properties(${LIBRARY_NAME}) endif() -ENDMACRO() +endfunction() -MACRO(KOKKOS_ADD_BENCHMARK_DIRECTORIES) - IF(KOKKOS_ENABLE_BENCHMARKS) - FOREACH(BENCHMARK_DIR ${ARGN}) - ADD_SUBDIRECTORY(${BENCHMARK_DIR}) - ENDFOREACH() - ENDIF() -ENDMACRO() +function(KOKKOS_ADD_INTERFACE_LIBRARY NAME) + add_library(${NAME} INTERFACE) + kokkos_internal_add_library_install(${NAME}) +endfunction() + +function(KOKKOS_LIB_INCLUDE_DIRECTORIES TARGET) + kokkos_lib_type(${TARGET} INCTYPE) + foreach(DIR ${ARGN}) + target_include_directories(${TARGET} ${INCTYPE} $) + endforeach() +endfunction() + +function(KOKKOS_LIB_COMPILE_OPTIONS TARGET) + kokkos_lib_type(${TARGET} INCTYPE) + target_compile_options(${${PROJECT_NAME}_LIBRARY_NAME_PREFIX}${TARGET} ${INCTYPE} ${ARGN}) +endfunction() + +macro(KOKKOS_ADD_TEST_DIRECTORIES) + if(KOKKOS_ENABLE_TESTS) + foreach(TEST_DIR ${ARGN}) + add_subdirectory(${TEST_DIR}) + endforeach() + endif() +endmacro() + +macro(KOKKOS_ADD_EXAMPLE_DIRECTORIES) + if(KOKKOS_ENABLE_EXAMPLES) + foreach(EXAMPLE_DIR ${ARGN}) + add_subdirectory(${EXAMPLE_DIR}) + endforeach() + endif() +endmacro() + +macro(KOKKOS_ADD_BENCHMARK_DIRECTORIES) + if(KOKKOS_ENABLE_BENCHMARKS) + foreach(BENCHMARK_DIR ${ARGN}) + add_subdirectory(${BENCHMARK_DIR}) + endforeach() + endif() +endmacro() diff --git a/lib/kokkos/cmake/msvc.cmake b/lib/kokkos/cmake/msvc.cmake index 85421bdbaa..1de13585c7 100644 --- a/lib/kokkos/cmake/msvc.cmake +++ b/lib/kokkos/cmake/msvc.cmake @@ -1,11 +1,9 @@ - -FUNCTION(kokkos_set_msvc_flags full_standard int_standard) - IF (CMAKE_CXX_EXTENSIONS) - SET(KOKKOS_CXX_STANDARD_FLAG "" PARENT_SCOPE) - SET(KOKKOS_CXX_INTERMEDIATE_STANDARD_FLAG "" PARENT_SCOPE) - ELSE() - SET(KOKKOS_CXX_STANDARD_FLAG "" PARENT_SCOPE) - SET(KOKKOS_CXX_INTERMEDIATE_STANDARD_FLAG "" PARENT_SCOPE) - ENDIF() -ENDFUNCTION() - +function(kokkos_set_msvc_flags full_standard int_standard) + if(CMAKE_CXX_EXTENSIONS) + set(KOKKOS_CXX_STANDARD_FLAG "" PARENT_SCOPE) + set(KOKKOS_CXX_INTERMEDIATE_STANDARD_FLAG "" PARENT_SCOPE) + else() + set(KOKKOS_CXX_STANDARD_FLAG "" PARENT_SCOPE) + set(KOKKOS_CXX_INTERMEDIATE_STANDARD_FLAG "" PARENT_SCOPE) + endif() +endfunction() diff --git a/lib/kokkos/cmake/pgi.cmake b/lib/kokkos/cmake/pgi.cmake index e98e849558..45f59dcd10 100644 --- a/lib/kokkos/cmake/pgi.cmake +++ b/lib/kokkos/cmake/pgi.cmake @@ -1,8 +1,6 @@ - function(kokkos_set_pgi_flags full_standard int_standard) - STRING(TOLOWER ${full_standard} FULL_LC_STANDARD) - STRING(TOLOWER ${int_standard} INT_LC_STANDARD) - SET(KOKKOS_CXX_STANDARD_FLAG "--c++${FULL_LC_STANDARD}" PARENT_SCOPE) - SET(KOKKOS_CXX_INTERMDIATE_STANDARD_FLAG "--c++${INT_LC_STANDARD}" PARENT_SCOPE) + string(TOLOWER ${full_standard} FULL_LC_STANDARD) + string(TOLOWER ${int_standard} INT_LC_STANDARD) + set(KOKKOS_CXX_STANDARD_FLAG "--c++${FULL_LC_STANDARD}" PARENT_SCOPE) + set(KOKKOS_CXX_INTERMDIATE_STANDARD_FLAG "--c++${INT_LC_STANDARD}" PARENT_SCOPE) endfunction() - diff --git a/lib/kokkos/cmake/tpls/FindTPLHWLOC.cmake b/lib/kokkos/cmake/tpls/FindTPLHWLOC.cmake index 4e05d22534..52d8368d04 100644 --- a/lib/kokkos/cmake/tpls/FindTPLHWLOC.cmake +++ b/lib/kokkos/cmake/tpls/FindTPLHWLOC.cmake @@ -15,7 +15,6 @@ # ************************************************************************ # @HEADER - #----------------------------------------------------------------------------- # Hardware locality detection and control library. # @@ -26,8 +25,4 @@ # Version: 1.3 # -KOKKOS_TPL_FIND_INCLUDE_DIRS_AND_LIBRARIES( HWLOC - REQUIRED_HEADERS hwloc.h - REQUIRED_LIBS_NAMES "hwloc" - ) - +kokkos_tpl_find_include_dirs_and_libraries(HWLOC REQUIRED_HEADERS hwloc.h REQUIRED_LIBS_NAMES "hwloc") diff --git a/lib/kokkos/cmake/tpls/FindTPLPthread.cmake b/lib/kokkos/cmake/tpls/FindTPLPthread.cmake index 3d5b03805d..f51bce5d64 100644 --- a/lib/kokkos/cmake/tpls/FindTPLPthread.cmake +++ b/lib/kokkos/cmake/tpls/FindTPLPthread.cmake @@ -15,29 +15,26 @@ # ************************************************************************ # @HEADER -SET(USE_THREADS FALSE) +set(USE_THREADS FALSE) -IF(NOT TPL_Pthread_INCLUDE_DIRS AND NOT TPL_Pthread_LIBRARY_DIRS AND NOT TPL_Pthread_LIBRARIES) +if(NOT TPL_Pthread_INCLUDE_DIRS AND NOT TPL_Pthread_LIBRARY_DIRS AND NOT TPL_Pthread_LIBRARIES) # Use CMake's Thread finder since it is a bit smarter in determining # whether pthreads is already built into the compiler and doesn't need # a library to link. - FIND_PACKAGE(Threads) + find_package(Threads) #If Threads found a copy of pthreads make sure it is one of the cases the tribits #tpl system cannot handle. - IF(Threads_FOUND AND CMAKE_USE_PTHREADS_INIT) - IF(CMAKE_THREAD_LIBS_INIT STREQUAL "" OR CMAKE_THREAD_LIBS_INIT STREQUAL "-pthread") - SET(USE_THREADS TRUE) - ENDIF() - ENDIF() -ENDIF() + if(Threads_FOUND AND CMAKE_USE_PTHREADS_INIT) + if(CMAKE_THREAD_LIBS_INIT STREQUAL "" OR CMAKE_THREAD_LIBS_INIT STREQUAL "-pthread") + set(USE_THREADS TRUE) + endif() + endif() +endif() -IF(USE_THREADS) - SET(TPL_Pthread_INCLUDE_DIRS "") - SET(TPL_Pthread_LIBRARIES "${CMAKE_THREAD_LIBS_INIT}") - SET(TPL_Pthread_LIBRARY_DIRS "") -ELSE() - KOKKOS_TPL_FIND_INCLUDE_DIRS_AND_LIBRARIES( Pthread - REQUIRED_HEADERS pthread.h - REQUIRED_LIBS_NAMES pthread - ) -ENDIF() +if(USE_THREADS) + set(TPL_Pthread_INCLUDE_DIRS "") + set(TPL_Pthread_LIBRARIES "${CMAKE_THREAD_LIBS_INIT}") + set(TPL_Pthread_LIBRARY_DIRS "") +else() + kokkos_tpl_find_include_dirs_and_libraries(Pthread REQUIRED_HEADERS pthread.h REQUIRED_LIBS_NAMES pthread) +endif() diff --git a/lib/kokkos/cmake/tpls/FindTPLquadmath.cmake b/lib/kokkos/cmake/tpls/FindTPLquadmath.cmake index 8560ec60f1..b449f45135 100644 --- a/lib/kokkos/cmake/tpls/FindTPLquadmath.cmake +++ b/lib/kokkos/cmake/tpls/FindTPLquadmath.cmake @@ -15,7 +15,4 @@ # ************************************************************************ # @HEADER -TRIBITS_TPL_FIND_INCLUDE_DIRS_AND_LIBRARIES( quadmath - REQUIRED_HEADERS quadmath.h - REQUIRED_LIBS_NAMES quadmath -) +tribits_tpl_find_include_dirs_and_libraries(quadmath REQUIRED_HEADERS quadmath.h REQUIRED_LIBS_NAMES quadmath) diff --git a/lib/kokkos/containers/CMakeLists.txt b/lib/kokkos/containers/CMakeLists.txt index 0857d7007b..8ee8bb41a2 100644 --- a/lib/kokkos/containers/CMakeLists.txt +++ b/lib/kokkos/containers/CMakeLists.txt @@ -1,9 +1,9 @@ -IF (NOT Kokkos_INSTALL_TESTING) - ADD_SUBDIRECTORY(src) -ENDIF() +if(NOT Kokkos_INSTALL_TESTING) + add_subdirectory(src) +endif() # FIXME_OPENACC: temporarily disabled due to unimplemented features -IF(NOT KOKKOS_ENABLE_OPENACC) -KOKKOS_ADD_TEST_DIRECTORIES(unit_tests) -KOKKOS_ADD_TEST_DIRECTORIES(performance_tests) -ENDIF() +if(NOT KOKKOS_ENABLE_OPENACC) + kokkos_add_test_directories(unit_tests) + kokkos_add_test_directories(performance_tests) +endif() diff --git a/lib/kokkos/containers/performance_tests/CMakeLists.txt b/lib/kokkos/containers/performance_tests/CMakeLists.txt index e325e45e85..8d4d605b08 100644 --- a/lib/kokkos/containers/performance_tests/CMakeLists.txt +++ b/lib/kokkos/containers/performance_tests/CMakeLists.txt @@ -1,7 +1,6 @@ - -KOKKOS_INCLUDE_DIRECTORIES(${CMAKE_CURRENT_BINARY_DIR}) -KOKKOS_INCLUDE_DIRECTORIES(REQUIRED_DURING_INSTALLATION_TESTING ${CMAKE_CURRENT_SOURCE_DIR}) -KOKKOS_INCLUDE_DIRECTORIES(${CMAKE_CURRENT_SOURCE_DIR}/../src ) +kokkos_include_directories(${CMAKE_CURRENT_BINARY_DIR}) +kokkos_include_directories(REQUIRED_DURING_INSTALLATION_TESTING ${CMAKE_CURRENT_SOURCE_DIR}) +kokkos_include_directories(${CMAKE_CURRENT_SOURCE_DIR}/../src) foreach(Tag Threads;OpenMP;Cuda;HPX;HIP) string(TOUPPER ${Tag} DEVICE) @@ -10,14 +9,8 @@ foreach(Tag Threads;OpenMP;Cuda;HPX;HIP) if(Kokkos_ENABLE_${DEVICE}) message(STATUS "Sources Test${Tag}.cpp") - set(SOURCES - TestMain.cpp - Test${Tag}.cpp - ) + set(SOURCES TestMain.cpp Test${Tag}.cpp) - KOKKOS_ADD_EXECUTABLE_AND_TEST( - ContainersPerformanceTest_${Tag} - SOURCES ${SOURCES} - ) + kokkos_add_executable_and_test(ContainersPerformanceTest_${Tag} SOURCES ${SOURCES}) endif() endforeach() diff --git a/lib/kokkos/containers/performance_tests/TestScatterView.hpp b/lib/kokkos/containers/performance_tests/TestScatterView.hpp index a74f833b9f..953b8bff6e 100644 --- a/lib/kokkos/containers/performance_tests/TestScatterView.hpp +++ b/lib/kokkos/containers/performance_tests/TestScatterView.hpp @@ -25,8 +25,8 @@ namespace Perf { template void test_scatter_view(int m, int n) { - Kokkos::View original_view("original_view", - n); + Kokkos::View original_view("original_view", + n); { auto scatter_view = Kokkos::Experimental::create_scatter_view< Kokkos::Experimental::ScatterSum, Duplication, Contribution>( @@ -40,8 +40,8 @@ void test_scatter_view(int m, int n) { { auto num_threads = unique_token.size(); std::cout << "num_threads " << num_threads << '\n'; - Kokkos::View - hand_coded_duplicate_view("hand_coded_duplicate", num_threads, n); + Kokkos::View hand_coded_duplicate_view( + "hand_coded_duplicate", num_threads, n); auto f2 = KOKKOS_LAMBDA(int i) { auto thread_id = unique_token.acquire(); for (int j = 0; j < 10; ++j) { diff --git a/lib/kokkos/containers/src/CMakeLists.txt b/lib/kokkos/containers/src/CMakeLists.txt index b7d85ebf11..b386fbe675 100644 --- a/lib/kokkos/containers/src/CMakeLists.txt +++ b/lib/kokkos/containers/src/CMakeLists.txt @@ -1,33 +1,27 @@ #need these here for now -KOKKOS_INCLUDE_DIRECTORIES(${CMAKE_CURRENT_BINARY_DIR}) -KOKKOS_INCLUDE_DIRECTORIES(${CMAKE_CURRENT_SOURCE_DIR}) +kokkos_include_directories(${CMAKE_CURRENT_BINARY_DIR}) +kokkos_include_directories(${CMAKE_CURRENT_SOURCE_DIR}) #----------------------------------------------------------------------------- -SET(KOKKOS_CONTAINERS_SRCS) -APPEND_GLOB(KOKKOS_CONTAINERS_SRCS ${CMAKE_CURRENT_SOURCE_DIR}/impl/*.cpp) -SET(KOKKOS_CONTAINER_HEADERS) -APPEND_GLOB(KOKKOS_CONTAINERS_HEADERS ${CMAKE_CURRENT_SOURCE_DIR}/impl/*.hpp) -APPEND_GLOB(KOKKOS_CONTAINERS_HEADERS ${CMAKE_CURRENT_SOURCE_DIR}/*.hpp) +set(KOKKOS_CONTAINERS_SRCS) +append_glob(KOKKOS_CONTAINERS_SRCS ${CMAKE_CURRENT_SOURCE_DIR}/impl/*.cpp) +set(KOKKOS_CONTAINER_HEADERS) +append_glob(KOKKOS_CONTAINERS_HEADERS ${CMAKE_CURRENT_SOURCE_DIR}/impl/*.hpp) +append_glob(KOKKOS_CONTAINERS_HEADERS ${CMAKE_CURRENT_SOURCE_DIR}/*.hpp) - -INSTALL ( +install( DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/" DESTINATION ${KOKKOS_HEADER_DIR} - FILES_MATCHING PATTERN "*.hpp" + FILES_MATCHING + PATTERN "*.hpp" ) -KOKKOS_ADD_LIBRARY( - kokkoscontainers - SOURCES ${KOKKOS_CONTAINERS_SRCS} - HEADERS ${KOKKOS_CONTAINERS_HEADERS} -) +kokkos_add_library(kokkoscontainers SOURCES ${KOKKOS_CONTAINERS_SRCS} HEADERS ${KOKKOS_CONTAINERS_HEADERS}) -KOKKOS_LIB_INCLUDE_DIRECTORIES(kokkoscontainers - ${KOKKOS_TOP_BUILD_DIR} - ${CMAKE_CURRENT_BINARY_DIR} - ${CMAKE_CURRENT_SOURCE_DIR} +kokkos_lib_include_directories( + kokkoscontainers ${KOKKOS_TOP_BUILD_DIR} ${CMAKE_CURRENT_BINARY_DIR} ${CMAKE_CURRENT_SOURCE_DIR} ) -KOKKOS_LINK_INTERNAL_LIBRARY(kokkoscontainers kokkoscore) +kokkos_link_internal_library(kokkoscontainers kokkoscore) #----------------------------------------------------------------------------- diff --git a/lib/kokkos/containers/src/Kokkos_Bitset.hpp b/lib/kokkos/containers/src/Kokkos_Bitset.hpp index f50ab0a0f7..409260f021 100644 --- a/lib/kokkos/containers/src/Kokkos_Bitset.hpp +++ b/lib/kokkos/containers/src/Kokkos_Bitset.hpp @@ -271,7 +271,7 @@ class Bitset { offset = !(scan_direction & BIT_SCAN_REVERSE) ? offset : (offset + block_mask) & block_mask; - block = Impl::rotate_right(block, offset); + block = Impl::rotate_right(block, offset); return (((!(scan_direction & BIT_SCAN_REVERSE) ? Impl::bit_scan_forward(block) : Impl::int_log2(block)) + diff --git a/lib/kokkos/containers/src/Kokkos_DualView.hpp b/lib/kokkos/containers/src/Kokkos_DualView.hpp index a37a2bdceb..6a2e6f73a1 100644 --- a/lib/kokkos/containers/src/Kokkos_DualView.hpp +++ b/lib/kokkos/containers/src/Kokkos_DualView.hpp @@ -275,14 +275,29 @@ class DualView : public ViewTraits { const size_t n5 = KOKKOS_IMPL_CTOR_DEFAULT_ARG, const size_t n6 = KOKKOS_IMPL_CTOR_DEFAULT_ARG, const size_t n7 = KOKKOS_IMPL_CTOR_DEFAULT_ARG) - : modified_flags(t_modified_flags("DualView::modified_flags")), - d_view(arg_prop, n0, n1, n2, n3, n4, n5, n6, n7) { - // without UVM, host View mirrors - if constexpr (Kokkos::Impl::has_type::value) - h_view = Kokkos::create_mirror_view(Kokkos::WithoutInitializing, d_view); - else - h_view = Kokkos::create_mirror_view(d_view); + : modified_flags(t_modified_flags("DualView::modified_flags")) { + if constexpr (Impl::ViewCtorProp::sequential_host_init) { + h_view = t_host(arg_prop, n0, n1, n2, n3, n4, n5, n6, n7); + static_assert(Impl::ViewCtorProp::initialize, + "DualView: SequentialHostInit isn't compatible with " + "WithoutInitializing!"); + static_assert(!Impl::ViewCtorProp::has_execution_space, + "DualView: SequentialHostInit isn't compatible with " + "providing an execution space instance!"); + + d_view = Kokkos::create_mirror_view_and_copy( + typename traits::memory_space{}, h_view); + } else { + d_view = t_dev(arg_prop, n0, n1, n2, n3, n4, n5, n6, n7); + + // without UVM, host View mirrors + if constexpr (Kokkos::Impl::has_type::value) + h_view = + Kokkos::create_mirror_view(Kokkos::WithoutInitializing, d_view); + else + h_view = Kokkos::create_mirror_view(d_view); + } } //! Copy constructor (shallow copy) @@ -338,23 +353,21 @@ class DualView : public ViewTraits { // does the DualView have only one device struct impl_dualview_is_single_device { enum : bool { - value = std::is_same::value + value = std::is_same_v }; }; // does the given device match the device of t_dev? template struct impl_device_matches_tdev_device { - enum : bool { - value = std::is_same::value - }; + enum : bool { value = std::is_same_v }; }; // does the given device match the device of t_host? template struct impl_device_matches_thost_device { enum : bool { - value = std::is_same::value + value = std::is_same_v }; }; @@ -362,7 +375,7 @@ class DualView : public ViewTraits { template struct impl_device_matches_thost_exec { enum : bool { - value = std::is_same::value + value = std::is_same_v }; }; @@ -370,7 +383,7 @@ class DualView : public ViewTraits { template struct impl_device_matches_tdev_exec { enum : bool { - value = std::is_same::value + value = std::is_same_v }; }; @@ -378,8 +391,8 @@ class DualView : public ViewTraits { template struct impl_device_matches_tdev_memory_space { enum : bool { - value = std::is_same::value + value = std::is_same_v }; }; @@ -389,11 +402,6 @@ class DualView : public ViewTraits { /// \brief Return a View on a specific device \c Device. /// - /// Please don't be afraid of the nested if_c expressions in the return - /// value's type. That just tells the method what the return type - /// should be: t_dev if the \c Device template parameter matches - /// this DualView's device type, else t_host. - /// /// For example, suppose you create a DualView on Cuda, like this: /// \code /// using dual_view_type = @@ -410,56 +418,47 @@ class DualView : public ViewTraits { /// typename dual_view_type::t_host hostView = DV.view (); /// \endcode template - KOKKOS_INLINE_FUNCTION const typename std::conditional_t< - impl_device_matches_tdev_device::value, t_dev, - typename std::conditional_t< - impl_device_matches_thost_device::value, t_host, - typename std::conditional_t< - impl_device_matches_thost_exec::value, t_host, - typename std::conditional_t< - impl_device_matches_tdev_exec::value, t_dev, - typename std::conditional_t< - impl_device_matches_tdev_memory_space::value, - t_dev, t_host>>>>> - view() const { - constexpr bool device_is_memspace = - std::is_same::value; - constexpr bool device_is_execspace = - std::is_same::value; - constexpr bool device_exec_is_t_dev_exec = - std::is_same::value; - constexpr bool device_mem_is_t_dev_mem = - std::is_same::value; - constexpr bool device_exec_is_t_host_exec = - std::is_same::value; - constexpr bool device_mem_is_t_host_mem = - std::is_same::value; - constexpr bool device_is_t_host_device = - std::is_same::value; - constexpr bool device_is_t_dev_device = - std::is_same::value; - - static_assert( - device_is_t_dev_device || device_is_t_host_device || - (device_is_memspace && - (device_mem_is_t_dev_mem || device_mem_is_t_host_mem)) || - (device_is_execspace && - (device_exec_is_t_dev_exec || device_exec_is_t_host_exec)) || - ((!device_is_execspace && !device_is_memspace) && - ((device_mem_is_t_dev_mem || device_mem_is_t_host_mem) || - (device_exec_is_t_dev_exec || device_exec_is_t_host_exec))), - "Template parameter to .view() must exactly match one of the " - "DualView's device types or one of the execution or memory spaces"); - - return Impl::if_c::value, - t_dev, t_host>::select(d_view, h_view); + KOKKOS_FUNCTION auto view() const { + if constexpr (std::is_same_v) { + if constexpr (std::is_same_v) { + return d_view; + } else { + static_assert(std::is_same_v, + "The template argument is a memory space but doesn't " + "match either of DualView's memory spaces!"); + return h_view; + } + } else { + if constexpr (std::is_same_v) { + if constexpr (std::is_same_v) { + return d_view; + } else { + static_assert(std::is_same_v, + "The template argument is an execution space but " + "doesn't match either of DualView's execution spaces!"); + return h_view; + } + } else { + static_assert(std::is_same_v, + "The template argument is neither a memory space, " + "execution space, or device!"); + if constexpr (std::is_same_v) + return d_view; + else { + static_assert(std::is_same_v, + "The template argument is a device but " + "doesn't match either of DualView's devices!"); + return h_view; + } + } + } +#ifdef KOKKOS_COMPILER_INTEL + __builtin_unreachable(); +#endif } KOKKOS_INLINE_FUNCTION @@ -475,27 +474,27 @@ class DualView : public ViewTraits { template static int get_device_side() { constexpr bool device_is_memspace = - std::is_same::value; + std::is_same_v; constexpr bool device_is_execspace = - std::is_same::value; + std::is_same_v; constexpr bool device_exec_is_t_dev_exec = - std::is_same::value; + std::is_same_v; constexpr bool device_mem_is_t_dev_mem = - std::is_same::value; + std::is_same_v; constexpr bool device_exec_is_t_host_exec = - std::is_same::value; + std::is_same_v; constexpr bool device_mem_is_t_host_mem = - std::is_same::value; + std::is_same_v; constexpr bool device_is_t_host_device = - std::is_same::value; + std::is_same_v; constexpr bool device_is_t_dev_device = - std::is_same::value; + std::is_same_v; static_assert( device_is_t_dev_device || device_is_t_host_device || @@ -627,9 +626,9 @@ class DualView : public ViewTraits { template void sync(const std::enable_if_t< - (std::is_same::value) || - (std::is_same::value), + (std::is_same_v) || + (std::is_same_v), int>& = 0) { sync_impl(std::true_type{}); } @@ -637,9 +636,9 @@ class DualView : public ViewTraits { template void sync(const ExecutionSpace& exec, const std::enable_if_t< - (std::is_same::value) || - (std::is_same::value), + (std::is_same_v) || + (std::is_same_v), int>& = 0) { sync_impl(std::true_type{}, exec); } @@ -669,18 +668,18 @@ class DualView : public ViewTraits { template void sync(const std::enable_if_t< - (!std::is_same::value) || - (std::is_same::value), + (!std::is_same_v) || + (std::is_same_v), int>& = 0) { sync_impl(std::false_type{}); } template void sync(const ExecutionSpace& exec, const std::enable_if_t< - (!std::is_same::value) || - (std::is_same::value), + (!std::is_same_v) || + (std::is_same_v), int>& = 0) { sync_impl(std::false_type{}, exec); } @@ -943,12 +942,21 @@ class DualView : public ViewTraits { Impl::size_mismatch(h_view, h_view.rank_dynamic, new_extents); if (sizeMismatch) { - ::Kokkos::realloc(arg_prop, d_view, n0, n1, n2, n3, n4, n5, n6, n7); - if constexpr (alloc_prop_input::initialize) { - h_view = create_mirror_view(typename t_host::memory_space(), d_view); + if constexpr (alloc_prop_input::sequential_host_init) { + static_assert(alloc_prop_input::initialize, + "DualView: SequentialHostInit isn't compatible with " + "WithoutInitializing!"); + ::Kokkos::realloc(arg_prop, h_view, n0, n1, n2, n3, n4, n5, n6, n7); + d_view = + create_mirror_view_and_copy(typename t_dev::memory_space(), h_view); } else { - h_view = create_mirror_view(Kokkos::WithoutInitializing, - typename t_host::memory_space(), d_view); + ::Kokkos::realloc(arg_prop, d_view, n0, n1, n2, n3, n4, n5, n6, n7); + if constexpr (alloc_prop_input::initialize) { + h_view = create_mirror_view(typename t_host::memory_space(), d_view); + } else { + h_view = create_mirror_view(Kokkos::WithoutInitializing, + typename t_host::memory_space(), d_view); + } } } else if constexpr (alloc_prop_input::initialize) { if constexpr (alloc_prop_input::has_execution_space) { @@ -1062,9 +1070,22 @@ class DualView : public ViewTraits { } }; - constexpr bool has_execution_space = alloc_prop_input::has_execution_space; + if constexpr (alloc_prop_input::sequential_host_init) { + static_assert(alloc_prop_input::initialize, + "DualView: SequentialHostInit isn't compatible with " + "WithoutInitializing!"); + static_assert(!alloc_prop_input::has_execution_space, + "DualView: SequentialHostInit isn't compatible with " + "providing an execution space instance!"); - if constexpr (has_execution_space) { + if (sizeMismatch) { + sync(); + ::Kokkos::resize(arg_prop, h_view, n0, n1, n2, n3, n4, n5, n6, n7); + d_view = + create_mirror_view_and_copy(typename t_dev::memory_space(), h_view); + } + return; + } else if constexpr (alloc_prop_input::has_execution_space) { using ExecSpace = typename alloc_prop_input::execution_space; const auto& exec_space = Impl::get_property(arg_prop); @@ -1182,15 +1203,15 @@ class DualView : public ViewTraits { } template - KOKKOS_INLINE_FUNCTION constexpr std::enable_if_t< - std::is_integral::value, size_t> + KOKKOS_INLINE_FUNCTION constexpr std::enable_if_t, + size_t> extent(const iType& r) const { return d_view.extent(r); } template - KOKKOS_INLINE_FUNCTION constexpr std::enable_if_t< - std::is_integral::value, int> + KOKKOS_INLINE_FUNCTION constexpr std::enable_if_t, + int> extent_int(const iType& r) const { return static_cast(d_view.extent(r)); } diff --git a/lib/kokkos/containers/src/Kokkos_DynRankView.hpp b/lib/kokkos/containers/src/Kokkos_DynRankView.hpp index 5f7fcaf69e..b860359526 100644 --- a/lib/kokkos/containers/src/Kokkos_DynRankView.hpp +++ b/lib/kokkos/containers/src/Kokkos_DynRankView.hpp @@ -38,6 +38,23 @@ class DynRankView; // forward declare namespace Impl { +template +struct ViewDataTypeFromRank { + using type = typename ViewDataTypeFromRank::type*; +}; + +template +struct ViewDataTypeFromRank { + using type = T; +}; + +template +KOKKOS_FUNCTION View::type, Args...> +as_view_of_rank_n( + DynRankView v, + std::enable_if_t::specialize, + void>>* = nullptr); + template struct DynRankDimTraits { enum : size_t { unspecified = KOKKOS_INVALID_INDEX }; @@ -91,54 +108,59 @@ struct DynRankDimTraits { } // Create the layout for the rank-7 view. + // Because the underlying View is rank-7, preserve "unspecified" for + // dimension 8. + // Non-strided Layout template KOKKOS_INLINE_FUNCTION static std::enable_if_t< - (std::is_same::value || - std::is_same::value), + (std::is_same_v || + std::is_same_v), Layout> createLayout(const Layout& layout) { - return Layout(layout.dimension[0] != unspecified ? layout.dimension[0] : 1, - layout.dimension[1] != unspecified ? layout.dimension[1] : 1, - layout.dimension[2] != unspecified ? layout.dimension[2] : 1, - layout.dimension[3] != unspecified ? layout.dimension[3] : 1, - layout.dimension[4] != unspecified ? layout.dimension[4] : 1, - layout.dimension[5] != unspecified ? layout.dimension[5] : 1, - layout.dimension[6] != unspecified ? layout.dimension[6] : 1, - layout.dimension[7] != unspecified ? layout.dimension[7] : 1); + Layout new_layout( + layout.dimension[0] != unspecified ? layout.dimension[0] : 1, + layout.dimension[1] != unspecified ? layout.dimension[1] : 1, + layout.dimension[2] != unspecified ? layout.dimension[2] : 1, + layout.dimension[3] != unspecified ? layout.dimension[3] : 1, + layout.dimension[4] != unspecified ? layout.dimension[4] : 1, + layout.dimension[5] != unspecified ? layout.dimension[5] : 1, + layout.dimension[6] != unspecified ? layout.dimension[6] : 1, + layout.dimension[7] != unspecified ? layout.dimension[7] : unspecified); + new_layout.stride = layout.stride; + return new_layout; } // LayoutStride template KOKKOS_INLINE_FUNCTION static std::enable_if_t< - (std::is_same::value), Layout> + (std::is_same_v), Layout> createLayout(const Layout& layout) { - return Layout(layout.dimension[0] != unspecified ? layout.dimension[0] : 1, - layout.stride[0], - layout.dimension[1] != unspecified ? layout.dimension[1] : 1, - layout.stride[1], - layout.dimension[2] != unspecified ? layout.dimension[2] : 1, - layout.stride[2], - layout.dimension[3] != unspecified ? layout.dimension[3] : 1, - layout.stride[3], - layout.dimension[4] != unspecified ? layout.dimension[4] : 1, - layout.stride[4], - layout.dimension[5] != unspecified ? layout.dimension[5] : 1, - layout.stride[5], - layout.dimension[6] != unspecified ? layout.dimension[6] : 1, - layout.stride[6], - layout.dimension[7] != unspecified ? layout.dimension[7] : 1, - layout.stride[7]); + return Layout( + layout.dimension[0] != unspecified ? layout.dimension[0] : 1, + layout.stride[0], + layout.dimension[1] != unspecified ? layout.dimension[1] : 1, + layout.stride[1], + layout.dimension[2] != unspecified ? layout.dimension[2] : 1, + layout.stride[2], + layout.dimension[3] != unspecified ? layout.dimension[3] : 1, + layout.stride[3], + layout.dimension[4] != unspecified ? layout.dimension[4] : 1, + layout.stride[4], + layout.dimension[5] != unspecified ? layout.dimension[5] : 1, + layout.stride[5], + layout.dimension[6] != unspecified ? layout.dimension[6] : 1, + layout.stride[6], + layout.dimension[7] != unspecified ? layout.dimension[7] : unspecified, + layout.stride[7]); } // Extra overload to match that for specialize types template KOKKOS_INLINE_FUNCTION static std::enable_if_t< - (std::is_same::value || - std::is_same::value || - std::is_same::value), + (std::is_same_v || + std::is_same_v || + std::is_same_v), typename Traits::array_layout> createLayout(const Kokkos::Impl::ViewCtorProp& /* prop */, const typename Traits::array_layout& layout) { @@ -164,9 +186,8 @@ struct DynRankDimTraits { // Non-strided Layout template KOKKOS_INLINE_FUNCTION static std::enable_if_t< - (std::is_same::value || - std::is_same::value) && - std::is_integral::value, + (std::is_same_v || + std::is_same_v)&&std::is_integral_v, Layout> reconstructLayout(const Layout& layout, iType dynrank) { return Layout(dynrank > 0 ? layout.dimension[0] : KOKKOS_INVALID_INDEX, @@ -182,8 +203,7 @@ reconstructLayout(const Layout& layout, iType dynrank) { // LayoutStride template KOKKOS_INLINE_FUNCTION static std::enable_if_t< - (std::is_same::value) && - std::is_integral::value, + (std::is_same_v)&&std::is_integral_v, Layout> reconstructLayout(const Layout& layout, iType dynrank) { return Layout(dynrank > 0 ? layout.dimension[0] : KOKKOS_INVALID_INDEX, @@ -284,40 +304,43 @@ namespace Impl { template class ViewMapping< DstTraits, SrcTraits, - std::enable_if_t<(std::is_same::value && - std::is_void::value && - std::is_void::value && - (std::is_same::value || - ((std::is_same::value || - std::is_same::value || - std::is_same::value) && - (std::is_same::value || - std::is_same::value || - std::is_same::value)))), - Kokkos::Impl::ViewToDynRankViewTag>> { + std::enable_if_t< + (std::is_same_v && + std::is_void_v && + std::is_void_v && + (std::is_same_v || + ((std::is_same_v || + std::is_same_v || + std::is_same_v< + typename DstTraits::array_layout, + Kokkos::LayoutStride>)&&(std::is_same_v || + std::is_same_v< + typename SrcTraits::array_layout, + Kokkos::LayoutRight> || + std::is_same_v< + typename SrcTraits::array_layout, + Kokkos::LayoutStride>)))), + Kokkos::Impl::ViewToDynRankViewTag>> { private: enum { is_assignable_value_type = - std::is_same::value || - std::is_same::value + std::is_same_v || + std::is_same_v }; enum { is_assignable_layout = - std::is_same::value || - std::is_same::value + std::is_same_v || + std::is_same_v }; public: @@ -345,7 +368,7 @@ class ViewMapping< src.layout()); // Check this for integer input1 for padding, etc dst.m_map.m_impl_handle = Kokkos::Impl::ViewDataHandle::assign( src.m_map.m_impl_handle, src.m_track.m_tracker); - dst.m_track.assign(src.m_track.m_tracker, DstTraits::is_managed); + dst.m_track.m_tracker.assign(src.m_track.m_tracker, DstTraits::is_managed); dst.m_rank = Kokkos::View::rank(); } }; @@ -378,10 +401,11 @@ struct is_dyn_rank_view> : public std::true_type { template inline constexpr bool is_dyn_rank_view_v = is_dyn_rank_view::value; +// Inherit privately from View, this way we don't import anything funky +// for example the rank member vs the rank() function of DynRankView template -class DynRankView : public ViewTraits { - static_assert(!std::is_array::value && - !std::is_pointer::value, +class DynRankView : private View { + static_assert(!std::is_array_v && !std::is_pointer_v, "Cannot template DynRankView with array or pointer datatype - " "must be pod"); @@ -391,28 +415,66 @@ class DynRankView : public ViewTraits { template friend class Kokkos::Impl::ViewMapping; + size_t m_rank{}; + public: using drvtraits = ViewTraits; using view_type = View; - using traits = ViewTraits; - private: - using map_type = - Kokkos::Impl::ViewMapping; - using track_type = Kokkos::Impl::SharedAllocationTracker; - - track_type m_track; - map_type m_map; - unsigned m_rank; + using drdtraits = Impl::DynRankDimTraits; public: - KOKKOS_INLINE_FUNCTION + // typedefs from ViewTraits, overriden + using data_type = typename drvtraits::data_type; + using const_data_type = typename drvtraits::const_data_type; + using non_const_data_type = typename drvtraits::non_const_data_type; + + // typedefs from ViewTraits not overriden + using value_type = typename view_type::value_type; + using const_value_type = typename view_type::const_value_type; + using non_const_value_type = typename view_type::non_const_value_type; + using traits = typename view_type::traits; + using array_layout = typename view_type::array_layout; + + using execution_space = typename view_type::execution_space; + using memory_space = typename view_type::memory_space; + using device_type = typename view_type::device_type; + + using memory_traits = typename view_type::memory_traits; + using host_mirror_space = typename view_type::host_mirror_space; + using size_type = typename view_type::size_type; + + using reference_type = typename view_type::reference_type; + using pointer_type = typename view_type::pointer_type; + + using scalar_array_type = value_type; + using const_scalar_array_type = const_value_type; + using non_const_scalar_array_type = non_const_value_type; + using specialize = typename view_type::specialize; + + // typedefs in View for mdspan compatibility + // cause issues with MSVC+CUDA + // using layout_type = typename view_type::layout_type; + using index_type = typename view_type::index_type; + using element_type = typename view_type::element_type; + using rank_type = typename view_type::rank_type; + using reference = reference_type; + using data_handle_type = pointer_type; + + KOKKOS_FUNCTION view_type& DownCast() const { return (view_type&)(*this); } - KOKKOS_INLINE_FUNCTION + + // FIXME: this function make NO sense, the above one already is marked const + // Maybe one would want to get back a view of const?? + KOKKOS_FUNCTION const view_type& ConstDownCast() const { return (const view_type&)(*this); } + // FIXME: deprecate DownCast in favor of to_view + // KOKKOS_FUNCTION + // view_type to_view() const { return *this; } + // Types below - at least the HostMirror requires the value_type, NOT the rank // 7 data_type of the traits @@ -436,114 +498,36 @@ class DynRankView : public ViewTraits { typename drvtraits::array_layout, typename drvtraits::host_mirror_space>; + using host_mirror_type = HostMirror; //---------------------------------------- // Domain rank and extents // enum { Rank = map_type::Rank }; //Will be dyn rank of 7 always, keep the // enum? - template - KOKKOS_INLINE_FUNCTION constexpr std::enable_if_t< - std::is_integral::value, size_t> - extent(const iType& r) const { - return m_map.extent(r); - } - - template - KOKKOS_INLINE_FUNCTION constexpr std::enable_if_t< - std::is_integral::value, int> - extent_int(const iType& r) const { - return static_cast(m_map.extent(r)); - } - - KOKKOS_INLINE_FUNCTION constexpr typename traits::array_layout layout() const; - //---------------------------------------- /* Deprecate all 'dimension' functions in favor of * ISO/C++ vocabulary 'extent'. */ - KOKKOS_INLINE_FUNCTION constexpr size_t size() const { - return m_map.extent(0) * m_map.extent(1) * m_map.extent(2) * - m_map.extent(3) * m_map.extent(4) * m_map.extent(5) * - m_map.extent(6) * m_map.extent(7); - } - - KOKKOS_INLINE_FUNCTION constexpr size_t stride_0() const { - return m_map.stride_0(); - } - KOKKOS_INLINE_FUNCTION constexpr size_t stride_1() const { - return m_map.stride_1(); - } - KOKKOS_INLINE_FUNCTION constexpr size_t stride_2() const { - return m_map.stride_2(); - } - KOKKOS_INLINE_FUNCTION constexpr size_t stride_3() const { - return m_map.stride_3(); - } - KOKKOS_INLINE_FUNCTION constexpr size_t stride_4() const { - return m_map.stride_4(); - } - KOKKOS_INLINE_FUNCTION constexpr size_t stride_5() const { - return m_map.stride_5(); - } - KOKKOS_INLINE_FUNCTION constexpr size_t stride_6() const { - return m_map.stride_6(); - } - KOKKOS_INLINE_FUNCTION constexpr size_t stride_7() const { - return m_map.stride_7(); - } - - template - KOKKOS_INLINE_FUNCTION void stride(iType* const s) const { - m_map.stride(s); - } - - //---------------------------------------- - // Range span is the span which contains all members. - - using reference_type = typename map_type::reference_type; - using pointer_type = typename map_type::pointer_type; - - enum { - reference_type_is_lvalue_reference = - std::is_lvalue_reference::value - }; - - KOKKOS_INLINE_FUNCTION constexpr size_t span() const { return m_map.span(); } - KOKKOS_INLINE_FUNCTION constexpr bool span_is_contiguous() const { - return m_map.span_is_contiguous(); - } - KOKKOS_INLINE_FUNCTION constexpr pointer_type data() const { - return m_map.data(); - } - KOKKOS_INLINE_FUNCTION constexpr bool is_allocated() const { - return (m_map.data() != nullptr); - } - - //---------------------------------------- - // Allow specializations to query their specialized map - KOKKOS_INLINE_FUNCTION - const Kokkos::Impl::ViewMapping& - impl_map() const { - return m_map; - } - //---------------------------------------- private: enum { is_layout_left = - std::is_same::value, + std::is_same_v, is_layout_right = - std::is_same::value, + std::is_same_v, - is_layout_stride = std::is_same::value, + is_layout_stride = + std::is_same_v, - is_default_map = std::is_void::value && - (is_layout_left || is_layout_right || is_layout_stride) + is_default_map = std::is_void_v && + (is_layout_left || is_layout_right || is_layout_stride), + + is_default_access = + is_default_map && std::is_same_v }; // Bounds checking macros @@ -570,476 +554,272 @@ class DynRankView : public ViewTraits { #endif public: - KOKKOS_INLINE_FUNCTION + KOKKOS_FUNCTION constexpr unsigned rank() const { return m_rank; } - // operators () - // Rank 0 - KOKKOS_INLINE_FUNCTION - reference_type operator()() const { - KOKKOS_IMPL_VIEW_OPERATOR_VERIFY((0, this->rank(), m_track, m_map)) - return impl_map().reference(); - // return m_map.reference(0,0,0,0,0,0,0); + using view_type::data; + using view_type::extent; + using view_type::extent_int; // FIXME: not tested + using view_type::impl_map; // FIXME: not tested + using view_type::is_allocated; + using view_type::label; + using view_type::size; + using view_type::span; + using view_type::span_is_contiguous; // FIXME: not tested + using view_type::stride; // FIXME: not tested + using view_type::stride_0; // FIXME: not tested + using view_type::stride_1; // FIXME: not tested + using view_type::stride_2; // FIXME: not tested + using view_type::stride_3; // FIXME: not tested + using view_type::stride_4; // FIXME: not tested + using view_type::stride_5; // FIXME: not tested + using view_type::stride_6; // FIXME: not tested + using view_type::stride_7; // FIXME: not tested + using view_type::use_count; + +#ifdef KOKKOS_ENABLE_CUDA + KOKKOS_FUNCTION reference_type + operator()(index_type i0 = 0, index_type i1 = 0, index_type i2 = 0, + index_type i3 = 0, index_type i4 = 0, index_type i5 = 0, + index_type i6 = 0) const { + return view_type::operator()(i0, i1, i2, i3, i4, i5, i6); + } +#else + // Adding shortcut operators for rank-0 to rank-3 for default layouts + // and access modalities. + // This removes performance overhead for always using rank-7 mapping. + // See https://github.com/kokkos/kokkos/issues/7604 + // When boundschecking is enabled we still go through the underlying + // rank-7 View to leverage the error checks there. + + KOKKOS_FUNCTION reference_type operator()() const { +#ifdef KOKKOS_ENABLE_DEBUG + if (rank() != 0u) + Kokkos::abort( + "DynRankView rank 0 operator() called with invalid number of " + "arguments."); +#endif +#ifndef KOKKOS_ENABLE_DEBUG_BOUNDS_CHECK + if constexpr (is_default_access) { + return view_type::data()[0]; + } else +#endif + return view_type::operator()(0, 0, 0, 0, 0, 0, 0); } - // Rank 1 - // This assumes a contiguous underlying memory (i.e. no padding, no - // striding...) - template - KOKKOS_INLINE_FUNCTION std::enable_if_t< - std::is_same::value && - std::is_integral::value, - reference_type> - operator[](const iType& i0) const { - // Phalanx is violating this, since they use the operator to access ALL - // elements in the allocation KOKKOS_IMPL_VIEW_OPERATOR_VERIFY( (1 , - // this->rank(), m_track, m_map) ) - return data()[i0]; + KOKKOS_FUNCTION reference_type operator()(index_type i0) const { +#ifdef KOKKOS_ENABLE_DEBUG + // FIXME: Should be equal, only access(...) allows mismatch of rank and + // index args + if (rank() > 1u) + Kokkos::abort( + "DynRankView rank 1 operator() called with invalid number of " + "arguments."); +#endif +#ifndef KOKKOS_ENABLE_DEBUG_BOUNDS_CHECK + if constexpr (is_default_access) { + if constexpr (is_layout_stride) { + return view_type::data()[i0 * view_type::stride(0)]; + } else { + return view_type::data()[i0]; + } + } else +#endif + return view_type::operator()(i0, 0, 0, 0, 0, 0, 0); +#if defined KOKKOS_COMPILER_INTEL || \ + (defined(KOKKOS_COMPILER_NVCC) && KOKKOS_COMPILER_NVCC >= 1130 && \ + !defined(KOKKOS_COMPILER_MSVC)) + __builtin_unreachable(); +#endif } - // This assumes a contiguous underlying memory (i.e. no padding, no - // striding... AND a Trilinos/Sacado scalar type ) - template - KOKKOS_INLINE_FUNCTION std::enable_if_t< - !std::is_same::value && - std::is_integral::value, - reference_type> - operator[](const iType& i0) const { - // auto map = impl_map(); - const size_t dim_scalar = m_map.dimension_scalar(); - const size_t bytes = this->span() / dim_scalar; - - using tmp_view_type = Kokkos::View< - DataType*, typename traits::array_layout, typename traits::device_type, - Kokkos::MemoryTraits>; - tmp_view_type rankone_view(this->data(), bytes, dim_scalar); - return rankone_view(i0); + KOKKOS_FUNCTION reference_type operator()(index_type i0, + index_type i1) const { +#ifdef KOKKOS_ENABLE_DEBUG + // FIXME: Should be equal, only access(...) allows mismatch of rank and + // index args + if (rank() > 2u) + Kokkos::abort( + "DynRankView rank 2 operator() called with invalid number of " + "arguments."); +#endif +#ifndef KOKKOS_ENABLE_DEBUG_BOUNDS_CHECK + if constexpr (is_default_access) { + if constexpr (is_layout_left) { + return view_type::data()[i0 + i1 * view_type::stride(1)]; + } else if constexpr (is_layout_right) { + return view_type::data()[i0 * view_type::extent(1) + i1]; + } else { + return view_type::data()[i0 * view_type::stride(0) + + i1 * view_type::stride(1)]; + } + } else +#endif + return view_type::operator()(i0, i1, 0, 0, 0, 0, 0); +#if defined KOKKOS_COMPILER_INTEL || \ + (defined(KOKKOS_COMPILER_NVCC) && KOKKOS_COMPILER_NVCC >= 1130 && \ + !defined(KOKKOS_COMPILER_MSVC)) + __builtin_unreachable(); +#endif } - // Rank 1 parenthesis - template - KOKKOS_INLINE_FUNCTION - std::enable_if_t<(std::is_void::value && - std::is_integral::value), - reference_type> - operator()(const iType& i0) const { - KOKKOS_IMPL_VIEW_OPERATOR_VERIFY((1, this->rank(), m_track, m_map, i0)) - return m_map.reference(i0); + KOKKOS_FUNCTION reference_type operator()(index_type i0, index_type i1, + index_type i2) const { +#ifdef KOKKOS_ENABLE_DEBUG + // FIXME: Should be equal, only access(...) allows mismatch of rank and + // index args + if (rank() > 3u) + Kokkos::abort( + "DynRankView rank 3 operator() called with invalid number of " + "arguments."); +#endif +#ifndef KOKKOS_ENABLE_DEBUG_BOUNDS_CHECK + if constexpr (is_default_access) { + if constexpr (is_layout_left) { + return view_type::data()[i0 + view_type::stride(1) * + (i1 + i2 * view_type::extent(1))]; + } else if constexpr (is_layout_right) { + return view_type::data()[(i0 * view_type::extent(1) + i1) * + view_type::extent(2) + + i2]; + } else { + return view_type::data()[i0 * view_type::stride(0) + + i1 * view_type::stride(1) + + i2 * view_type::stride(2)]; + } + } else +#endif + return view_type::operator()(i0, i1, i2, 0, 0, 0, 0); +#if defined KOKKOS_COMPILER_INTEL || \ + (defined(KOKKOS_COMPILER_NVCC) && KOKKOS_COMPILER_NVCC >= 1130 && \ + !defined(KOKKOS_COMPILER_MSVC)) + __builtin_unreachable(); +#endif } - template - KOKKOS_INLINE_FUNCTION - std::enable_if_t::value && - std::is_integral::value), - reference_type> - operator()(const iType& i0) const { - KOKKOS_IMPL_VIEW_OPERATOR_VERIFY((1, this->rank(), m_track, m_map, i0)) - return m_map.reference(i0, 0, 0, 0, 0, 0, 0); + KOKKOS_FUNCTION reference_type operator()(index_type i0, index_type i1, + index_type i2, index_type i3, + index_type i4 = 0, + index_type i5 = 0, + index_type i6 = 0) const { + return view_type::operator()(i0, i1, i2, i3, i4, i5, i6); } +#endif - // Rank 2 - template - KOKKOS_INLINE_FUNCTION std::enable_if_t< - (std::is_void::value && - std::is_integral::value && std::is_integral::value), - reference_type> - operator()(const iType0& i0, const iType1& i1) const { - KOKKOS_IMPL_VIEW_OPERATOR_VERIFY((2, this->rank(), m_track, m_map, i0, i1)) - return m_map.reference(i0, i1); +// This is an accomodation for Phalanx, that is usint the operator[] to access +// all elements in a linear fashion even when the rank is not 1 +#ifdef KOKKOS_ENABLE_DEPRECATED_CODE_4 + KOKKOS_FUNCTION reference_type operator[](index_type i0) const { + if constexpr (std::is_same_v) { + return view_type::data()[i0]; + } else { + const size_t dim_scalar = view_type::impl_map().dimension_scalar(); + const size_t bytes = view_type::span() / dim_scalar; + + using tmp_view_type = + Kokkos::View>; + tmp_view_type rankone_view(view_type::data(), bytes, dim_scalar); + return rankone_view(i0); + } } - - template - KOKKOS_INLINE_FUNCTION - std::enable_if_t::value && - std::is_integral::value), - reference_type> - operator()(const iType0& i0, const iType1& i1) const { - KOKKOS_IMPL_VIEW_OPERATOR_VERIFY((2, this->rank(), m_track, m_map, i0, i1)) - return m_map.reference(i0, i1, 0, 0, 0, 0, 0); +#else + KOKKOS_FUNCTION reference_type operator[](index_type i0) const { +#ifdef KOKKOS_ENABLE_DEBUG + if (rank() != 1u) + Kokkos::abort("DynRankView operator[] can only be used for rank-1"); +#endif + return view_type::operator()(i0, 0, 0, 0, 0, 0, 0); } +#endif - // Rank 3 - template - KOKKOS_INLINE_FUNCTION std::enable_if_t< - (std::is_void::value && - std::is_integral::value && std::is_integral::value && - std::is_integral::value), - reference_type> - operator()(const iType0& i0, const iType1& i1, const iType2& i2) const { - KOKKOS_IMPL_VIEW_OPERATOR_VERIFY( - (3, this->rank(), m_track, m_map, i0, i1, i2)) - return m_map.reference(i0, i1, i2); + KOKKOS_FUNCTION reference_type access(index_type i0 = 0, index_type i1 = 0, + index_type i2 = 0, index_type i3 = 0, + index_type i4 = 0, index_type i5 = 0, + index_type i6 = 0) const { + return view_type::operator()(i0, i1, i2, i3, i4, i5, i6); } - template - KOKKOS_INLINE_FUNCTION - std::enable_if_t::value && - std::is_integral::value), - reference_type> - operator()(const iType0& i0, const iType1& i1, const iType2& i2) const { - KOKKOS_IMPL_VIEW_OPERATOR_VERIFY( - (3, this->rank(), m_track, m_map, i0, i1, i2)) - return m_map.reference(i0, i1, i2, 0, 0, 0, 0); - } - - // Rank 4 - template - KOKKOS_INLINE_FUNCTION std::enable_if_t< - (std::is_void::value && - std::is_integral::value && std::is_integral::value && - std::is_integral::value && std::is_integral::value), - reference_type> - operator()(const iType0& i0, const iType1& i1, const iType2& i2, - const iType3& i3) const { - KOKKOS_IMPL_VIEW_OPERATOR_VERIFY( - (4, this->rank(), m_track, m_map, i0, i1, i2, i3)) - return m_map.reference(i0, i1, i2, i3); - } - - template - KOKKOS_INLINE_FUNCTION - std::enable_if_t::value && - std::is_integral::value), - reference_type> - operator()(const iType0& i0, const iType1& i1, const iType2& i2, - const iType3& i3) const { - KOKKOS_IMPL_VIEW_OPERATOR_VERIFY( - (4, this->rank(), m_track, m_map, i0, i1, i2, i3)) - return m_map.reference(i0, i1, i2, i3, 0, 0, 0); - } - - // Rank 5 - template - KOKKOS_INLINE_FUNCTION std::enable_if_t< - (std::is_void::value && - std::is_integral::value && std::is_integral::value && - std::is_integral::value && std::is_integral::value && - std::is_integral::value), - reference_type> - operator()(const iType0& i0, const iType1& i1, const iType2& i2, - const iType3& i3, const iType4& i4) const { - KOKKOS_IMPL_VIEW_OPERATOR_VERIFY( - (5, this->rank(), m_track, m_map, i0, i1, i2, i3, i4)) - return m_map.reference(i0, i1, i2, i3, i4); - } - - template - KOKKOS_INLINE_FUNCTION - std::enable_if_t::value && - std::is_integral::value), - reference_type> - operator()(const iType0& i0, const iType1& i1, const iType2& i2, - const iType3& i3, const iType4& i4) const { - KOKKOS_IMPL_VIEW_OPERATOR_VERIFY( - (5, this->rank(), m_track, m_map, i0, i1, i2, i3, i4)) - return m_map.reference(i0, i1, i2, i3, i4, 0, 0); - } - - // Rank 6 - template - KOKKOS_INLINE_FUNCTION std::enable_if_t< - (std::is_void::value && - std::is_integral::value && std::is_integral::value && - std::is_integral::value && std::is_integral::value && - std::is_integral::value && std::is_integral::value), - reference_type> - operator()(const iType0& i0, const iType1& i1, const iType2& i2, - const iType3& i3, const iType4& i4, const iType5& i5) const { - KOKKOS_IMPL_VIEW_OPERATOR_VERIFY( - (6, this->rank(), m_track, m_map, i0, i1, i2, i3, i4, i5)) - return m_map.reference(i0, i1, i2, i3, i4, i5); - } - - template - KOKKOS_INLINE_FUNCTION - std::enable_if_t::value && - std::is_integral::value), - reference_type> - operator()(const iType0& i0, const iType1& i1, const iType2& i2, - const iType3& i3, const iType4& i4, const iType5& i5) const { - KOKKOS_IMPL_VIEW_OPERATOR_VERIFY( - (6, this->rank(), m_track, m_map, i0, i1, i2, i3, i4, i5)) - return m_map.reference(i0, i1, i2, i3, i4, i5, 0); - } - - // Rank 7 - template - KOKKOS_INLINE_FUNCTION std::enable_if_t< - (std::is_integral::value && std::is_integral::value && - std::is_integral::value && std::is_integral::value && - std::is_integral::value && std::is_integral::value && - std::is_integral::value), - reference_type> - operator()(const iType0& i0, const iType1& i1, const iType2& i2, - const iType3& i3, const iType4& i4, const iType5& i5, - const iType6& i6) const { - KOKKOS_IMPL_VIEW_OPERATOR_VERIFY( - (7, this->rank(), m_track, m_map, i0, i1, i2, i3, i4, i5, i6)) - return m_map.reference(i0, i1, i2, i3, i4, i5, i6); - } - - // Rank 0 - KOKKOS_INLINE_FUNCTION - reference_type access() const { - KOKKOS_IMPL_VIEW_OPERATOR_VERIFY((0, this->rank(), m_track, m_map)) - return impl_map().reference(); - // return m_map.reference(0,0,0,0,0,0,0); - } - - // Rank 1 - // Rank 1 parenthesis - template - KOKKOS_INLINE_FUNCTION - std::enable_if_t<(std::is_void::value && - std::is_integral::value), - reference_type> - access(const iType& i0) const { - KOKKOS_IMPL_VIEW_OPERATOR_VERIFY((1, this->rank(), m_track, m_map, i0)) - return m_map.reference(i0); - } - - template - KOKKOS_INLINE_FUNCTION - std::enable_if_t::value && - std::is_integral::value), - reference_type> - access(const iType& i0) const { - KOKKOS_IMPL_VIEW_OPERATOR_VERIFY((1, this->rank(), m_track, m_map, i0)) - return m_map.reference(i0, 0, 0, 0, 0, 0, 0); - } - - // Rank 2 - template - KOKKOS_INLINE_FUNCTION std::enable_if_t< - (std::is_void::value && - std::is_integral::value && std::is_integral::value), - reference_type> - access(const iType0& i0, const iType1& i1) const { - KOKKOS_IMPL_VIEW_OPERATOR_VERIFY((2, this->rank(), m_track, m_map, i0, i1)) - return m_map.reference(i0, i1); - } - - template - KOKKOS_INLINE_FUNCTION - std::enable_if_t::value && - std::is_integral::value), - reference_type> - access(const iType0& i0, const iType1& i1) const { - KOKKOS_IMPL_VIEW_OPERATOR_VERIFY((2, this->rank(), m_track, m_map, i0, i1)) - return m_map.reference(i0, i1, 0, 0, 0, 0, 0); - } - - // Rank 3 - template - KOKKOS_INLINE_FUNCTION std::enable_if_t< - (std::is_void::value && - std::is_integral::value && std::is_integral::value && - std::is_integral::value), - reference_type> - access(const iType0& i0, const iType1& i1, const iType2& i2) const { - KOKKOS_IMPL_VIEW_OPERATOR_VERIFY( - (3, this->rank(), m_track, m_map, i0, i1, i2)) - return m_map.reference(i0, i1, i2); - } - - template - KOKKOS_INLINE_FUNCTION - std::enable_if_t::value && - std::is_integral::value), - reference_type> - access(const iType0& i0, const iType1& i1, const iType2& i2) const { - KOKKOS_IMPL_VIEW_OPERATOR_VERIFY( - (3, this->rank(), m_track, m_map, i0, i1, i2)) - return m_map.reference(i0, i1, i2, 0, 0, 0, 0); - } - - // Rank 4 - template - KOKKOS_INLINE_FUNCTION std::enable_if_t< - (std::is_void::value && - std::is_integral::value && std::is_integral::value && - std::is_integral::value && std::is_integral::value), - reference_type> - access(const iType0& i0, const iType1& i1, const iType2& i2, - const iType3& i3) const { - KOKKOS_IMPL_VIEW_OPERATOR_VERIFY( - (4, this->rank(), m_track, m_map, i0, i1, i2, i3)) - return m_map.reference(i0, i1, i2, i3); - } - - template - KOKKOS_INLINE_FUNCTION - std::enable_if_t::value && - std::is_integral::value), - reference_type> - access(const iType0& i0, const iType1& i1, const iType2& i2, - const iType3& i3) const { - KOKKOS_IMPL_VIEW_OPERATOR_VERIFY( - (4, this->rank(), m_track, m_map, i0, i1, i2, i3)) - return m_map.reference(i0, i1, i2, i3, 0, 0, 0); - } - - // Rank 5 - template - KOKKOS_INLINE_FUNCTION std::enable_if_t< - (std::is_void::value && - std::is_integral::value && std::is_integral::value && - std::is_integral::value && std::is_integral::value && - std::is_integral::value), - reference_type> - access(const iType0& i0, const iType1& i1, const iType2& i2, const iType3& i3, - const iType4& i4) const { - KOKKOS_IMPL_VIEW_OPERATOR_VERIFY( - (5, this->rank(), m_track, m_map, i0, i1, i2, i3, i4)) - return m_map.reference(i0, i1, i2, i3, i4); - } - - template - KOKKOS_INLINE_FUNCTION - std::enable_if_t::value && - std::is_integral::value), - reference_type> - access(const iType0& i0, const iType1& i1, const iType2& i2, - const iType3& i3, const iType4& i4) const { - KOKKOS_IMPL_VIEW_OPERATOR_VERIFY( - (5, this->rank(), m_track, m_map, i0, i1, i2, i3, i4)) - return m_map.reference(i0, i1, i2, i3, i4, 0, 0); - } - - // Rank 6 - template - KOKKOS_INLINE_FUNCTION std::enable_if_t< - (std::is_void::value && - std::is_integral::value && std::is_integral::value && - std::is_integral::value && std::is_integral::value && - std::is_integral::value && std::is_integral::value), - reference_type> - access(const iType0& i0, const iType1& i1, const iType2& i2, const iType3& i3, - const iType4& i4, const iType5& i5) const { - KOKKOS_IMPL_VIEW_OPERATOR_VERIFY( - (6, this->rank(), m_track, m_map, i0, i1, i2, i3, i4, i5)) - return m_map.reference(i0, i1, i2, i3, i4, i5); - } - - template - KOKKOS_INLINE_FUNCTION - std::enable_if_t::value && - std::is_integral::value), - reference_type> - access(const iType0& i0, const iType1& i1, const iType2& i2, - const iType3& i3, const iType4& i4, const iType5& i5) const { - KOKKOS_IMPL_VIEW_OPERATOR_VERIFY( - (6, this->rank(), m_track, m_map, i0, i1, i2, i3, i4, i5)) - return m_map.reference(i0, i1, i2, i3, i4, i5, 0); - } - - // Rank 7 - template - KOKKOS_INLINE_FUNCTION std::enable_if_t< - (std::is_integral::value && std::is_integral::value && - std::is_integral::value && std::is_integral::value && - std::is_integral::value && std::is_integral::value && - std::is_integral::value), - reference_type> - access(const iType0& i0, const iType1& i1, const iType2& i2, const iType3& i3, - const iType4& i4, const iType5& i5, const iType6& i6) const { - KOKKOS_IMPL_VIEW_OPERATOR_VERIFY( - (7, this->rank(), m_track, m_map, i0, i1, i2, i3, i4, i5, i6)) - return m_map.reference(i0, i1, i2, i3, i4, i5, i6); - } - -#undef KOKKOS_IMPL_VIEW_OPERATOR_VERIFY - //---------------------------------------- // Standard constructor, destructor, and assignment operators... KOKKOS_DEFAULTED_FUNCTION ~DynRankView() = default; - KOKKOS_INLINE_FUNCTION - DynRankView() : m_track(), m_map(), m_rank() {} // Default ctor - - KOKKOS_INLINE_FUNCTION - DynRankView(const DynRankView& rhs) - : m_track(rhs.m_track), m_map(rhs.m_map), m_rank(rhs.m_rank) {} - - KOKKOS_INLINE_FUNCTION - DynRankView(DynRankView&& rhs) - : m_track(rhs.m_track), m_map(rhs.m_map), m_rank(rhs.m_rank) {} - - KOKKOS_INLINE_FUNCTION - DynRankView& operator=(const DynRankView& rhs) { - m_track = rhs.m_track; - m_map = rhs.m_map; - m_rank = rhs.m_rank; - return *this; - } - - KOKKOS_INLINE_FUNCTION - DynRankView& operator=(DynRankView&& rhs) { - m_track = rhs.m_track; - m_map = rhs.m_map; - m_rank = rhs.m_rank; - return *this; - } + KOKKOS_DEFAULTED_FUNCTION DynRankView() = default; //---------------------------------------- // Compatible view copy constructor and assignment // may assign unmanaged from managed. + // Make this conditionally explicit? template - KOKKOS_INLINE_FUNCTION DynRankView(const DynRankView& rhs) - : m_track(rhs.m_track, traits::is_managed), m_map(), m_rank(rhs.m_rank) { - using SrcTraits = typename DynRankView::traits; - using Mapping = Kokkos::Impl::ViewMapping; - static_assert(Mapping::is_assignable, - "Incompatible DynRankView copy construction"); - Mapping::assign(m_map, rhs.m_map, rhs.m_track); - } + KOKKOS_FUNCTION DynRankView(const DynRankView& rhs) + : view_type(rhs), m_rank(rhs.m_rank) {} template - KOKKOS_INLINE_FUNCTION DynRankView& operator=( - const DynRankView& rhs) { - using SrcTraits = typename DynRankView::traits; - using Mapping = Kokkos::Impl::ViewMapping; - static_assert(Mapping::is_assignable, - "Incompatible DynRankView copy construction"); - Mapping::assign(m_map, rhs.m_map, rhs.m_track); - m_track.assign(rhs.m_track, traits::is_managed); - m_rank = rhs.rank(); + KOKKOS_FUNCTION DynRankView& operator=(const DynRankView& rhs) { + view_type::operator=(rhs); + m_rank = rhs.m_rank; return *this; } +#if 0 // TODO: this will later be swapped in depending on whether the new View + // impl is active + private: + template + KOKKOS_FUNCTION typename view_type::extents_type create_rank7_extents( + const Ext& ext) { + return typename view_type::extents_type( + ext.rank() > 0 ? ext.extent(0) : 1, ext.rank() > 1 ? ext.extent(1) : 1, + ext.rank() > 2 ? ext.extent(2) : 1, ext.rank() > 3 ? ext.extent(3) : 1, + ext.rank() > 4 ? ext.extent(4) : 1, ext.rank() > 5 ? ext.extent(5) : 1, + ext.rank() > 6 ? ext.extent(6) : 1); + } + + public: // Copy/Assign View to DynRankView template - KOKKOS_INLINE_FUNCTION DynRankView(const View& rhs) - : m_track(), m_map(), m_rank(View::rank()) { + KOKKOS_INLINE_FUNCTION DynRankView(const View& rhs, + size_t new_rank) + : view_type(rhs.data_handle(), drdtraits::createLayout(rhs.layout())), + m_rank(new_rank) { + if (new_rank > rhs.rank()) + Kokkos::abort( + "Attempting to construct DynRankView from View and new rank, with " + "the new rank being too large."); + } + + template + KOKKOS_INLINE_FUNCTION DynRankView& operator=(const View& rhs) { + view_type::operator=(view_type( + rhs.data_handle(), + typename view_type::mapping_type(create_rank7_extents(rhs.extents())), + rhs.accessor())); + m_rank = rhs.rank(); + return *this; + } +#else + template + KOKKOS_FUNCTION DynRankView(const View& rhs, size_t new_rank) { using SrcTraits = typename View::traits; using Mapping = Kokkos::Impl::ViewMapping; static_assert(Mapping::is_assignable, - "Incompatible View to DynRankView copy construction"); + "Incompatible View to DynRankView copy assignment"); + if (new_rank > View::rank()) + Kokkos::abort( + "Attempting to construct DynRankView from View and new rank, with " + "the new rank being too large."); Mapping::assign(*this, rhs); + m_rank = new_rank; } template - KOKKOS_INLINE_FUNCTION DynRankView& operator=(const View& rhs) { + KOKKOS_FUNCTION DynRankView& operator=(const View& rhs) { using SrcTraits = typename View::traits; using Mapping = Kokkos::Impl::ViewMapping { static_assert(Mapping::is_assignable, "Incompatible View to DynRankView copy assignment"); Mapping::assign(*this, rhs); + m_rank = View::rank(); return *this; } +#endif + + template + KOKKOS_FUNCTION DynRankView(const View& rhs) + : DynRankView(rhs, View::rank()) {} //---------------------------------------- // Allocation tracking properties - KOKKOS_INLINE_FUNCTION - int use_count() const { return m_track.use_count(); } - - inline const std::string label() const { - return m_track.template get_label(); - } - //---------------------------------------- // Allocation according to allocation properties and array layout // unused arg_layout dimensions must be set to KOKKOS_INVALID_INDEX so that // rank deduction can properly take place + // We need two variants to avoid calling host function from host device + // function warnings template - explicit inline DynRankView( - const Kokkos::Impl::ViewCtorProp& arg_prop, - std::enable_if_t::has_pointer, - typename traits::array_layout> const& arg_layout) - : m_track(), - m_map(), - m_rank(Impl::DynRankDimTraits:: - template computeRank( - arg_prop, arg_layout)) { - // Copy the input allocation properties with possibly defaulted properties - auto prop_copy = Impl::with_properties_if_unset( - arg_prop, std::string{}, typename traits::device_type::memory_space{}, - typename traits::device_type::execution_space{}); - using alloc_prop = decltype(prop_copy); - - static_assert(traits::is_managed, - "View allocation constructor requires managed memory"); - - if (alloc_prop::initialize && - !alloc_prop::execution_space::impl_is_initialized()) { - // If initializing view data then - // the execution space must be initialized. - Kokkos::Impl::throw_runtime_exception( - "Constructing DynRankView and initializing data with uninitialized " - "execution space"); - } - - Kokkos::Impl::SharedAllocationRecord<>* record = m_map.allocate_shared( - prop_copy, - Impl::DynRankDimTraits:: - template createLayout(arg_prop, arg_layout), - Impl::ViewCtorProp::has_execution_space); - - // Setup and initialization complete, start tracking - m_track.assign_allocated_record_to_uninitialized(record); - } - - // Wrappers - template - explicit KOKKOS_INLINE_FUNCTION DynRankView( + explicit KOKKOS_FUNCTION DynRankView( const Kokkos::Impl::ViewCtorProp& arg_prop, std::enable_if_t::has_pointer, - typename traits::array_layout> const& arg_layout) - : m_track() // No memory tracking - , - m_map(arg_prop, - Impl::DynRankDimTraits:: - template createLayout(arg_prop, arg_layout)), - m_rank(Impl::DynRankDimTraits:: - template computeRank( - arg_prop, arg_layout)) { - static_assert( - std::is_same::pointer_type>::value, - "Constructing DynRankView to wrap user memory must supply matching " - "pointer type"); - } + typename traits::array_layout const&> + arg_layout) + : view_type(arg_prop, drdtraits::template createLayout( + arg_prop, arg_layout)), + m_rank(drdtraits::computeRank(arg_prop, arg_layout)) {} + + template + explicit DynRankView( + const Kokkos::Impl::ViewCtorProp& arg_prop, + std::enable_if_t::has_pointer, + typename traits::array_layout const&> + arg_layout) + : view_type(arg_prop, drdtraits::template createLayout( + arg_prop, arg_layout)), + m_rank(drdtraits::computeRank(arg_prop, arg_layout)) {} //---------------------------------------- // Constructor(s) // Simple dimension-only layout + // We need two variants to avoid calling host function from host device + // function warnings template - explicit inline DynRankView( + explicit KOKKOS_FUNCTION DynRankView( const Kokkos::Impl::ViewCtorProp& arg_prop, - std::enable_if_t::has_pointer, - size_t> const arg_N0 = KOKKOS_INVALID_INDEX, - const size_t arg_N1 = KOKKOS_INVALID_INDEX, - const size_t arg_N2 = KOKKOS_INVALID_INDEX, - const size_t arg_N3 = KOKKOS_INVALID_INDEX, - const size_t arg_N4 = KOKKOS_INVALID_INDEX, - const size_t arg_N5 = KOKKOS_INVALID_INDEX, - const size_t arg_N6 = KOKKOS_INVALID_INDEX, - const size_t arg_N7 = KOKKOS_INVALID_INDEX) + std::enable_if_t::has_pointer, + const size_t> + arg_N0 = KOKKOS_INVALID_INDEX, + const size_t arg_N1 = KOKKOS_INVALID_INDEX, + const size_t arg_N2 = KOKKOS_INVALID_INDEX, + const size_t arg_N3 = KOKKOS_INVALID_INDEX, + const size_t arg_N4 = KOKKOS_INVALID_INDEX, + const size_t arg_N5 = KOKKOS_INVALID_INDEX, + const size_t arg_N6 = KOKKOS_INVALID_INDEX, + const size_t arg_N7 = KOKKOS_INVALID_INDEX) : DynRankView(arg_prop, typename traits::array_layout( arg_N0, arg_N1, arg_N2, arg_N3, arg_N4, arg_N5, arg_N6, arg_N7)) {} template - explicit KOKKOS_INLINE_FUNCTION DynRankView( + explicit DynRankView( const Kokkos::Impl::ViewCtorProp& arg_prop, - std::enable_if_t::has_pointer, - size_t> const arg_N0 = KOKKOS_INVALID_INDEX, - const size_t arg_N1 = KOKKOS_INVALID_INDEX, - const size_t arg_N2 = KOKKOS_INVALID_INDEX, - const size_t arg_N3 = KOKKOS_INVALID_INDEX, - const size_t arg_N4 = KOKKOS_INVALID_INDEX, - const size_t arg_N5 = KOKKOS_INVALID_INDEX, - const size_t arg_N6 = KOKKOS_INVALID_INDEX, - const size_t arg_N7 = KOKKOS_INVALID_INDEX) + std::enable_if_t::has_pointer, + const size_t> + arg_N0 = KOKKOS_INVALID_INDEX, + const size_t arg_N1 = KOKKOS_INVALID_INDEX, + const size_t arg_N2 = KOKKOS_INVALID_INDEX, + const size_t arg_N3 = KOKKOS_INVALID_INDEX, + const size_t arg_N4 = KOKKOS_INVALID_INDEX, + const size_t arg_N5 = KOKKOS_INVALID_INDEX, + const size_t arg_N6 = KOKKOS_INVALID_INDEX, + const size_t arg_N7 = KOKKOS_INVALID_INDEX) : DynRankView(arg_prop, typename traits::array_layout( arg_N0, arg_N1, arg_N2, arg_N3, arg_N4, arg_N5, arg_N6, arg_N7)) {} @@ -1188,16 +934,20 @@ class DynRankView : public ViewTraits { //---------------------------------------- // Memory span required to wrap these dimensions. + // FIXME: this function needs to be tested static constexpr size_t required_allocation_size( - const size_t arg_N0 = 0, const size_t arg_N1 = 0, const size_t arg_N2 = 0, - const size_t arg_N3 = 0, const size_t arg_N4 = 0, const size_t arg_N5 = 0, - const size_t arg_N6 = 0, const size_t arg_N7 = 0) { - return map_type::memory_span(typename traits::array_layout( - arg_N0, arg_N1, arg_N2, arg_N3, arg_N4, arg_N5, arg_N6, arg_N7)); + const size_t arg_N0 = 1, const size_t arg_N1 = 1, const size_t arg_N2 = 1, + const size_t arg_N3 = 1, const size_t arg_N4 = 1, const size_t arg_N5 = 1, + const size_t arg_N6 = 1, + [[maybe_unused]] const size_t arg_N7 = KOKKOS_INVALID_INDEX) { + // FIXME: check that arg_N7 is not set by user (in debug mode) + return view_type::required_allocation_size(arg_N0, arg_N1, arg_N2, arg_N3, + arg_N4, arg_N5, arg_N6); } - explicit KOKKOS_INLINE_FUNCTION DynRankView( - pointer_type arg_ptr, const size_t arg_N0 = KOKKOS_INVALID_INDEX, + explicit KOKKOS_FUNCTION DynRankView( + typename view_type::pointer_type arg_ptr, + const size_t arg_N0 = KOKKOS_INVALID_INDEX, const size_t arg_N1 = KOKKOS_INVALID_INDEX, const size_t arg_N2 = KOKKOS_INVALID_INDEX, const size_t arg_N3 = KOKKOS_INVALID_INDEX, @@ -1205,55 +955,38 @@ class DynRankView : public ViewTraits { const size_t arg_N5 = KOKKOS_INVALID_INDEX, const size_t arg_N6 = KOKKOS_INVALID_INDEX, const size_t arg_N7 = KOKKOS_INVALID_INDEX) - : DynRankView(Kokkos::Impl::ViewCtorProp(arg_ptr), arg_N0, - arg_N1, arg_N2, arg_N3, arg_N4, arg_N5, arg_N6, arg_N7) {} + : DynRankView( + Kokkos::Impl::ViewCtorProp( + arg_ptr), + arg_N0, arg_N1, arg_N2, arg_N3, arg_N4, arg_N5, arg_N6, arg_N7) {} - explicit KOKKOS_INLINE_FUNCTION DynRankView( - pointer_type arg_ptr, typename traits::array_layout& arg_layout) - : DynRankView(Kokkos::Impl::ViewCtorProp(arg_ptr), - arg_layout) {} + explicit KOKKOS_FUNCTION DynRankView( + typename view_type::pointer_type arg_ptr, + typename traits::array_layout& arg_layout) + : DynRankView( + Kokkos::Impl::ViewCtorProp( + arg_ptr), + arg_layout) {} //---------------------------------------- // Shared scratch memory constructor - static inline size_t shmem_size(const size_t arg_N0 = KOKKOS_INVALID_INDEX, - const size_t arg_N1 = KOKKOS_INVALID_INDEX, - const size_t arg_N2 = KOKKOS_INVALID_INDEX, - const size_t arg_N3 = KOKKOS_INVALID_INDEX, - const size_t arg_N4 = KOKKOS_INVALID_INDEX, - const size_t arg_N5 = KOKKOS_INVALID_INDEX, - const size_t arg_N6 = KOKKOS_INVALID_INDEX, - const size_t arg_N7 = KOKKOS_INVALID_INDEX) { - const size_t num_passed_args = - (arg_N0 != KOKKOS_INVALID_INDEX) + (arg_N1 != KOKKOS_INVALID_INDEX) + - (arg_N2 != KOKKOS_INVALID_INDEX) + (arg_N3 != KOKKOS_INVALID_INDEX) + - (arg_N4 != KOKKOS_INVALID_INDEX) + (arg_N5 != KOKKOS_INVALID_INDEX) + - (arg_N6 != KOKKOS_INVALID_INDEX) + (arg_N7 != KOKKOS_INVALID_INDEX); - - if (std::is_void::value && - num_passed_args != traits::rank_dynamic) { - Kokkos::abort( - "Kokkos::View::shmem_size() rank_dynamic != number of arguments.\n"); - } - {} - - return map_type::memory_span(typename traits::array_layout( - arg_N0, arg_N1, arg_N2, arg_N3, arg_N4, arg_N5, arg_N6, arg_N7)); + // Note: We must pass 7 valid args since view_type is rank 7 + static inline size_t shmem_size( + const size_t arg_N0 = 1, const size_t arg_N1 = 1, const size_t arg_N2 = 1, + const size_t arg_N3 = 1, const size_t arg_N4 = 1, const size_t arg_N5 = 1, + const size_t arg_N6 = 1, const size_t arg_N7 = KOKKOS_INVALID_INDEX) { + return view_type::shmem_size(arg_N0, arg_N1, arg_N2, arg_N3, arg_N4, arg_N5, + arg_N6, arg_N7); } - explicit KOKKOS_INLINE_FUNCTION DynRankView( + explicit KOKKOS_FUNCTION DynRankView( const typename traits::execution_space::scratch_memory_space& arg_space, const typename traits::array_layout& arg_layout) - : DynRankView( - Kokkos::Impl::ViewCtorProp( - reinterpret_cast( - arg_space.get_shmem(map_type::memory_span( - Impl::DynRankDimTraits:: - createLayout(arg_layout) // is this correct? - )))), - arg_layout) {} + : view_type(arg_space, drdtraits::createLayout(arg_layout)), + m_rank(drdtraits::computeRank(arg_layout)) {} - explicit KOKKOS_INLINE_FUNCTION DynRankView( + explicit KOKKOS_FUNCTION DynRankView( const typename traits::execution_space::scratch_memory_space& arg_space, const size_t arg_N0 = KOKKOS_INVALID_INDEX, const size_t arg_N1 = KOKKOS_INVALID_INDEX, @@ -1264,21 +997,38 @@ class DynRankView : public ViewTraits { const size_t arg_N6 = KOKKOS_INVALID_INDEX, const size_t arg_N7 = KOKKOS_INVALID_INDEX) - : DynRankView( - Kokkos::Impl::ViewCtorProp( - reinterpret_cast( - arg_space.get_shmem(map_type::memory_span( - Impl::DynRankDimTraits:: - createLayout(typename traits::array_layout( - arg_N0, arg_N1, arg_N2, arg_N3, arg_N4, arg_N5, - arg_N6, arg_N7)))))), - typename traits::array_layout(arg_N0, arg_N1, arg_N2, arg_N3, - arg_N4, arg_N5, arg_N6, arg_N7)) {} + : DynRankView(arg_space, typename traits::array_layout( + arg_N0, arg_N1, arg_N2, arg_N3, arg_N4, + arg_N5, arg_N6, arg_N7)) {} + + KOKKOS_FUNCTION constexpr auto layout() const { + switch (rank()) { + case 0: return Impl::as_view_of_rank_n<0>(*this).layout(); + case 1: return Impl::as_view_of_rank_n<1>(*this).layout(); + case 2: return Impl::as_view_of_rank_n<2>(*this).layout(); + case 3: return Impl::as_view_of_rank_n<3>(*this).layout(); + case 4: return Impl::as_view_of_rank_n<4>(*this).layout(); + case 5: return Impl::as_view_of_rank_n<5>(*this).layout(); + case 6: return Impl::as_view_of_rank_n<6>(*this).layout(); + case 7: return Impl::as_view_of_rank_n<7>(*this).layout(); + default: + KOKKOS_IF_ON_HOST( + Kokkos::abort( + std::string( + "Calling DynRankView::layout on DRV of unexpected rank " + + std::to_string(rank())) + .c_str());) + KOKKOS_IF_ON_DEVICE( + Kokkos::abort( + "Calling DynRankView::layout on DRV of unexpected rank");) + } + // control flow should never reach here + return view_type::layout(); + } }; template -KOKKOS_INLINE_FUNCTION constexpr unsigned rank( - const DynRankView& DRV) { +KOKKOS_FUNCTION constexpr unsigned rank(const DynRankView& DRV) { return DRV.rank(); } // needed for transition to common constexpr method in view and dynrankview // to return rank @@ -1293,181 +1043,46 @@ struct DynRankSubviewTag {}; } // namespace Impl -namespace Impl { - -template -class ViewMapping< - std::enable_if_t<(std::is_void::value && - (std::is_same::value || - std::is_same::value || - std::is_same::value)), - Kokkos::Impl::DynRankSubviewTag>, - SrcTraits, Args...> { - private: - enum { - RZ = false, - R0 = bool(is_integral_extent<0, Args...>::value), - R1 = bool(is_integral_extent<1, Args...>::value), - R2 = bool(is_integral_extent<2, Args...>::value), - R3 = bool(is_integral_extent<3, Args...>::value), - R4 = bool(is_integral_extent<4, Args...>::value), - R5 = bool(is_integral_extent<5, Args...>::value), - R6 = bool(is_integral_extent<6, Args...>::value) - }; - - enum { - rank = unsigned(R0) + unsigned(R1) + unsigned(R2) + unsigned(R3) + - unsigned(R4) + unsigned(R5) + unsigned(R6) - }; - - using array_layout = Kokkos::LayoutStride; - - using value_type = typename SrcTraits::value_type; - - using data_type = value_type*******; - - public: - using traits_type = Kokkos::ViewTraits; - - using type = - Kokkos::View; - - template - struct apply { - static_assert(Kokkos::is_memory_traits::value); - - using traits_type = - Kokkos::ViewTraits; - - using type = Kokkos::View; - }; - - using dimension = typename SrcTraits::dimension; - - template - struct ExtentGenerator { - KOKKOS_INLINE_FUNCTION - static SubviewExtents<7, rank> generator( - const dimension& dim, Arg0 arg0 = Arg0(), Arg1 arg1 = Arg1(), - Arg2 arg2 = Arg2(), Arg3 arg3 = Arg3(), Arg4 arg4 = Arg4(), - Arg5 arg5 = Arg5(), Arg6 arg6 = Arg6()) { - return SubviewExtents<7, rank>(dim, arg0, arg1, arg2, arg3, arg4, arg5, - arg6); - } - }; - - using ret_type = Kokkos::DynRankView; - - template - KOKKOS_INLINE_FUNCTION static ret_type subview( - const unsigned src_rank, Kokkos::DynRankView const& src, - Args... args) { - using DstType = ViewMapping; - - using DstDimType = std::conditional_t< - (rank == 0), ViewDimension<>, - std::conditional_t< - (rank == 1), ViewDimension<0>, - std::conditional_t< - (rank == 2), ViewDimension<0, 0>, - std::conditional_t< - (rank == 3), ViewDimension<0, 0, 0>, - std::conditional_t< - (rank == 4), ViewDimension<0, 0, 0, 0>, - std::conditional_t< - (rank == 5), ViewDimension<0, 0, 0, 0, 0>, - std::conditional_t< - (rank == 6), ViewDimension<0, 0, 0, 0, 0, 0>, - ViewDimension<0, 0, 0, 0, 0, 0, 0>>>>>>>>; - - using dst_offset_type = ViewOffset; - using dst_handle_type = typename DstType::handle_type; - - ret_type dst; - - const SubviewExtents<7, rank> extents = ExtentGenerator::generator( - src.m_map.m_impl_offset.m_dim, args...); - - dst_offset_type tempdst(src.m_map.m_impl_offset, extents); - - dst.m_track = src.m_track; - - dst.m_map.m_impl_offset.m_dim.N0 = tempdst.m_dim.N0; - dst.m_map.m_impl_offset.m_dim.N1 = tempdst.m_dim.N1; - dst.m_map.m_impl_offset.m_dim.N2 = tempdst.m_dim.N2; - dst.m_map.m_impl_offset.m_dim.N3 = tempdst.m_dim.N3; - dst.m_map.m_impl_offset.m_dim.N4 = tempdst.m_dim.N4; - dst.m_map.m_impl_offset.m_dim.N5 = tempdst.m_dim.N5; - dst.m_map.m_impl_offset.m_dim.N6 = tempdst.m_dim.N6; - - dst.m_map.m_impl_offset.m_stride.S0 = tempdst.m_stride.S0; - dst.m_map.m_impl_offset.m_stride.S1 = tempdst.m_stride.S1; - dst.m_map.m_impl_offset.m_stride.S2 = tempdst.m_stride.S2; - dst.m_map.m_impl_offset.m_stride.S3 = tempdst.m_stride.S3; - dst.m_map.m_impl_offset.m_stride.S4 = tempdst.m_stride.S4; - dst.m_map.m_impl_offset.m_stride.S5 = tempdst.m_stride.S5; - dst.m_map.m_impl_offset.m_stride.S6 = tempdst.m_stride.S6; - - dst.m_map.m_impl_handle = - dst_handle_type(src.m_map.m_impl_handle + - src.m_map.m_impl_offset( - extents.domain_offset(0), extents.domain_offset(1), - extents.domain_offset(2), extents.domain_offset(3), - extents.domain_offset(4), extents.domain_offset(5), - extents.domain_offset(6))); - - dst.m_rank = - (src_rank > 0 ? unsigned(R0) : 0) + (src_rank > 1 ? unsigned(R1) : 0) + - (src_rank > 2 ? unsigned(R2) : 0) + (src_rank > 3 ? unsigned(R3) : 0) + - (src_rank > 4 ? unsigned(R4) : 0) + (src_rank > 5 ? unsigned(R5) : 0) + - (src_rank > 6 ? unsigned(R6) : 0); - - return dst; - } -}; - -} // namespace Impl - template using Subdynrankview = typename Kokkos::Impl::ViewMapping::ret_type; -template -KOKKOS_INLINE_FUNCTION Subdynrankview, Args...> -subdynrankview(const Kokkos::DynRankView& src, Args... args) { - if (src.rank() > sizeof...(Args)) // allow sizeof...(Args) >= src.rank(), - // ignore the remaining args - { - Kokkos::abort( - "subdynrankview: num of args must be >= rank of the source " - "DynRankView"); - } +template +KOKKOS_INLINE_FUNCTION auto subdynrankview( + const DynRankView& drv, SubArg0 arg0 = SubArg0{}, + SubArg1 arg1 = SubArg1{}, SubArg2 arg2 = SubArg2{}, + SubArg3 arg3 = SubArg3{}, SubArg4 arg4 = SubArg4{}, + SubArg5 arg5 = SubArg5{}, SubArg6 arg6 = SubArg6{}) { + auto sub = subview(drv.DownCast(), arg0, arg1, arg2, arg3, arg4, arg5, arg6); + using sub_t = decltype(sub); + size_t new_rank = (drv.rank() > 0 && !std::is_integral_v ? 1 : 0) + + (drv.rank() > 1 && !std::is_integral_v ? 1 : 0) + + (drv.rank() > 2 && !std::is_integral_v ? 1 : 0) + + (drv.rank() > 3 && !std::is_integral_v ? 1 : 0) + + (drv.rank() > 4 && !std::is_integral_v ? 1 : 0) + + (drv.rank() > 5 && !std::is_integral_v ? 1 : 0) + + (drv.rank() > 6 && !std::is_integral_v ? 1 : 0); - using metafcn = - Kokkos::Impl::ViewMapping, Args...>; - - return metafcn::subview(src.rank(), src, args...); + using return_type = + DynRankView; + return static_cast( + DynRankView( + sub, new_rank)); } - -// Wrapper to allow subview function name -template -KOKKOS_INLINE_FUNCTION Subdynrankview, Args...> -subview(const Kokkos::DynRankView& src, Args... args) { - return subdynrankview(src, args...); +template +KOKKOS_INLINE_FUNCTION auto subview( + const DynRankView& drv, SubArg0 arg0 = SubArg0{}, + SubArg1 arg1 = SubArg1{}, SubArg2 arg2 = SubArg2{}, + SubArg3 arg3 = SubArg3{}, SubArg4 arg4 = SubArg4{}, + SubArg5 arg5 = SubArg5{}, SubArg6 arg6 = SubArg6{}) { + return subdynrankview(drv, arg0, arg1, arg2, arg3, arg4, arg5, arg6); } } // namespace Kokkos @@ -1482,12 +1097,12 @@ KOKKOS_INLINE_FUNCTION bool operator==(const DynRankView& lhs, using lhs_traits = ViewTraits; using rhs_traits = ViewTraits; - return std::is_same::value && - std::is_same::value && - std::is_same::value && + return std::is_same_v && + std::is_same_v && + std::is_same_v && lhs.rank() == rhs.rank() && lhs.data() == rhs.data() && lhs.span() == rhs.span() && lhs.extent(0) == rhs.extent(0) && lhs.extent(1) == rhs.extent(1) && lhs.extent(2) == rhs.extent(2) && @@ -1638,11 +1253,11 @@ namespace Impl { underlying memory, to facilitate implementation of deep_copy() and other routines that are defined on View */ template -KOKKOS_FUNCTION auto as_view_of_rank_n( +KOKKOS_FUNCTION View::type, Args...> +as_view_of_rank_n( DynRankView v, - typename std::enable_if::specialize, void>::value>::type* = - nullptr) { + std::enable_if_t< + std::is_same_v::specialize, void>>*) { if (v.rank() != N) { KOKKOS_IF_ON_HOST( const std::string message = @@ -1653,7 +1268,7 @@ KOKKOS_FUNCTION auto as_view_of_rank_n( Kokkos::abort("Converting DynRankView to a View of mis-matched rank!");) } - auto layout = v.impl_map().layout(); + auto layout = v.DownCast().layout(); if constexpr (std::is_same_v || std::is_same_v || @@ -1691,43 +1306,16 @@ void apply_to_view_of_static_rank(Function&& f, DynRankView a) { } // namespace Impl -template -KOKKOS_INLINE_FUNCTION constexpr auto DynRankView::layout() const -> - typename traits::array_layout { - switch (rank()) { - case 0: return Impl::as_view_of_rank_n<0>(*this).layout(); - case 1: return Impl::as_view_of_rank_n<1>(*this).layout(); - case 2: return Impl::as_view_of_rank_n<2>(*this).layout(); - case 3: return Impl::as_view_of_rank_n<3>(*this).layout(); - case 4: return Impl::as_view_of_rank_n<4>(*this).layout(); - case 5: return Impl::as_view_of_rank_n<5>(*this).layout(); - case 6: return Impl::as_view_of_rank_n<6>(*this).layout(); - case 7: return Impl::as_view_of_rank_n<7>(*this).layout(); - default: - KOKKOS_IF_ON_HOST( - Kokkos::abort( - std::string( - "Calling DynRankView::layout on DRV of unexpected rank " + - std::to_string(rank())) - .c_str());) - KOKKOS_IF_ON_DEVICE( - Kokkos::abort( - "Calling DynRankView::layout on DRV of unexpected rank");) - } - // control flow should never reach here - return m_map.layout(); -} - /** \brief Deep copy a value from Host memory into a view. */ template inline void deep_copy( const ExecSpace& e, const DynRankView& dst, typename ViewTraits::const_value_type& value, - std::enable_if_t::specialize, - void>::value>* = nullptr) { + std::enable_if_t::specialize, + void>>* = nullptr) { static_assert( - std::is_same::non_const_value_type, - typename ViewTraits::value_type>::value, + std::is_same_v::non_const_value_type, + typename ViewTraits::value_type>, "deep_copy requires non-const type"); Impl::apply_to_view_of_static_rank( @@ -1738,8 +1326,8 @@ template inline void deep_copy( const DynRankView& dst, typename ViewTraits::const_value_type& value, - std::enable_if_t::specialize, - void>::value>* = nullptr) { + std::enable_if_t::specialize, + void>>* = nullptr) { Impl::apply_to_view_of_static_rank([=](auto view) { deep_copy(view, value); }, dst); } @@ -1750,8 +1338,8 @@ inline void deep_copy( const ExecSpace& e, typename ViewTraits::non_const_value_type& dst, const DynRankView& src, - std::enable_if_t::specialize, - void>::value>* = 0) { + std::enable_if_t::specialize, + void>>* = 0) { deep_copy(e, dst, Impl::as_view_of_rank_n<0>(src)); } @@ -1759,8 +1347,8 @@ template inline void deep_copy( typename ViewTraits::non_const_value_type& dst, const DynRankView& src, - std::enable_if_t::specialize, - void>::value>* = 0) { + std::enable_if_t::specialize, + void>>* = 0) { deep_copy(dst, Impl::as_view_of_rank_n<0>(src)); } @@ -1773,15 +1361,13 @@ inline void deep_copy( template inline void deep_copy( const ExecSpace& exec_space, const DstType& dst, const SrcType& src, - std::enable_if_t< - (std::is_void::value && - std::is_void::value && - (Kokkos::is_dyn_rank_view::value || - Kokkos::is_dyn_rank_view::value))>* = nullptr) { - static_assert( - std::is_same::value, - "deep_copy requires non-const destination type"); + std::enable_if_t<(std::is_void_v && + std::is_void_v && + (Kokkos::is_dyn_rank_view::value || + Kokkos::is_dyn_rank_view::value))>* = nullptr) { + static_assert(std::is_same_v, + "deep_copy requires non-const destination type"); switch (rank(dst)) { case 0: @@ -1826,15 +1412,13 @@ inline void deep_copy( template inline void deep_copy( const DstType& dst, const SrcType& src, - std::enable_if_t< - (std::is_void::value && - std::is_void::value && - (Kokkos::is_dyn_rank_view::value || - Kokkos::is_dyn_rank_view::value))>* = nullptr) { - static_assert( - std::is_same::value, - "deep_copy requires non-const destination type"); + std::enable_if_t<(std::is_void_v && + std::is_void_v && + (Kokkos::is_dyn_rank_view::value || + Kokkos::is_dyn_rank_view::value))>* = nullptr) { + static_assert(std::is_same_v, + "deep_copy requires non-const destination type"); switch (rank(dst)) { case 0: @@ -1894,7 +1478,7 @@ struct MirrorDRViewType { // Check whether it is the same memory space enum { is_same_memspace = - std::is_same::value + std::is_same_v }; // The array_layout using array_layout = typename src_view_type::array_layout; @@ -1909,26 +1493,6 @@ struct MirrorDRViewType { std::conditional_t; }; -template -struct MirrorDRVType { - // The incoming view_type - using src_view_type = typename Kokkos::DynRankView; - // The memory space for the mirror view - using memory_space = typename Space::memory_space; - // Check whether it is the same memory space - enum { - is_same_memspace = - std::is_same::value - }; - // The array_layout - using array_layout = typename src_view_type::array_layout; - // The data type (we probably want it non-const since otherwise we can't even - // deep_copy to it. - using data_type = typename src_view_type::non_const_data_type; - // The destination view type if it is not the same memory space - using view_type = Kokkos::DynRankView; -}; - } // namespace Impl namespace Impl { @@ -1945,10 +1509,9 @@ inline auto create_mirror(const DynRankView& src, arg_prop, std::string(src.label()).append("_mirror")); if constexpr (Impl::ViewCtorProp::has_memory_space) { - using dst_type = typename Impl::MirrorDRVType< + using dst_type = typename Impl::MirrorDRViewType< typename Impl::ViewCtorProp::memory_space, T, - P...>::view_type; - + P...>::dest_view_type; return dst_type(prop_copy, Impl::reconstructLayout(src.layout(), src.rank())); } else { @@ -1989,7 +1552,8 @@ template ::value && std::is_void_v::specialize>>> -auto create_mirror(const Space&, const Kokkos::DynRankView& src) { +inline auto create_mirror(const Space&, + const Kokkos::DynRankView& src) { return Impl::create_mirror( src, Kokkos::view_alloc(typename Space::memory_space{})); } @@ -1999,8 +1563,8 @@ template ::value && std::is_void_v::specialize>>> -auto create_mirror(Kokkos::Impl::WithoutInitializing_t wi, const Space&, - const Kokkos::DynRankView& src) { +inline auto create_mirror(Kokkos::Impl::WithoutInitializing_t wi, const Space&, + const Kokkos::DynRankView& src) { return Impl::create_mirror( src, Kokkos::view_alloc(wi, typename Space::memory_space{})); } @@ -2026,12 +1590,12 @@ inline auto create_mirror_view( [[maybe_unused]] const typename Impl::ViewCtorProp& arg_prop) { if constexpr (!Impl::ViewCtorProp::has_memory_space) { - if constexpr (std::is_same::memory_space, - typename DynRankView< - T, P...>::HostMirror::memory_space>::value && - std::is_same::data_type, - typename DynRankView< - T, P...>::HostMirror::data_type>::value) { + if constexpr (std::is_same_v::memory_space, + typename DynRankView< + T, P...>::HostMirror::memory_space> && + std::is_same_v< + typename DynRankView::data_type, + typename DynRankView::HostMirror::data_type>) { return typename DynRankView::HostMirror(src); } else { return Kokkos::Impl::choose_create_mirror(src, arg_prop); @@ -2102,7 +1666,7 @@ inline auto create_mirror_view( // view_alloc template ::specialize>::value>> + std::is_void_v::specialize>>> auto create_mirror_view_and_copy( [[maybe_unused]] const Impl::ViewCtorProp& arg_prop, const Kokkos::DynRankView& src) { diff --git a/lib/kokkos/containers/src/Kokkos_DynamicView.hpp b/lib/kokkos/containers/src/Kokkos_DynamicView.hpp index a4b74e246e..caae3f791f 100644 --- a/lib/kokkos/containers/src/Kokkos_DynamicView.hpp +++ b/lib/kokkos/containers/src/Kokkos_DynamicView.hpp @@ -40,10 +40,10 @@ struct ChunkedArrayManager { using pointer_type = ValueType*; using track_type = Kokkos::Impl::SharedAllocationTracker; - ChunkedArrayManager() = default; - ChunkedArrayManager(ChunkedArrayManager const&) = default; - ChunkedArrayManager(ChunkedArrayManager&&) = default; - ChunkedArrayManager& operator=(ChunkedArrayManager&&) = default; + ChunkedArrayManager() = default; + ChunkedArrayManager(ChunkedArrayManager const&) = default; + ChunkedArrayManager(ChunkedArrayManager&&) = default; + ChunkedArrayManager& operator=(ChunkedArrayManager&&) = default; ChunkedArrayManager& operator=(const ChunkedArrayManager&) = default; template @@ -129,10 +129,10 @@ struct ChunkedArrayManager { /// allocation template struct Destroy { - Destroy() = default; - Destroy(Destroy&&) = default; - Destroy(const Destroy&) = default; - Destroy& operator=(Destroy&&) = default; + Destroy() = default; + Destroy(Destroy&&) = default; + Destroy(const Destroy&) = default; + Destroy& operator=(Destroy&&) = default; Destroy& operator=(const Destroy&) = default; Destroy(std::string label, value_type** arg_chunk, @@ -250,7 +250,7 @@ class DynamicView : public Kokkos::ViewTraits { // It is assumed that the value_type is trivially copyable; // when this is not the case, potential problems can occur. - static_assert(std::is_void::value, + static_assert(std::is_void_v, "DynamicView only implemented for non-specialized View type"); private: @@ -363,7 +363,7 @@ class DynamicView : public Kokkos::ViewTraits { enum { reference_type_is_lvalue_reference = - std::is_lvalue_reference::value + std::is_lvalue_reference_v }; KOKKOS_INLINE_FUNCTION constexpr bool span_is_contiguous() const { @@ -463,11 +463,11 @@ class DynamicView : public Kokkos::ViewTraits { //---------------------------------------------------------------------- - ~DynamicView() = default; - DynamicView() = default; - DynamicView(DynamicView&&) = default; - DynamicView(const DynamicView&) = default; - DynamicView& operator=(DynamicView&&) = default; + ~DynamicView() = default; + DynamicView() = default; + DynamicView(DynamicView&&) = default; + DynamicView(const DynamicView&) = default; + DynamicView& operator=(DynamicView&&) = default; DynamicView& operator=(const DynamicView&) = default; template @@ -572,7 +572,7 @@ struct MirrorDynamicViewType { // Check whether it is the same memory space enum { is_same_memspace = - std::is_same::value + std::is_same_v }; // The array_layout using array_layout = typename src_view_type::array_layout; @@ -665,9 +665,9 @@ template ::value && std::is_void_v::specialize>>> -typename Kokkos::Impl::MirrorDynamicViewType::view_type -create_mirror(Kokkos::Impl::WithoutInitializing_t wi, const Space&, - const Kokkos::Experimental::DynamicView& src) { +inline auto create_mirror( + Kokkos::Impl::WithoutInitializing_t wi, const Space&, + const Kokkos::Experimental::DynamicView& src) { return Impl::create_mirror( src, Kokkos::view_alloc(wi, typename Space::memory_space{})); } @@ -693,14 +693,14 @@ inline auto create_mirror_view( const Kokkos::Experimental::DynamicView& src, [[maybe_unused]] const Impl::ViewCtorProp& arg_prop) { if constexpr (!Impl::ViewCtorProp::has_memory_space) { - if constexpr (std::is_same::memory_space, - typename Kokkos::Experimental::DynamicView< - T, P...>::HostMirror::memory_space>::value && - std::is_same::data_type, - typename Kokkos::Experimental::DynamicView< - T, P...>::HostMirror::data_type>::value) { + if constexpr (std::is_same_v::memory_space, + typename Kokkos::Experimental::DynamicView< + T, P...>::HostMirror::memory_space> && + std::is_same_v::data_type, + typename Kokkos::Experimental::DynamicView< + T, P...>::HostMirror::data_type>) { return typename Kokkos::Experimental::DynamicView::HostMirror(src); } else { @@ -835,21 +835,17 @@ inline void deep_copy(const View& dst, using dst_execution_space = typename ViewTraits::execution_space; using src_memory_space = typename ViewTraits::memory_space; - enum { - DstExecCanAccessSrc = - Kokkos::SpaceAccessibility::accessible - }; + constexpr bool DstExecCanAccessSrc = + Kokkos::SpaceAccessibility::accessible; + static_assert( + DstExecCanAccessSrc, + "deep_copy given views that would require a temporary allocation"); - if (DstExecCanAccessSrc) { - // Copying data between views in accessible memory spaces and either - // non-contiguous or incompatible shape. - Kokkos::Impl::ViewRemap(dst, src); - Kokkos::fence("Kokkos::deep_copy(DynamicView)"); - } else { - Kokkos::Impl::throw_runtime_exception( - "deep_copy given views that would require a temporary allocation"); - } + // Copying data between views in accessible memory spaces and either + // non-contiguous or incompatible shape. + Kokkos::Impl::ViewRemap(dst, src); + Kokkos::fence("Kokkos::deep_copy(DynamicView)"); } template @@ -861,21 +857,17 @@ inline void deep_copy(const Kokkos::Experimental::DynamicView& dst, using dst_execution_space = typename ViewTraits::execution_space; using src_memory_space = typename ViewTraits::memory_space; - enum { - DstExecCanAccessSrc = - Kokkos::SpaceAccessibility::accessible - }; + constexpr bool DstExecCanAccessSrc = + Kokkos::SpaceAccessibility::accessible; + static_assert( + DstExecCanAccessSrc, + "deep_copy given views that would require a temporary allocation"); - if (DstExecCanAccessSrc) { - // Copying data between views in accessible memory spaces and either - // non-contiguous or incompatible shape. - Kokkos::Impl::ViewRemap(dst, src); - Kokkos::fence("Kokkos::deep_copy(DynamicView)"); - } else { - Kokkos::Impl::throw_runtime_exception( - "deep_copy given views that would require a temporary allocation"); - } + // Copying data between views in accessible memory spaces and either + // non-contiguous or incompatible shape. + Kokkos::Impl::ViewRemap(dst, src); + Kokkos::fence("Kokkos::deep_copy(DynamicView)"); } namespace Impl { @@ -964,7 +956,7 @@ struct ViewCopy, // view_alloc template ::specialize>::value>> + std::is_void_v::specialize>>> auto create_mirror_view_and_copy( [[maybe_unused]] const Impl::ViewCtorProp& arg_prop, const Kokkos::Experimental::DynamicView& src) { diff --git a/lib/kokkos/containers/src/Kokkos_OffsetView.hpp b/lib/kokkos/containers/src/Kokkos_OffsetView.hpp index 3adc70b190..cf23c25b86 100644 --- a/lib/kokkos/containers/src/Kokkos_OffsetView.hpp +++ b/lib/kokkos/containers/src/Kokkos_OffsetView.hpp @@ -50,9 +50,9 @@ inline constexpr bool is_offset_view_v = is_offset_view::value; #define KOKKOS_INVALID_INDEX_RANGE \ { KOKKOS_INVALID_OFFSET, KOKKOS_INVALID_OFFSET } -template ::value && - std::is_signed::value, - iType> = 0> +template && std::is_signed_v, + iType> = 0> using IndexRange = Kokkos::Array; using index_list_type = std::initializer_list; @@ -118,11 +118,11 @@ KOKKOS_INLINE_FUNCTION void offsetview_verify_operator_bounds( (enum {LEN = 1024}; char buffer[LEN]; const std::string label = tracker.template get_label(); int n = snprintf(buffer, LEN, - "OffsetView bounds error of view labeled %s (", - label.c_str()); + "OffsetView bounds error of view labeled %s (", + label.c_str()); offsetview_error_operator_bounds<0>(buffer + n, LEN - n, map, begins, args...); - Kokkos::Impl::throw_runtime_exception(std::string(buffer));)) + Kokkos::abort(buffer);)) KOKKOS_IF_ON_DEVICE( (Kokkos::abort("OffsetView bounds error"); (void)tracker;)) @@ -180,44 +180,40 @@ void runtime_check_rank_device(const size_t rank_dynamic, const size_t rank, } // namespace Impl template -class OffsetView : public ViewTraits { - public: - using traits = ViewTraits; - +class OffsetView : public View { private: template friend class OffsetView; - template - friend class View; // FIXME delete this line - template - friend class Kokkos::Impl::ViewMapping; - using map_type = Kokkos::Impl::ViewMapping; - using track_type = Kokkos::Impl::SharedAllocationTracker; + using base_t = View; public: - enum { Rank = map_type::Rank }; - using begins_type = Kokkos::Array; + // typedefs to reduce typing base_t:: further down + using traits = typename base_t::traits; + // FIXME: should be base_t::index_type after refactor + using index_type = typename base_t::memory_space::size_type; + using pointer_type = typename base_t::pointer_type; + + using begins_type = Kokkos::Array; template ::value, iType> = 0> + std::enable_if_t, iType> = 0> KOKKOS_FUNCTION int64_t begin(const iType local_dimension) const { - return local_dimension < Rank ? m_begins[local_dimension] - : KOKKOS_INVALID_OFFSET; + return static_cast(local_dimension) < base_t::rank() + ? m_begins[local_dimension] + : KOKKOS_INVALID_OFFSET; } KOKKOS_FUNCTION begins_type begins() const { return m_begins; } template ::value, iType> = 0> + std::enable_if_t, iType> = 0> KOKKOS_FUNCTION int64_t end(const iType local_dimension) const { - return begin(local_dimension) + m_map.extent(local_dimension); + return begin(local_dimension) + base_t::extent(local_dimension); } private: - track_type m_track; - map_type m_map; begins_type m_begins; public: @@ -245,529 +241,60 @@ class OffsetView : public ViewTraits { typename traits::array_layout, typename traits::host_mirror_space>; - //---------------------------------------- - // Domain rank and extents - - /** \brief rank() to be implemented - */ - // KOKKOS_FUNCTION - // static - // constexpr unsigned rank() { return map_type::Rank; } - - template - KOKKOS_FUNCTION constexpr std::enable_if_t::value, - size_t> - extent(const iType& r) const { - return m_map.extent(r); + template + KOKKOS_FUNCTION typename base_t::reference_type offset_operator( + std::integer_sequence, OtherIndexTypes... indices) const { + return base_t::operator()((indices - m_begins[I])...); } - template - KOKKOS_FUNCTION constexpr std::enable_if_t::value, - int> - extent_int(const iType& r) const { - return static_cast(m_map.extent(r)); - } - - KOKKOS_FUNCTION constexpr typename traits::array_layout layout() const { - return m_map.layout(); - } - - KOKKOS_FUNCTION constexpr size_t size() const { - return m_map.dimension_0() * m_map.dimension_1() * m_map.dimension_2() * - m_map.dimension_3() * m_map.dimension_4() * m_map.dimension_5() * - m_map.dimension_6() * m_map.dimension_7(); - } - - KOKKOS_FUNCTION constexpr size_t stride_0() const { return m_map.stride_0(); } - KOKKOS_FUNCTION constexpr size_t stride_1() const { return m_map.stride_1(); } - KOKKOS_FUNCTION constexpr size_t stride_2() const { return m_map.stride_2(); } - KOKKOS_FUNCTION constexpr size_t stride_3() const { return m_map.stride_3(); } - KOKKOS_FUNCTION constexpr size_t stride_4() const { return m_map.stride_4(); } - KOKKOS_FUNCTION constexpr size_t stride_5() const { return m_map.stride_5(); } - KOKKOS_FUNCTION constexpr size_t stride_6() const { return m_map.stride_6(); } - KOKKOS_FUNCTION constexpr size_t stride_7() const { return m_map.stride_7(); } - - template - KOKKOS_FUNCTION constexpr std::enable_if_t::value, - size_t> - stride(iType r) const { - return ( - r == 0 - ? m_map.stride_0() - : (r == 1 - ? m_map.stride_1() - : (r == 2 - ? m_map.stride_2() - : (r == 3 - ? m_map.stride_3() - : (r == 4 - ? m_map.stride_4() - : (r == 5 - ? m_map.stride_5() - : (r == 6 - ? m_map.stride_6() - : m_map.stride_7()))))))); - } - - template - KOKKOS_FUNCTION void stride(iType* const s) const { - m_map.stride(s); - } - - //---------------------------------------- - // Range span is the span which contains all members. - - using reference_type = typename map_type::reference_type; - using pointer_type = typename map_type::pointer_type; - - enum { - reference_type_is_lvalue_reference = - std::is_lvalue_reference::value - }; - - KOKKOS_FUNCTION constexpr size_t span() const { return m_map.span(); } - KOKKOS_FUNCTION bool span_is_contiguous() const { - return m_map.span_is_contiguous(); - } - KOKKOS_FUNCTION constexpr bool is_allocated() const { - return m_map.data() != nullptr; - } - KOKKOS_FUNCTION constexpr pointer_type data() const { return m_map.data(); } - - //---------------------------------------- - // Allow specializations to query their specialized map - - KOKKOS_FUNCTION - const Kokkos::Impl::ViewMapping& implementation_map() const { - return m_map; - } - - //---------------------------------------- - - private: - static constexpr bool is_layout_left = - std::is_same::value; - - static constexpr bool is_layout_right = - std::is_same::value; - - static constexpr bool is_layout_stride = - std::is_same::value; - - static constexpr bool is_default_map = - std::is_void::value && - (is_layout_left || is_layout_right || is_layout_stride); - -#if defined(KOKKOS_ENABLE_DEBUG_BOUNDS_CHECK) - -#define KOKKOS_IMPL_OFFSETVIEW_OPERATOR_VERIFY(ARG) \ - Kokkos::Impl::runtime_check_memory_access_violation< \ - typename traits::memory_space>( \ - "Kokkos::OffsetView ERROR: attempt to access inaccessible memory " \ - "space"); \ - Kokkos::Experimental::Impl::offsetview_verify_operator_bounds< \ - typename traits::memory_space> \ - ARG; - -#else - -#define KOKKOS_IMPL_OFFSETVIEW_OPERATOR_VERIFY(ARG) \ - Kokkos::Impl::runtime_check_memory_access_violation< \ - typename traits::memory_space>( \ - "Kokkos::OffsetView ERROR: attempt to access inaccessible memory " \ - "space"); - + template +#ifndef KOKKOS_ENABLE_CXX17 + requires(std::is_convertible_v && + std::is_nothrow_constructible_v && + (base_t::rank() == 1)) #endif - public: - //------------------------------ - // Rank 0 operator() - - KOKKOS_FORCEINLINE_FUNCTION - reference_type operator()() const { return m_map.reference(); } - //------------------------------ - // Rank 1 operator() - - template - KOKKOS_FORCEINLINE_FUNCTION std::enable_if_t< - (Kokkos::Impl::are_integral::value && (1 == Rank) && !is_default_map), - reference_type> - operator()(const I0& i0) const { - KOKKOS_IMPL_OFFSETVIEW_OPERATOR_VERIFY((m_track, m_map, m_begins, i0)) - const size_t j0 = i0 - m_begins[0]; - return m_map.reference(j0); - } - - template - KOKKOS_FORCEINLINE_FUNCTION - std::enable_if_t<(Kokkos::Impl::are_integral::value && (1 == Rank) && - is_default_map && !is_layout_stride), - reference_type> - operator()(const I0& i0) const { - KOKKOS_IMPL_OFFSETVIEW_OPERATOR_VERIFY((m_track, m_map, m_begins, i0)) - const size_t j0 = i0 - m_begins[0]; - return m_map.m_impl_handle[j0]; - } - - template - KOKKOS_FORCEINLINE_FUNCTION - std::enable_if_t<(Kokkos::Impl::are_integral::value && (1 == Rank) && - is_default_map && is_layout_stride), - reference_type> - operator()(const I0& i0) const { - KOKKOS_IMPL_OFFSETVIEW_OPERATOR_VERIFY((m_track, m_map, m_begins, i0)) - const size_t j0 = i0 - m_begins[0]; - return m_map.m_impl_handle[m_map.m_impl_offset.m_stride.S0 * j0]; - } - //------------------------------ - // Rank 1 operator[] - - template - KOKKOS_FORCEINLINE_FUNCTION std::enable_if_t< - (Kokkos::Impl::are_integral::value && (1 == Rank) && !is_default_map), - reference_type> - operator[](const I0& i0) const { - KOKKOS_IMPL_OFFSETVIEW_OPERATOR_VERIFY((m_track, m_map, m_begins, i0)) - const size_t j0 = i0 - m_begins[0]; - return m_map.reference(j0); - } - - template - KOKKOS_FORCEINLINE_FUNCTION - std::enable_if_t<(Kokkos::Impl::are_integral::value && (1 == Rank) && - is_default_map && !is_layout_stride), - reference_type> - operator[](const I0& i0) const { - KOKKOS_IMPL_OFFSETVIEW_OPERATOR_VERIFY((m_track, m_map, m_begins, i0)) - const size_t j0 = i0 - m_begins[0]; - return m_map.m_impl_handle[j0]; - } - - template - KOKKOS_FORCEINLINE_FUNCTION - std::enable_if_t<(Kokkos::Impl::are_integral::value && (1 == Rank) && - is_default_map && is_layout_stride), - reference_type> - operator[](const I0& i0) const { - KOKKOS_IMPL_OFFSETVIEW_OPERATOR_VERIFY((m_track, m_map, m_begins, i0)) - const size_t j0 = i0 - m_begins[0]; - return m_map.m_impl_handle[m_map.m_impl_offset.m_stride.S0 * j0]; - } - - //------------------------------ - // Rank 2 - - template - KOKKOS_FORCEINLINE_FUNCTION - std::enable_if_t<(Kokkos::Impl::are_integral::value && - (2 == Rank) && !is_default_map), - reference_type> - operator()(const I0& i0, const I1& i1) const { - KOKKOS_IMPL_OFFSETVIEW_OPERATOR_VERIFY((m_track, m_map, m_begins, i0, i1)) - const size_t j0 = i0 - m_begins[0]; - const size_t j1 = i1 - m_begins[1]; - return m_map.reference(j0, j1); - } - - template - KOKKOS_FORCEINLINE_FUNCTION std::enable_if_t< - (Kokkos::Impl::are_integral::value && (2 == Rank) && - is_default_map && - (is_layout_left || is_layout_right || is_layout_stride)), - reference_type> - operator()(const I0& i0, const I1& i1) const { - KOKKOS_IMPL_OFFSETVIEW_OPERATOR_VERIFY((m_track, m_map, m_begins, i0, i1)) - const size_t j0 = i0 - m_begins[0]; - const size_t j1 = i1 - m_begins[1]; - if constexpr (is_layout_left) { - if constexpr (traits::rank_dynamic == 0) - return m_map.m_impl_handle[j0 + m_map.m_impl_offset.m_dim.N0 * j1]; - else - return m_map.m_impl_handle[j0 + m_map.m_impl_offset.m_stride * j1]; - } else if constexpr (is_layout_right) { - if constexpr (traits::rank_dynamic == 0) - return m_map.m_impl_handle[j1 + m_map.m_impl_offset.m_dim.N1 * j0]; - else - return m_map.m_impl_handle[j1 + m_map.m_impl_offset.m_stride * j0]; - } else { - static_assert(is_layout_stride); - return m_map.m_impl_handle[j0 * m_map.m_impl_offset.m_stride.S0 + - j1 * m_map.m_impl_offset.m_stride.S1]; - } -#if defined(KOKKOS_COMPILER_INTEL) - __builtin_unreachable(); + KOKKOS_FUNCTION constexpr typename base_t::reference_type operator[]( + const OtherIndexType& idx) const { +#ifdef KOKKOS_ENABLE_CXX17 + static_assert(std::is_convertible_v && + std::is_nothrow_constructible_v && + (base_t::rank() == 1)); #endif + return base_t::operator[](idx - m_begins[0]); } - //------------------------------ - // Rank 3 - - template - KOKKOS_FORCEINLINE_FUNCTION - std::enable_if_t<(Kokkos::Impl::are_integral::value && - (3 == Rank) && is_default_map), - reference_type> - operator()(const I0& i0, const I1& i1, const I2& i2) const { - KOKKOS_IMPL_OFFSETVIEW_OPERATOR_VERIFY( - (m_track, m_map, m_begins, i0, i1, i2)) - const size_t j0 = i0 - m_begins[0]; - const size_t j1 = i1 - m_begins[1]; - const size_t j2 = i2 - m_begins[2]; - return m_map.m_impl_handle[m_map.m_impl_offset(j0, j1, j2)]; + template +#ifndef KOKKOS_ENABLE_CXX17 + requires((std::is_convertible_v && ...) && + (std::is_nothrow_constructible_v && + ...) && + (sizeof...(OtherIndexTypes) == base_t::rank())) +#endif + KOKKOS_FUNCTION constexpr typename base_t::reference_type operator()( + OtherIndexTypes... indices) const { +#ifdef KOKKOS_ENABLE_CXX17 + static_assert( + (std::is_convertible_v && ...) && + (std::is_nothrow_constructible_v && ...) && + (sizeof...(OtherIndexTypes) == base_t::rank())); +#endif + return offset_operator(std::make_index_sequence(), + indices...); } - template - KOKKOS_FORCEINLINE_FUNCTION - std::enable_if_t<(Kokkos::Impl::are_integral::value && - (3 == Rank) && !is_default_map), - reference_type> - operator()(const I0& i0, const I1& i1, const I2& i2) const { - KOKKOS_IMPL_OFFSETVIEW_OPERATOR_VERIFY( - (m_track, m_map, m_begins, i0, i1, i2)) - const size_t j0 = i0 - m_begins[0]; - const size_t j1 = i1 - m_begins[1]; - const size_t j2 = i2 - m_begins[2]; - return m_map.reference(j0, j1, j2); - } + template + KOKKOS_FUNCTION constexpr typename base_t::reference_type access( + OtherIndexTypes... args) const = delete; - //------------------------------ - // Rank 4 - - template - KOKKOS_FORCEINLINE_FUNCTION - std::enable_if_t<(Kokkos::Impl::are_integral::value && - (4 == Rank) && is_default_map), - reference_type> - operator()(const I0& i0, const I1& i1, const I2& i2, const I3& i3) const { - KOKKOS_IMPL_OFFSETVIEW_OPERATOR_VERIFY( - (m_track, m_map, m_begins, i0, i1, i2, i3)) - const size_t j0 = i0 - m_begins[0]; - const size_t j1 = i1 - m_begins[1]; - const size_t j2 = i2 - m_begins[2]; - const size_t j3 = i3 - m_begins[3]; - return m_map.m_impl_handle[m_map.m_impl_offset(j0, j1, j2, j3)]; - } - - template - KOKKOS_FORCEINLINE_FUNCTION - std::enable_if_t<(Kokkos::Impl::are_integral::value && - (4 == Rank) && !is_default_map), - reference_type> - operator()(const I0& i0, const I1& i1, const I2& i2, const I3& i3) const { - KOKKOS_IMPL_OFFSETVIEW_OPERATOR_VERIFY( - (m_track, m_map, m_begins, i0, i1, i2, i3)) - const size_t j0 = i0 - m_begins[0]; - const size_t j1 = i1 - m_begins[1]; - const size_t j2 = i2 - m_begins[2]; - const size_t j3 = i3 - m_begins[3]; - return m_map.reference(j0, j1, j2, j3); - } - - //------------------------------ - // Rank 5 - - template - KOKKOS_FORCEINLINE_FUNCTION - std::enable_if_t<(Kokkos::Impl::are_integral::value && - (5 == Rank) && is_default_map), - reference_type> - operator()(const I0& i0, const I1& i1, const I2& i2, const I3& i3, - const I4& i4) const { - KOKKOS_IMPL_OFFSETVIEW_OPERATOR_VERIFY( - (m_track, m_map, m_begins, i0, i1, i2, i3, i4)) - const size_t j0 = i0 - m_begins[0]; - const size_t j1 = i1 - m_begins[1]; - const size_t j2 = i2 - m_begins[2]; - const size_t j3 = i3 - m_begins[3]; - const size_t j4 = i4 - m_begins[4]; - return m_map.m_impl_handle[m_map.m_impl_offset(j0, j1, j2, j3, j4)]; - } - - template - KOKKOS_FORCEINLINE_FUNCTION - std::enable_if_t<(Kokkos::Impl::are_integral::value && - (5 == Rank) && !is_default_map), - reference_type> - operator()(const I0& i0, const I1& i1, const I2& i2, const I3& i3, - const I4& i4) const { - KOKKOS_IMPL_OFFSETVIEW_OPERATOR_VERIFY( - (m_track, m_map, m_begins, i0, i1, i2, i3, i4)) - const size_t j0 = i0 - m_begins[0]; - const size_t j1 = i1 - m_begins[1]; - const size_t j2 = i2 - m_begins[2]; - const size_t j3 = i3 - m_begins[3]; - const size_t j4 = i4 - m_begins[4]; - return m_map.reference(j0, j1, j2, j3, j4); - } - - //------------------------------ - // Rank 6 - - template - KOKKOS_FORCEINLINE_FUNCTION std::enable_if_t< - (Kokkos::Impl::are_integral::value && - (6 == Rank) && is_default_map), - reference_type> - operator()(const I0& i0, const I1& i1, const I2& i2, const I3& i3, - const I4& i4, const I5& i5) const { - KOKKOS_IMPL_OFFSETVIEW_OPERATOR_VERIFY( - (m_track, m_map, m_begins, i0, i1, i2, i3, i4, i5)) - const size_t j0 = i0 - m_begins[0]; - const size_t j1 = i1 - m_begins[1]; - const size_t j2 = i2 - m_begins[2]; - const size_t j3 = i3 - m_begins[3]; - const size_t j4 = i4 - m_begins[4]; - const size_t j5 = i5 - m_begins[5]; - return m_map.m_impl_handle[m_map.m_impl_offset(j0, j1, j2, j3, j4, j5)]; - } - - template - KOKKOS_FORCEINLINE_FUNCTION std::enable_if_t< - (Kokkos::Impl::are_integral::value && - (6 == Rank) && !is_default_map), - reference_type> - operator()(const I0& i0, const I1& i1, const I2& i2, const I3& i3, - const I4& i4, const I5& i5) const { - KOKKOS_IMPL_OFFSETVIEW_OPERATOR_VERIFY( - (m_track, m_map, m_begins, i0, i1, i2, i3, i4, i5)) - const size_t j0 = i0 - m_begins[0]; - const size_t j1 = i1 - m_begins[1]; - const size_t j2 = i2 - m_begins[2]; - const size_t j3 = i3 - m_begins[3]; - const size_t j4 = i4 - m_begins[4]; - const size_t j5 = i5 - m_begins[5]; - return m_map.reference(j0, j1, j2, j3, j4, j5); - } - - //------------------------------ - // Rank 7 - - template - KOKKOS_FORCEINLINE_FUNCTION std::enable_if_t< - (Kokkos::Impl::are_integral::value && - (7 == Rank) && is_default_map), - reference_type> - operator()(const I0& i0, const I1& i1, const I2& i2, const I3& i3, - const I4& i4, const I5& i5, const I6& i6) const { - KOKKOS_IMPL_OFFSETVIEW_OPERATOR_VERIFY( - (m_track, m_map, m_begins, i0, i1, i2, i3, i4, i5, i6)) - const size_t j0 = i0 - m_begins[0]; - const size_t j1 = i1 - m_begins[1]; - const size_t j2 = i2 - m_begins[2]; - const size_t j3 = i3 - m_begins[3]; - const size_t j4 = i4 - m_begins[4]; - const size_t j5 = i5 - m_begins[5]; - const size_t j6 = i6 - m_begins[6]; - return m_map.m_impl_handle[m_map.m_impl_offset(j0, j1, j2, j3, j4, j5, j6)]; - } - - template - KOKKOS_FORCEINLINE_FUNCTION std::enable_if_t< - (Kokkos::Impl::are_integral::value && - (7 == Rank) && !is_default_map), - reference_type> - operator()(const I0& i0, const I1& i1, const I2& i2, const I3& i3, - const I4& i4, const I5& i5, const I6& i6) const { - KOKKOS_IMPL_OFFSETVIEW_OPERATOR_VERIFY( - (m_track, m_map, m_begins, i0, i1, i2, i3, i4, i5, i6)) - const size_t j0 = i0 - m_begins[0]; - const size_t j1 = i1 - m_begins[1]; - const size_t j2 = i2 - m_begins[2]; - const size_t j3 = i3 - m_begins[3]; - const size_t j4 = i4 - m_begins[4]; - const size_t j5 = i5 - m_begins[5]; - const size_t j6 = i6 - m_begins[6]; - return m_map.reference(j0, j1, j2, j3, j4, j5, j6); - } - - //------------------------------ - // Rank 8 - - template - KOKKOS_FORCEINLINE_FUNCTION std::enable_if_t< - (Kokkos::Impl::are_integral::value && - (8 == Rank) && is_default_map), - reference_type> - operator()(const I0& i0, const I1& i1, const I2& i2, const I3& i3, - const I4& i4, const I5& i5, const I6& i6, const I7& i7) const { - KOKKOS_IMPL_OFFSETVIEW_OPERATOR_VERIFY( - (m_track, m_map, m_begins, i0, i1, i2, i3, i4, i5, i6, i7)) - const size_t j0 = i0 - m_begins[0]; - const size_t j1 = i1 - m_begins[1]; - const size_t j2 = i2 - m_begins[2]; - const size_t j3 = i3 - m_begins[3]; - const size_t j4 = i4 - m_begins[4]; - const size_t j5 = i5 - m_begins[5]; - const size_t j6 = i6 - m_begins[6]; - const size_t j7 = i7 - m_begins[7]; - return m_map - .m_impl_handle[m_map.m_impl_offset(j0, j1, j2, j3, j4, j5, j6, j7)]; - } - - template - KOKKOS_FORCEINLINE_FUNCTION std::enable_if_t< - (Kokkos::Impl::are_integral::value && - (8 == Rank) && !is_default_map), - reference_type> - operator()(const I0& i0, const I1& i1, const I2& i2, const I3& i3, - const I4& i4, const I5& i5, const I6& i6, const I7& i7) const { - KOKKOS_IMPL_OFFSETVIEW_OPERATOR_VERIFY( - (m_track, m_map, m_begins, i0, i1, i2, i3, i4, i5, i6, i7)) - const size_t j0 = i0 - m_begins[0]; - const size_t j1 = i1 - m_begins[1]; - const size_t j2 = i2 - m_begins[2]; - const size_t j3 = i3 - m_begins[3]; - const size_t j4 = i4 - m_begins[4]; - const size_t j5 = i5 - m_begins[5]; - const size_t j6 = i6 - m_begins[6]; - const size_t j7 = i7 - m_begins[7]; - return m_map.reference(j0, j1, j2, j3, j4, j5, j6, j7); - } - -#undef KOKKOS_IMPL_OFFSETVIEW_OPERATOR_VERIFY + //---------------------------------------- //---------------------------------------- // Standard destructor, constructors, and assignment operators - KOKKOS_DEFAULTED_FUNCTION - ~OffsetView() = default; - KOKKOS_FUNCTION - OffsetView() : m_track(), m_map() { - for (size_t i = 0; i < Rank; ++i) m_begins[i] = KOKKOS_INVALID_OFFSET; - } - - KOKKOS_FUNCTION - OffsetView(const OffsetView& rhs) - : m_track(rhs.m_track, traits::is_managed), - m_map(rhs.m_map), - m_begins(rhs.m_begins) {} - - KOKKOS_FUNCTION - OffsetView(OffsetView&& rhs) - : m_track(std::move(rhs.m_track)), - m_map(std::move(rhs.m_map)), - m_begins(std::move(rhs.m_begins)) {} - - KOKKOS_FUNCTION - OffsetView& operator=(const OffsetView& rhs) { - m_track = rhs.m_track; - m_map = rhs.m_map; - m_begins = rhs.m_begins; - return *this; - } - - KOKKOS_FUNCTION - OffsetView& operator=(OffsetView&& rhs) { - m_track = std::move(rhs.m_track); - m_map = std::move(rhs.m_map); - m_begins = std::move(rhs.m_begins); - return *this; + OffsetView() : base_t() { + for (size_t i = 0; i < base_t::rank(); ++i) + m_begins[i] = KOKKOS_INVALID_OFFSET; } // interoperability with View @@ -778,20 +305,10 @@ class OffsetView : public ViewTraits { public: KOKKOS_FUNCTION - view_type view() const { - view_type v(m_track, m_map); - return v; - } + view_type view() const { return *this; } template - KOKKOS_FUNCTION OffsetView(const View& aview) - : m_track(aview.impl_track()), m_map() { - using SrcTraits = typename OffsetView::traits; - using Mapping = Kokkos::Impl::ViewMapping; - static_assert(Mapping::is_assignable, - "Incompatible OffsetView copy construction"); - Mapping::assign(m_map, aview.impl_map(), m_track); - + KOKKOS_FUNCTION OffsetView(const View& aview) : base_t(aview) { for (size_t i = 0; i < View::rank(); ++i) { m_begins[i] = 0; } @@ -800,19 +317,14 @@ class OffsetView : public ViewTraits { template KOKKOS_FUNCTION OffsetView(const View& aview, const index_list_type& minIndices) - : m_track(aview.impl_track()), m_map() { - using SrcTraits = typename OffsetView::traits; - using Mapping = Kokkos::Impl::ViewMapping; - static_assert(Mapping::is_assignable, - "Incompatible OffsetView copy construction"); - Mapping::assign(m_map, aview.impl_map(), m_track); - - KOKKOS_IF_ON_HOST((Kokkos::Experimental::Impl::runtime_check_rank_host( - traits::rank_dynamic, Rank, minIndices, label());)) - - KOKKOS_IF_ON_DEVICE((Kokkos::Experimental::Impl::runtime_check_rank_device( - traits::rank_dynamic, Rank, minIndices);)) + : base_t(aview) { + KOKKOS_IF_ON_HOST( + (Kokkos::Experimental::Impl::runtime_check_rank_host( + traits::rank_dynamic, base_t::rank(), minIndices, aview.label());)) + KOKKOS_IF_ON_DEVICE( + (Kokkos::Experimental::Impl::runtime_check_rank_device( + traits::rank_dynamic, base_t::rank(), minIndices);)) for (size_t i = 0; i < minIndices.size(); ++i) { m_begins[i] = minIndices.begin()[i]; } @@ -820,27 +332,13 @@ class OffsetView : public ViewTraits { template KOKKOS_FUNCTION OffsetView(const View& aview, const begins_type& beg) - : m_track(aview.impl_track()), m_map(), m_begins(beg) { - using SrcTraits = typename OffsetView::traits; - using Mapping = Kokkos::Impl::ViewMapping; - static_assert(Mapping::is_assignable, - "Incompatible OffsetView copy construction"); - Mapping::assign(m_map, aview.impl_map(), m_track); - } + : base_t(aview), m_begins(beg) {} // may assign unmanaged from managed. template KOKKOS_FUNCTION OffsetView(const OffsetView& rhs) - : m_track(rhs.m_track, traits::is_managed), - m_map(), - m_begins(rhs.m_begins) { - using SrcTraits = typename OffsetView::traits; - using Mapping = Kokkos::Impl::ViewMapping; - static_assert(Mapping::is_assignable, - "Incompatible OffsetView copy construction"); - Mapping::assign(m_map, rhs.m_map, rhs.m_track); // swb what about assign? - } + : base_t(rhs.view()), m_begins(rhs.m_begins) {} private: enum class subtraction_failure { @@ -879,7 +377,7 @@ class OffsetView : public ViewTraits { static subtraction_failure runtime_check_begins_ends_host(const B& begins, const E& ends) { std::string message; - if (begins.size() != Rank) + if (begins.size() != base_t::rank()) message += "begins.size() " "(" + @@ -887,19 +385,19 @@ class OffsetView : public ViewTraits { ")" " != Rank " "(" + - std::to_string(Rank) + + std::to_string(base_t::rank()) + ")" "\n"; - if (ends.size() != Rank) + if (ends.size() != base_t::rank()) message += "ends.size() " "(" + - std::to_string(begins.size()) + + std::to_string(ends.size()) + ")" " != Rank " "(" + - std::to_string(Rank) + + std::to_string(base_t::rank()) + ")" "\n"; @@ -941,7 +439,7 @@ class OffsetView : public ViewTraits { message = "Kokkos::Experimental::OffsetView ERROR: for unmanaged OffsetView\n" + message; - Kokkos::Impl::throw_runtime_exception(message); + Kokkos::abort(message.c_str()); } return subtraction_failure::none; @@ -951,11 +449,11 @@ class OffsetView : public ViewTraits { template KOKKOS_FUNCTION static subtraction_failure runtime_check_begins_ends_device( const B& begins, const E& ends) { - if (begins.size() != Rank) + if (begins.size() != base_t::rank()) Kokkos::abort( "Kokkos::Experimental::OffsetView ERROR: for unmanaged " "OffsetView: begins has bad Rank"); - if (ends.size() != Rank) + if (ends.size() != base_t::rank()) Kokkos::abort( "Kokkos::Experimental::OffsetView ERROR: for unmanaged " "OffsetView: ends has bad Rank"); @@ -993,20 +491,25 @@ class OffsetView : public ViewTraits { // Precondition: begins.size() == ends.size() == m_begins.size() == Rank template KOKKOS_FUNCTION OffsetView(const pointer_type& p, const B& begins_, - const E& ends_, - subtraction_failure) - : m_track() // no tracking - , - m_map(Kokkos::Impl::ViewCtorProp(p), - typename traits::array_layout( - Rank > 0 ? at(ends_, 0) - at(begins_, 0) : 0, - Rank > 1 ? at(ends_, 1) - at(begins_, 1) : 0, - Rank > 2 ? at(ends_, 2) - at(begins_, 2) : 0, - Rank > 3 ? at(ends_, 3) - at(begins_, 3) : 0, - Rank > 4 ? at(ends_, 4) - at(begins_, 4) : 0, - Rank > 5 ? at(ends_, 5) - at(begins_, 5) : 0, - Rank > 6 ? at(ends_, 6) - at(begins_, 6) : 0, - Rank > 7 ? at(ends_, 7) - at(begins_, 7) : 0)) { + const E& ends_, subtraction_failure) + : base_t(Kokkos::view_wrap(p), + typename traits::array_layout( + base_t::rank() > 0 ? at(ends_, 0) - at(begins_, 0) + : KOKKOS_IMPL_CTOR_DEFAULT_ARG, + base_t::rank() > 1 ? at(ends_, 1) - at(begins_, 1) + : KOKKOS_IMPL_CTOR_DEFAULT_ARG, + base_t::rank() > 2 ? at(ends_, 2) - at(begins_, 2) + : KOKKOS_IMPL_CTOR_DEFAULT_ARG, + base_t::rank() > 3 ? at(ends_, 3) - at(begins_, 3) + : KOKKOS_IMPL_CTOR_DEFAULT_ARG, + base_t::rank() > 4 ? at(ends_, 4) - at(begins_, 4) + : KOKKOS_IMPL_CTOR_DEFAULT_ARG, + base_t::rank() > 5 ? at(ends_, 5) - at(begins_, 5) + : KOKKOS_IMPL_CTOR_DEFAULT_ARG, + base_t::rank() > 6 ? at(ends_, 6) - at(begins_, 6) + : KOKKOS_IMPL_CTOR_DEFAULT_ARG, + base_t::rank() > 7 ? at(ends_, 7) - at(begins_, 7) + : KOKKOS_IMPL_CTOR_DEFAULT_ARG)) { for (size_t i = 0; i != m_begins.size(); ++i) { m_begins[i] = at(begins_, i); }; @@ -1040,15 +543,6 @@ class OffsetView : public ViewTraits { : OffsetView(p, begins_, ends_, runtime_check_begins_ends(begins_, ends_)) {} - //---------------------------------------- - // Allocation tracking properties - KOKKOS_FUNCTION - int use_count() const { return m_track.use_count(); } - - const std::string label() const { - return m_track.template get_label(); - } - // Choosing std::pair as type for the arguments allows constructing an // OffsetView using list initialization syntax, e.g., // OffsetView dummy("dummy", {-1, 3}, {-2,2}); @@ -1070,18 +564,34 @@ class OffsetView : public ViewTraits { const std::pair range7 = KOKKOS_INVALID_INDEX_RANGE ) - : OffsetView( - Kokkos::Impl::ViewCtorProp(arg_label), - typename traits::array_layout(range0.second - range0.first + 1, - range1.second - range1.first + 1, - range2.second - range2.first + 1, - range3.second - range3.first + 1, - range4.second - range4.first + 1, - range5.second - range5.first + 1, - range6.second - range6.first + 1, - range7.second - range7.first + 1), - {range0.first, range1.first, range2.first, range3.first, - range4.first, range5.first, range6.first, range7.first}) {} + : OffsetView(Kokkos::Impl::ViewCtorProp(arg_label), + typename traits::array_layout( + range0.first == KOKKOS_INVALID_OFFSET + ? KOKKOS_IMPL_CTOR_DEFAULT_ARG - 1 + : range0.second - range0.first + 1, + range1.first == KOKKOS_INVALID_OFFSET + ? KOKKOS_IMPL_CTOR_DEFAULT_ARG + : range1.second - range1.first + 1, + range2.first == KOKKOS_INVALID_OFFSET + ? KOKKOS_IMPL_CTOR_DEFAULT_ARG + : range2.second - range2.first + 1, + range3.first == KOKKOS_INVALID_OFFSET + ? KOKKOS_IMPL_CTOR_DEFAULT_ARG + : range3.second - range3.first + 1, + range4.first == KOKKOS_INVALID_OFFSET + ? KOKKOS_IMPL_CTOR_DEFAULT_ARG + : range4.second - range4.first + 1, + range5.first == KOKKOS_INVALID_OFFSET + ? KOKKOS_IMPL_CTOR_DEFAULT_ARG + : range5.second - range5.first + 1, + range6.first == KOKKOS_INVALID_OFFSET + ? KOKKOS_IMPL_CTOR_DEFAULT_ARG + : range6.second - range6.first + 1, + range7.first == KOKKOS_INVALID_OFFSET + ? KOKKOS_IMPL_CTOR_DEFAULT_ARG + : range7.second - range7.first + 1), + {range0.first, range1.first, range2.first, range3.first, + range4.first, range5.first, range6.first, range7.first}) {} template explicit OffsetView( @@ -1094,18 +604,34 @@ class OffsetView : public ViewTraits { const std::pair range5 = KOKKOS_INVALID_INDEX_RANGE, const std::pair range6 = KOKKOS_INVALID_INDEX_RANGE, const std::pair range7 = KOKKOS_INVALID_INDEX_RANGE) - : OffsetView( - arg_prop, - typename traits::array_layout(range0.second - range0.first + 1, - range1.second - range1.first + 1, - range2.second - range2.first + 1, - range3.second - range3.first + 1, - range4.second - range4.first + 1, - range5.second - range5.first + 1, - range6.second - range6.first + 1, - range7.second - range7.first + 1), - {range0.first, range1.first, range2.first, range3.first, - range4.first, range5.first, range6.first, range7.first}) {} + : OffsetView(arg_prop, + typename traits::array_layout( + range0.first == KOKKOS_INVALID_OFFSET + ? KOKKOS_IMPL_CTOR_DEFAULT_ARG + : range0.second - range0.first + 1, + range1.first == KOKKOS_INVALID_OFFSET + ? KOKKOS_IMPL_CTOR_DEFAULT_ARG + : range1.second - range1.first + 1, + range2.first == KOKKOS_INVALID_OFFSET + ? KOKKOS_IMPL_CTOR_DEFAULT_ARG + : range2.second - range2.first + 1, + range3.first == KOKKOS_INVALID_OFFSET + ? KOKKOS_IMPL_CTOR_DEFAULT_ARG + : range3.second - range3.first + 1, + range4.first == KOKKOS_INVALID_OFFSET + ? KOKKOS_IMPL_CTOR_DEFAULT_ARG + : range4.second - range4.first + 1, + range5.first == KOKKOS_INVALID_OFFSET + ? KOKKOS_IMPL_CTOR_DEFAULT_ARG + : range5.second - range5.first + 1, + range6.first == KOKKOS_INVALID_OFFSET + ? KOKKOS_IMPL_CTOR_DEFAULT_ARG + : range6.second - range6.first + 1, + range7.first == KOKKOS_INVALID_OFFSET + ? KOKKOS_IMPL_CTOR_DEFAULT_ARG + : range7.second - range7.first + 1), + {range0.first, range1.first, range2.first, range3.first, + range4.first, range5.first, range6.first, range7.first}) {} template explicit KOKKOS_FUNCTION OffsetView( @@ -1113,9 +639,14 @@ class OffsetView : public ViewTraits { std::enable_if_t::has_pointer, typename traits::array_layout> const& arg_layout, const index_list_type minIndices) - : m_track() // No memory tracking - , - m_map(arg_prop, arg_layout) { + : base_t(arg_prop, arg_layout) { + KOKKOS_IF_ON_HOST((Kokkos::Experimental::Impl::runtime_check_rank_host( + traits::rank_dynamic, base_t::rank(), minIndices, + base_t::label());)) + + KOKKOS_IF_ON_DEVICE( + (Kokkos::Experimental::Impl::runtime_check_rank_device( + traits::rank_dynamic, base_t::rank(), minIndices);)) for (size_t i = 0; i < minIndices.size(); ++i) { m_begins[i] = minIndices.begin()[i]; } @@ -1132,42 +663,9 @@ class OffsetView : public ViewTraits { std::enable_if_t::has_pointer, typename traits::array_layout> const& arg_layout, const index_list_type minIndices) - : m_track(), - m_map() - - { - for (size_t i = 0; i < Rank; ++i) m_begins[i] = minIndices.begin()[i]; - - // Copy the input allocation properties with possibly defaulted properties - auto prop_copy = Kokkos::Impl::with_properties_if_unset( - arg_prop, std::string{}, typename traits::device_type::memory_space{}, - typename traits::device_type::execution_space{}); - using alloc_prop = decltype(prop_copy); - - static_assert(traits::is_managed, - "OffsetView allocation constructor requires managed memory"); - - if (alloc_prop::initialize && - !alloc_prop::execution_space::impl_is_initialized()) { - // If initializing view data then - // the execution space must be initialized. - Kokkos::Impl::throw_runtime_exception( - "Constructing OffsetView and initializing data with uninitialized " - "execution space"); - } - - Kokkos::Impl::SharedAllocationRecord<>* record = m_map.allocate_shared( - prop_copy, arg_layout, - Kokkos::Impl::ViewCtorProp::has_execution_space); - - // Setup and initialization complete, start tracking - m_track.assign_allocated_record_to_uninitialized(record); - - KOKKOS_IF_ON_HOST((Kokkos::Experimental::Impl::runtime_check_rank_host( - traits::rank_dynamic, Rank, minIndices, label());)) - - KOKKOS_IF_ON_DEVICE((Kokkos::Experimental::Impl::runtime_check_rank_device( - traits::rank_dynamic, Rank, minIndices);)) + : base_t(arg_prop, arg_layout) { + for (size_t i = 0; i < base_t::rank(); ++i) + m_begins[i] = minIndices.begin()[i]; } }; @@ -1177,7 +675,7 @@ class OffsetView : public ViewTraits { */ template KOKKOS_INLINE_FUNCTION constexpr unsigned rank(const OffsetView& V) { - return V.Rank; + return V.rank(); } // Temporary until added to view //---------------------------------------------------------------------------- @@ -1185,8 +683,8 @@ KOKKOS_INLINE_FUNCTION constexpr unsigned rank(const OffsetView& V) { namespace Impl { template -KOKKOS_INLINE_FUNCTION std::enable_if_t::value, T> -shift_input(const T arg, const int64_t offset) { +KOKKOS_INLINE_FUNCTION std::enable_if_t, T> shift_input( + const T arg, const int64_t offset) { return arg - offset; } @@ -1197,13 +695,13 @@ Kokkos::ALL_t shift_input(const Kokkos::ALL_t arg, const int64_t /*offset*/) { template KOKKOS_INLINE_FUNCTION - std::enable_if_t::value, Kokkos::pair> + std::enable_if_t, Kokkos::pair> shift_input(const Kokkos::pair arg, const int64_t offset) { return Kokkos::make_pair(arg.first - offset, arg.second - offset); } template -inline std::enable_if_t::value, std::pair> -shift_input(const std::pair arg, const int64_t offset) { +inline std::enable_if_t, std::pair> shift_input( + const std::pair arg, const int64_t offset) { return std::make_pair(arg.first - offset, arg.second - offset); } @@ -1212,7 +710,7 @@ KOKKOS_INLINE_FUNCTION void map_arg_to_new_begin( const size_t i, Kokkos::Array& subviewBegins, std::enable_if_t shiftedArg, const Arg arg, const A viewBegins, size_t& counter) { - if (!std::is_integral::value) { + if (!std::is_integral_v) { subviewBegins[counter] = shiftedArg == arg ? viewBegins[i] : 0; counter++; } @@ -1621,7 +1119,7 @@ KOKKOS_INLINE_FUNCTION ViewTraits, Args...>::type>::type subview(const OffsetView& src, Args... args) { static_assert( - OffsetView::Rank == sizeof...(Args), + OffsetView::rank() == sizeof...(Args), "subview requires one argument for each source OffsetView rank"); return Kokkos::Experimental::Impl::subview_offset(src, args...); @@ -1641,12 +1139,12 @@ KOKKOS_INLINE_FUNCTION bool operator==(const OffsetView& lhs, using lhs_traits = ViewTraits; using rhs_traits = ViewTraits; - return std::is_same::value && - std::is_same::value && - std::is_same::value && + return std::is_same_v && + std::is_same_v && + std::is_same_v && unsigned(lhs_traits::rank) == unsigned(rhs_traits::rank) && lhs.data() == rhs.data() && lhs.span() == rhs.span() && lhs.extent(0) == rhs.extent(0) && lhs.extent(1) == rhs.extent(1) && @@ -1672,12 +1170,12 @@ KOKKOS_INLINE_FUNCTION bool operator==(const View& lhs, using lhs_traits = ViewTraits; using rhs_traits = ViewTraits; - return std::is_same::value && - std::is_same::value && - std::is_same::value && + return std::is_same_v && + std::is_same_v && + std::is_same_v && unsigned(lhs_traits::rank) == unsigned(rhs_traits::rank) && lhs.data() == rhs.data() && lhs.span() == rhs.span() && lhs.extent(0) == rhs.extent(0) && lhs.extent(1) == rhs.extent(1) && @@ -1704,11 +1202,11 @@ template inline void deep_copy( const Experimental::OffsetView& dst, typename ViewTraits::const_value_type& value, - std::enable_if_t::specialize, - void>::value>* = nullptr) { + std::enable_if_t::specialize, + void>>* = nullptr) { static_assert( - std::is_same::non_const_value_type, - typename ViewTraits::value_type>::value, + std::is_same_v::non_const_value_type, + typename ViewTraits::value_type>, "deep_copy requires non-const type"); auto dstView = dst.view(); @@ -1719,11 +1217,11 @@ template inline void deep_copy( const Experimental::OffsetView& dst, const Experimental::OffsetView& value, - std::enable_if_t::specialize, - void>::value>* = nullptr) { + std::enable_if_t::specialize, + void>>* = nullptr) { static_assert( - std::is_same::value_type, - typename ViewTraits::non_const_value_type>::value, + std::is_same_v::value_type, + typename ViewTraits::non_const_value_type>, "deep_copy requires matching non-const destination type"); auto dstView = dst.view(); @@ -1733,11 +1231,11 @@ template inline void deep_copy( const Experimental::OffsetView& dst, const View& value, - std::enable_if_t::specialize, - void>::value>* = nullptr) { + std::enable_if_t::specialize, + void>>* = nullptr) { static_assert( - std::is_same::value_type, - typename ViewTraits::non_const_value_type>::value, + std::is_same_v::value_type, + typename ViewTraits::non_const_value_type>, "deep_copy requires matching non-const destination type"); auto dstView = dst.view(); @@ -1748,11 +1246,11 @@ template inline void deep_copy( const View& dst, const Experimental::OffsetView& value, - std::enable_if_t::specialize, - void>::value>* = nullptr) { + std::enable_if_t::specialize, + void>>* = nullptr) { static_assert( - std::is_same::value_type, - typename ViewTraits::non_const_value_type>::value, + std::is_same_v::value_type, + typename ViewTraits::non_const_value_type>, "deep_copy requires matching non-const destination type"); Kokkos::deep_copy(dst, value.view()); @@ -1770,7 +1268,7 @@ struct MirrorOffsetViewType { // Check whether it is the same memory space enum { is_same_memspace = - std::is_same::value + std::is_same_v }; // The array_layout using array_layout = typename src_view_type::array_layout; @@ -1786,27 +1284,6 @@ struct MirrorOffsetViewType { std::conditional_t; }; -template -struct MirrorOffsetType { - // The incoming view_type - using src_view_type = typename Kokkos::Experimental::OffsetView; - // The memory space for the mirror view - using memory_space = typename Space::memory_space; - // Check whether it is the same memory space - enum { - is_same_memspace = - std::is_same::value - }; - // The array_layout - using array_layout = typename src_view_type::array_layout; - // The data type (we probably want it non-const since otherwise we can't even - // deep_copy to it.) - using data_type = typename src_view_type::non_const_data_type; - // The destination view type if it is not the same memory space - using view_type = - Kokkos::Experimental::OffsetView; -}; - } // namespace Impl namespace Impl { @@ -1825,10 +1302,12 @@ inline auto create_mirror(const Kokkos::Experimental::OffsetView& src, auto prop_copy = Impl::with_properties_if_unset( arg_prop, std::string(src.label()).append("_mirror")); - return typename Kokkos::Impl::MirrorOffsetType::view_type( - prop_copy, src.layout(), - {src.begin(0), src.begin(1), src.begin(2), src.begin(3), src.begin(4), - src.begin(5), src.begin(6), src.begin(7)}); + return typename Kokkos::Impl::MirrorOffsetViewType< + Space, T, P...>::dest_view_type(prop_copy, src.layout(), + {src.begin(0), src.begin(1), + src.begin(2), src.begin(3), + src.begin(4), src.begin(5), + src.begin(6), src.begin(7)}); } else { return typename Kokkos::Experimental::OffsetView::HostMirror( Kokkos::create_mirror(arg_prop, src.view()), src.begins()); @@ -1877,9 +1356,9 @@ template ::value && std::is_void_v::specialize>>> -typename Kokkos::Impl::MirrorOffsetType::view_type -create_mirror(Kokkos::Impl::WithoutInitializing_t wi, const Space&, - const Kokkos::Experimental::OffsetView& src) { +inline auto create_mirror( + Kokkos::Impl::WithoutInitializing_t wi, const Space&, + const Kokkos::Experimental::OffsetView& src) { return Impl::create_mirror( src, Kokkos::view_alloc(typename Space::memory_space{}, wi)); } @@ -1905,14 +1384,14 @@ inline auto create_mirror_view( const Kokkos::Experimental::OffsetView& src, [[maybe_unused]] const Impl::ViewCtorProp& arg_prop) { if constexpr (!Impl::ViewCtorProp::has_memory_space) { - if constexpr (std::is_same::memory_space, - typename Kokkos::Experimental::OffsetView< - T, P...>::HostMirror::memory_space>::value && - std::is_same::data_type, - typename Kokkos::Experimental::OffsetView< - T, P...>::HostMirror::data_type>::value) { + if constexpr (std::is_same_v::memory_space, + typename Kokkos::Experimental::OffsetView< + T, P...>::HostMirror::memory_space> && + std::is_same_v::data_type, + typename Kokkos::Experimental::OffsetView< + T, P...>::HostMirror::data_type>) { return typename Kokkos::Experimental::OffsetView::HostMirror(src); } else { diff --git a/lib/kokkos/containers/src/Kokkos_ScatterView.hpp b/lib/kokkos/containers/src/Kokkos_ScatterView.hpp index 9d04cf6acd..52af567c61 100644 --- a/lib/kokkos/containers/src/Kokkos_ScatterView.hpp +++ b/lib/kokkos/containers/src/Kokkos_ScatterView.hpp @@ -184,16 +184,16 @@ struct DefaultContribution -struct DefaultDuplication { +struct DefaultDuplication { using type = Kokkos::Experimental::ScatterNonDuplicated; }; template <> -struct DefaultContribution { using type = Kokkos::Experimental::ScatterAtomic; }; template <> -struct DefaultContribution { using type = Kokkos::Experimental::ScatterAtomic; }; @@ -532,32 +532,56 @@ void args_to_array(size_t* array, int pos, T dim0, Dims... dims) { subview where the index specified is the largest-stride one. */ template struct Slice { - using next = Slice; - using value_type = typename next::value_type; - - static value_type get(V const& src, const size_t i, Args... args) { + using next = Slice; + static auto get(V const& src, const size_t i, Args... args) { return next::get(src, i, Kokkos::ALL, args...); } }; template struct Slice { - using value_type = - typename Kokkos::Impl::ViewMapping::type; - static value_type get(V const& src, const size_t i, Args... args) { + static auto get(V const& src, const size_t i, Args... args) { return Kokkos::subview(src, i, args...); } }; template struct Slice { - using value_type = - typename Kokkos::Impl::ViewMapping::type; - static value_type get(V const& src, const size_t i, Args... args) { + static auto get(V const& src, const size_t i, Args... args) { return Kokkos::subview(src, args..., i); } }; +#ifdef KOKKOS_ENABLE_IMPL_MDSPAN +template +struct Slice { + static auto get(V const& src, const size_t i, Args... args) { + return Kokkos::subview(src, i, args...); + } +}; + +template +struct Slice { + static auto get(V const& src, const size_t i, Args... args) { + return Kokkos::subview(src, args..., i); + } +}; + +template +struct Slice, 1, V, Args...> { + static auto get(V const& src, const size_t i, Args... args) { + return Kokkos::subview(src, i, args...); + } +}; + +template +struct Slice, 1, V, Args...> { + static auto get(V const& src, const size_t i, Args... args) { + return Kokkos::subview(src, args..., i); + } +}; +#endif + template struct ReduceDuplicates; @@ -905,7 +929,7 @@ class ScatterAccess KOKKOS_FORCEINLINE_FUNCTION std::enable_if_t< - view_type::original_view_type::rank == 1 && std::is_integral::value, + std::is_integral_v && view_type::original_view_type::rank == 1, value_type> operator[](Arg arg) const { return view.at(arg); @@ -1028,10 +1052,7 @@ class ScatterView::value_type - subview() const { + auto subview() const { return Kokkos::Impl::Experimental::Slice< Kokkos::LayoutRight, internal_view_type::rank, internal_view_type>::get(internal_view, 0); @@ -1233,8 +1254,8 @@ class ScatterView::value_type - subview() const { + auto subview() const { return Kokkos::Impl::Experimental::Slice< Kokkos::LayoutLeft, internal_view_type::rank, internal_view_type>::get(internal_view, 0); @@ -1460,7 +1478,7 @@ class ScatterAccess KOKKOS_FORCEINLINE_FUNCTION std::enable_if_t< - view_type::original_view_type::rank == 1 && std::is_integral::value, + std::is_integral_v && view_type::original_view_type::rank == 1, value_type> operator[](Arg arg) const { return view.at(thread_id, arg); @@ -1470,9 +1488,9 @@ class ScatterAccess::array_layout, typename ViewTraits::device_type, Op, std::conditional_t< - std::is_void::value, + std::is_void_v, typename Kokkos::Impl::Experimental::DefaultDuplication< typename ViewTraits::execution_space>::type, Duplication>, std::conditional_t< - std::is_void::value, + std::is_void_v, typename Kokkos::Impl::Experimental::DefaultContribution< typename ViewTraits::execution_space, typename std::conditional_t< - std::is_void::value, + std::is_void_v, typename Kokkos::Impl::Experimental::DefaultDuplication< typename ViewTraits::execution_space>::type, Duplication>>::type, diff --git a/lib/kokkos/containers/src/Kokkos_StaticCrsGraph.hpp b/lib/kokkos/containers/src/Kokkos_StaticCrsGraph.hpp index 8ce868cac2..ec1b8905c7 100644 --- a/lib/kokkos/containers/src/Kokkos_StaticCrsGraph.hpp +++ b/lib/kokkos/containers/src/Kokkos_StaticCrsGraph.hpp @@ -190,7 +190,7 @@ struct GraphRowViewConst { const typename GraphType::entries_type& colidx_in, const ordinal_type& stride, const ordinal_type& count, const OffsetType& idx, - const std::enable_if_t::value, int>& = 0) + const std::enable_if_t, int>& = 0) : colidx_(&colidx_in(idx)), stride_(stride), length(count) {} /// \brief Number of entries in the row. diff --git a/lib/kokkos/containers/src/Kokkos_UnorderedMap.hpp b/lib/kokkos/containers/src/Kokkos_UnorderedMap.hpp index c3a8b67df8..4f47051a5c 100644 --- a/lib/kokkos/containers/src/Kokkos_UnorderedMap.hpp +++ b/lib/kokkos/containers/src/Kokkos_UnorderedMap.hpp @@ -34,7 +34,7 @@ #include #include -#include +#include #include @@ -746,7 +746,7 @@ class UnorderedMap { /// 'const value_type' via Cuda texture fetch must return by value. template KOKKOS_FORCEINLINE_FUNCTION std::enable_if_t< - !std::is_void::value, // !is_set + !std::is_void_v, // !is_set std::conditional_t> value_at(size_type i) const { KOKKOS_EXPECTS(i < capacity()); @@ -808,8 +808,8 @@ class UnorderedMap { // Re-allocate the views of the calling UnorderedMap according to src // capacity, and deep copy the src data. template - std::enable_if_t, key_type>::value && - std::is_same, value_type>::value> + std::enable_if_t, key_type> && + std::is_same_v, value_type>> create_copy_view( UnorderedMap const &src) { if (m_hash_lists.data() != src.m_hash_lists.data()) { @@ -821,8 +821,8 @@ class UnorderedMap { // Allocate views of the calling UnorderedMap with the same capacity as the // src. template - std::enable_if_t, key_type>::value && - std::is_same, value_type>::value> + std::enable_if_t, key_type> && + std::is_same_v, value_type>> allocate_view( UnorderedMap const &src) { insertable_map_type tmp; @@ -852,8 +852,8 @@ class UnorderedMap { // Deep copy view data from src. This requires that the src capacity is // identical to the capacity of the calling UnorderedMap. template - std::enable_if_t, key_type>::value && - std::is_same, value_type>::value> + std::enable_if_t, key_type> && + std::is_same_v, value_type>> deep_copy_view( UnorderedMap const &src) { #ifndef KOKKOS_ENABLE_DEPRECATED_CODE_4 diff --git a/lib/kokkos/containers/src/Kokkos_Vector.hpp b/lib/kokkos/containers/src/Kokkos_Vector.hpp index 88109fb0ba..83ccfbf630 100644 --- a/lib/kokkos/containers/src/Kokkos_Vector.hpp +++ b/lib/kokkos/containers/src/Kokkos_Vector.hpp @@ -172,9 +172,8 @@ class KOKKOS_DEPRECATED vector private: template - struct impl_is_input_iterator - : /* TODO replace this */ std::bool_constant< - !std::is_convertible::value> {}; + struct impl_is_input_iterator : /* TODO replace this */ std::bool_constant< + !std::is_convertible_v> {}; public: // TODO: can use detection idiom to generate better error message here later diff --git a/lib/kokkos/containers/unit_tests/CMakeLists.txt b/lib/kokkos/containers/unit_tests/CMakeLists.txt index e69e46bb6a..6255a86c46 100644 --- a/lib/kokkos/containers/unit_tests/CMakeLists.txt +++ b/lib/kokkos/containers/unit_tests/CMakeLists.txt @@ -1,8 +1,7 @@ - -KOKKOS_INCLUDE_DIRECTORIES(${CMAKE_CURRENT_BINARY_DIR}) -KOKKOS_INCLUDE_DIRECTORIES(REQUIRED_DURING_INSTALLATION_TESTING ${CMAKE_CURRENT_SOURCE_DIR}) -KOKKOS_INCLUDE_DIRECTORIES(${CMAKE_CURRENT_SOURCE_DIR}/../src ) -KOKKOS_INCLUDE_DIRECTORIES(${KOKKOS_SOURCE_DIR}/core/unit_test/category_files) +kokkos_include_directories(${CMAKE_CURRENT_BINARY_DIR}) +kokkos_include_directories(REQUIRED_DURING_INSTALLATION_TESTING ${CMAKE_CURRENT_SOURCE_DIR}) +kokkos_include_directories(${CMAKE_CURRENT_SOURCE_DIR}/../src) +kokkos_include_directories(${KOKKOS_SOURCE_DIR}/core/unit_test/category_files) foreach(Tag Threads;Serial;OpenMP;HPX;Cuda;HIP;SYCL) string(TOUPPER ${Tag} DEVICE) @@ -12,57 +11,49 @@ foreach(Tag Threads;Serial;OpenMP;HPX;Cuda;HIP;SYCL) set(UnitTestSources UnitTestMain.cpp) set(dir ${CMAKE_CURRENT_BINARY_DIR}/${dir}) file(MAKE_DIRECTORY ${dir}) - foreach(Name - Bitset - DualView - DynamicView - DynViewAPI_generic - DynViewAPI_rank12345 - DynViewAPI_rank67 - ErrorReporter - OffsetView - ScatterView - StaticCrsGraph - WithoutInitializing - UnorderedMap - Vector - ViewCtorPropEmbeddedDim - ) + foreach( + Name + Bitset + DualView + DynamicView + DynViewAPI_generic + DynViewAPI_rank12345 + DynViewAPI_rank67 + DynRankView_TeamScratch + ErrorReporter + OffsetView + ScatterView + StaticCrsGraph + WithoutInitializing + UnorderedMap + Vector + ViewCtorPropEmbeddedDim + ) if(NOT Kokkos_ENABLE_DEPRECATED_CODE_4 AND Name STREQUAL "Vector") continue() # skip Kokkos::vector test if deprecated code 4 is not enabled endif() # Write to a temporary intermediate file and call configure_file to avoid # updating timestamps triggering unnecessary rebuilds on subsequent cmake runs. set(file ${dir}/Test${Tag}_${Name}.cpp) - file(WRITE ${dir}/dummy.cpp - "#include \n" - "#include \n" - ) + file(WRITE ${dir}/dummy.cpp "#include \n" "#include \n") configure_file(${dir}/dummy.cpp ${file}) list(APPEND UnitTestSources ${file}) endforeach() #fatal error C1128: number of sections exceeded object file format limit: compile with /bigobj if(KOKKOS_ENABLE_CUDA AND WIN32) - LIST(REMOVE_ITEM UnitTestSources ${dir}/TestCuda_DynViewAPI_generic.cpp) + list(REMOVE_ITEM UnitTestSources ${dir}/TestCuda_DynViewAPI_generic.cpp) endif() # FIXME_NVHPC: NVC++-S-0000-Internal compiler error. extractor: bad opc 0 if(KOKKOS_ENABLE_CUDA AND KOKKOS_CXX_COMPILER_ID STREQUAL NVHPC) - LIST(REMOVE_ITEM UnitTestSources ${dir}/TestCuda_WithoutInitializing.cpp) + list(REMOVE_ITEM UnitTestSources ${dir}/TestCuda_WithoutInitializing.cpp) endif() - KOKKOS_ADD_EXECUTABLE_AND_TEST(ContainersUnitTest_${Tag} SOURCES ${UnitTestSources}) + kokkos_add_executable_and_test(ContainersUnitTest_${Tag} SOURCES ${UnitTestSources}) endif() endforeach() -SET(COMPILE_ONLY_SOURCES - TestCreateMirror.cpp - TestDualViewParameterPack.cpp - TestIsViewTrait.cpp -) -KOKKOS_ADD_EXECUTABLE( - ContainersTestCompileOnly - SOURCES - TestCompileMain.cpp - ${COMPILE_ONLY_SOURCES} +set(COMPILE_ONLY_SOURCES TestCreateMirror.cpp TestDualViewParameterPack.cpp TestIsViewTrait.cpp + TestDynRankViewTypedefs.cpp ) +kokkos_add_executable(ContainersTestCompileOnly SOURCES TestCompileMain.cpp ${COMPILE_ONLY_SOURCES}) diff --git a/lib/kokkos/containers/unit_tests/TestBitset.hpp b/lib/kokkos/containers/unit_tests/TestBitset.hpp index 9923453f72..91dc1710e5 100644 --- a/lib/kokkos/containers/unit_tests/TestBitset.hpp +++ b/lib/kokkos/containers/unit_tests/TestBitset.hpp @@ -39,7 +39,7 @@ struct TestBitset { TestBitset(bitset_type const& bitset) : m_bitset(bitset) {} - unsigned testit(unsigned collisions) { + unsigned testit(unsigned long long collisions) { execution_space().fence(); unsigned count = 0; diff --git a/lib/kokkos/containers/unit_tests/TestDualView.hpp b/lib/kokkos/containers/unit_tests/TestDualView.hpp index 2512cb5c49..5d03e6202a 100644 --- a/lib/kokkos/containers/unit_tests/TestDualView.hpp +++ b/lib/kokkos/containers/unit_tests/TestDualView.hpp @@ -71,7 +71,7 @@ struct test_dualview_copy_construction_and_assignment { using SrcViewType = Kokkos::DualView; using DstViewType = - Kokkos::DualView; + Kokkos::DualView; SrcViewType a("A", n, m); @@ -520,58 +520,26 @@ namespace { * that we keep the semantics of UVM DualViews intact. */ // modify if we have other UVM enabled backends -#if defined(KOKKOS_ENABLE_CUDA) || defined(KOKKOS_ENABLE_SYCL) || \ - defined(KOKKOS_ENABLE_HIP) // OR other UVM builds -#define UVM_ENABLED_BUILD -#endif -#ifdef UVM_ENABLED_BUILD -template -struct UVMSpaceFor; -#endif - -#ifdef KOKKOS_ENABLE_CUDA // specific to CUDA -template <> -struct UVMSpaceFor { - using type = Kokkos::CudaUVMSpace; -}; -#endif - -#ifdef KOKKOS_ENABLE_SYCL // specific to SYCL -template <> -struct UVMSpaceFor { - using type = Kokkos::Experimental::SYCLSharedUSMSpace; -}; -#endif - -#ifdef KOKKOS_ENABLE_HIP // specific to HIP -template <> -struct UVMSpaceFor { - using type = Kokkos::HIPManagedSpace; -}; -#endif - -#ifdef UVM_ENABLED_BUILD -template <> -struct UVMSpaceFor { - using type = typename UVMSpaceFor::type; -}; +#ifdef KOKKOS_HAS_SHARED_SPACE +template +using TestSharedSpace = Kokkos::SharedSpace; #else -template -struct UVMSpaceFor { - using type = typename ExecSpace::memory_space; -}; +template +using TestSharedSpace = typename ExecutionSpace::memory_space; #endif using ExecSpace = Kokkos::DefaultExecutionSpace; -using MemSpace = typename UVMSpaceFor::type; +using MemSpace = TestSharedSpace; using DeviceType = Kokkos::Device; using DualViewType = Kokkos::DualView; -using d_device = DeviceType; -using h_device = Kokkos::Device< - Kokkos::DefaultHostExecutionSpace, - typename UVMSpaceFor::type>; +using ConstDualViewType = + Kokkos::DualView; +using d_device = DeviceType; +using h_device = + Kokkos::Device>; TEST(TEST_CATEGORY, dualview_device_correct_kokkos_device) { DualViewType dv("myView", 100); @@ -635,14 +603,69 @@ TEST(TEST_CATEGORY, dualview_template_views_return_correct_executionspace_views) { DualViewType dv("myView", 100); dv.clear_sync_state(); - using hvt = decltype(dv.view()); - using dvt = decltype(dv.view()); + using hvt = decltype(dv.view()); + using dvt = decltype(dv.view()); ASSERT_STREQ(Kokkos::DefaultExecutionSpace::name(), dvt::device_type::execution_space::name()); ASSERT_STREQ(Kokkos::DefaultHostExecutionSpace::name(), hvt::device_type::execution_space::name()); } +TEST(TEST_CATEGORY, + dualview_template_views_return_correct_views_from_const_dual_view) { + DualViewType dv("myView", 100); + ConstDualViewType const_dv = dv; + dv.clear_sync_state(); + ASSERT_EQ(dv.view(), + const_dv.view()); + ASSERT_EQ(dv.view(), + const_dv.view()); +} + +// User-defined types with a View data member, only host-constructible +template +class S { + V v_; + + public: + template + S(std::string label, Extents... extents) : v_(std::move(label), extents...) {} + S() : v_("v", 10) {} +}; + +template +auto initialize_view_of_views() { + Kokkos::DualView dv_v( + Kokkos::view_alloc("myView", Kokkos::SequentialHostInit), 3u); + + V v("v", 2); + V w("w", 2); + dv_v.h_view(0) = v; + dv_v.h_view(1) = w; + + dv_v.modify_host(); + dv_v.sync_device(); + + return dv_v; +} + +TEST(TEST_CATEGORY, dualview_sequential_host_init) { + auto dv_v = initialize_view_of_views>(); + dv_v.resize(Kokkos::view_alloc(Kokkos::SequentialHostInit), 2u); + ASSERT_EQ(dv_v.d_view.size(), 2u); + ASSERT_EQ(dv_v.h_view.size(), 2u); + + initialize_view_of_views>>(); + + Kokkos::DualView dv( + Kokkos::view_alloc("myView", Kokkos::SequentialHostInit), 1u); + dv.resize(Kokkos::view_alloc(Kokkos::SequentialHostInit), 2u); + ASSERT_EQ(dv.d_view.size(), 2u); + ASSERT_EQ(dv.h_view.size(), 2u); + dv.realloc(Kokkos::view_alloc(Kokkos::SequentialHostInit), 3u); + ASSERT_EQ(dv.d_view.size(), 3u); + ASSERT_EQ(dv.h_view.size(), 3u); +} } // anonymous namespace } // namespace Test diff --git a/lib/kokkos/containers/unit_tests/TestDynRankViewTypedefs.cpp b/lib/kokkos/containers/unit_tests/TestDynRankViewTypedefs.cpp new file mode 100644 index 0000000000..95117a22e6 --- /dev/null +++ b/lib/kokkos/containers/unit_tests/TestDynRankViewTypedefs.cpp @@ -0,0 +1,260 @@ +//@HEADER +// ************************************************************************ +// +// Kokkos v. 4.0 +// Copyright (2022) National Technology & Engineering +// Solutions of Sandia, LLC (NTESS). +// +// Under the terms of Contract DE-NA0003525 with NTESS, +// the U.S. Government retains certain rights in this software. +// +// Part of Kokkos, under the Apache License v2.0 with LLVM Exceptions. +// See https://kokkos.org/LICENSE for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//@HEADER + +#include +#include + +namespace { + +// clang-format off +template +struct data_analysis { + using data_type = DataType; + using const_data_type = const DataType; + using runtime_data_type = DataType; + using runtime_const_data_type = const DataType; + using non_const_data_type = std::remove_const_t; +}; + +template +struct data_analysis { + using data_type = typename data_analysis::data_type*; + using const_data_type = typename data_analysis::const_data_type*; + using runtime_data_type = typename data_analysis::runtime_data_type*; + using runtime_const_data_type = typename data_analysis::runtime_const_data_type*; + using non_const_data_type = typename data_analysis::non_const_data_type*; +}; + +template +struct data_analysis { + using data_type = typename data_analysis::data_type[N]; + using const_data_type = typename data_analysis::const_data_type[N]; + using runtime_data_type = typename data_analysis::runtime_data_type*; + using runtime_const_data_type = typename data_analysis::runtime_const_data_type*; + using non_const_data_type = typename data_analysis::non_const_data_type[N]; +}; + +template +constexpr bool test_view_typedefs_impl() { + // ======================== + // inherited from ViewTraits + // ======================== + static_assert(std::is_same_v); + static_assert(std::is_same_v::const_data_type>); + static_assert(std::is_same_v::non_const_data_type>); + + // FIXME: these should be deprecated and for proper testing (I.e. where this is different from data_type) + // we would need ensemble types which use the hidden View dimension facility of View (i.e. which make + // "specialize" not void) + static_assert(std::is_same_v); + static_assert(std::is_same_v::const_data_type>); + static_assert(std::is_same_v::non_const_data_type>); + static_assert(std::is_same_v); + + // FIXME: value_type definition conflicts with mdspan value_type + static_assert(std::is_same_v); + static_assert(std::is_same_v); + static_assert(std::is_same_v>); + + // FIXME: should maybe be deprecated + static_assert(std::is_same_v); + + // FIXME: should be deprecated and is some complicated impl type + // static_assert(!std::is_void_v); + + static_assert(std::is_same_v); + static_assert(std::is_same_v); + static_assert(std::is_same_v>); + static_assert(std::is_same_v); + static_assert(std::is_same_v); + static_assert(std::is_same_v); + + // FIXME: should be deprecated in favor of reference + static_assert(std::is_same_v); + // FIXME: should be deprecated in favor of data_handle_type + static_assert(std::is_same_v); + + // ========================================= + // in Legacy View: some helper View variants + // ========================================= + + // FIXME: in contrast to View, hooks_policy is not propagated + static_assert(std::is_same_v); + static_assert(std::is_same_v>); + static_assert(std::is_same_v>); + static_assert(std::is_same_v>); + static_assert(std::is_same_v>); + +/* FIXME: these don't exist in DynRankView, should they? + using uniform_layout_type = std::conditional_t), + Kokkos::LayoutLeft, Layout>; + + // Uhm uniformtype removes all memorytraits? + static_assert(std::is_same_v>>); + static_assert(std::is_same_v>>); + static_assert(std::is_same_v::runtime_data_type, uniform_layout_type, + typename ViewType::device_type, Kokkos::MemoryTraits<0>>>); + static_assert(std::is_same_v::runtime_const_data_type, uniform_layout_type, + typename ViewType::device_type, Kokkos::MemoryTraits<0>>>); + + using anonymous_device_type = Kokkos::Device; + static_assert(std::is_same_v>>); + static_assert(std::is_same_v>>); + static_assert(std::is_same_v::runtime_data_type, uniform_layout_type, + anonymous_device_type, Kokkos::MemoryTraits<0>>>); + static_assert(std::is_same_v::runtime_const_data_type, uniform_layout_type, + anonymous_device_type, Kokkos::MemoryTraits<0>>>); +*/ + + // ================================== + // mdspan compatibility + // ================================== + + // FIXME: This typedef caused some weird issue with MSVC+NVCC + // static_assert(std::is_same_v); + // FIXME: Not supported yet + // static_assert(std::is_same_v); + // static_assert(std::is_same_v); + // static_assert(std::is_same_v); + + static_assert(std::is_same_v); + // FIXME: should be remove_const_t + static_assert(std::is_same_v); + // FIXME: should be extents_type::index_type + static_assert(std::is_same_v); + static_assert(std::is_same_v); + + // FIXME: should come from accessor_type + static_assert(std::is_same_v); + static_assert(std::is_same_v); + return true; +} + +// Helper function to unpack data type and other args from the View, and pass them on +template +struct ViewParams {}; + +template +constexpr bool test_view_typedefs(ViewParams) { + return test_view_typedefs_impl, Kokkos::ViewTraits, + T, L, S, M, HostMirrorSpace, ValueType, ReferenceType>(); +} + + +constexpr bool is_host_exec = std::is_same_v; + +#if defined(KOKKOS_ENABLE_CUDA_UVM) || defined(KOKKOS_ENABLE_IMPL_CUDA_UNIFIED_MEMORY) || defined(KOKKOS_IMPL_HIP_UNIFIED_MEMORY) +constexpr bool has_unified_mem_space = true; +#else +constexpr bool has_unified_mem_space = false; +#endif + +// The test take explicit template arguments for: LayoutType, Space, MemoryTraits, HostMirrorSpace, ValueType, ReferenceType +// The ViewParams is just a type pack for the View template arguments + +// Kokkos::View +namespace TestInt { + using layout_type = Kokkos::DefaultExecutionSpace::array_layout; + using space = Kokkos::DefaultExecutionSpace; + using memory_traits = Kokkos::MemoryTraits<0>; + // HostMirrorSpace is a mess so: if the default exec is a host exec, that is it + using host_mirror_space = std::conditional_t>>; + static_assert(test_view_typedefs( + ViewParams{})); +} + +// Kokkos::View +namespace TestIntDefaultExecutionSpace { + using layout_type = Kokkos::DefaultExecutionSpace::array_layout; + using space = Kokkos::DefaultExecutionSpace; + using memory_traits = Kokkos::MemoryTraits<0>; + // HostMirrorSpace is a mess so: if the default exec is a host exec, it is HostSpace (note difference from View ...) + using host_mirror_space = std::conditional_t>; + static_assert(test_view_typedefs( + ViewParams{})); +} + +// Kokkos::View +namespace TestFloatPPHostSpace { + using layout_type = Kokkos::LayoutRight; + using space = Kokkos::HostSpace; + using memory_traits = Kokkos::MemoryTraits<0>; + using host_mirror_space = Kokkos::HostSpace; + static_assert(test_view_typedefs( + ViewParams{})); +} + +// Kokkos::View> +namespace TestFloatPPDeviceDefaultHostExecHostSpace { + using layout_type = Kokkos::LayoutRight; + using space = Kokkos::Device; + using memory_traits = Kokkos::MemoryTraits<0>; + using host_mirror_space = Kokkos::HostSpace; + static_assert(test_view_typedefs( + ViewParams>{})); +} + +// Kokkos::View> +namespace TestIntAtomic { + using layout_type = Kokkos::DefaultExecutionSpace::array_layout; + using space = Kokkos::DefaultExecutionSpace; + using memory_traits = Kokkos::MemoryTraits; + // HostMirrorSpace is a mess so: if the default exec is a host exec, that is it + using host_mirror_space = std::conditional_t>>; + static_assert(test_view_typedefs>>>( + ViewParams>{})); +} +// clang-format on +} // namespace diff --git a/lib/kokkos/containers/unit_tests/TestDynRankView_TeamScratch.hpp b/lib/kokkos/containers/unit_tests/TestDynRankView_TeamScratch.hpp new file mode 100644 index 0000000000..e5f8860de7 --- /dev/null +++ b/lib/kokkos/containers/unit_tests/TestDynRankView_TeamScratch.hpp @@ -0,0 +1,72 @@ +//@HEADER +// ************************************************************************ +// +// Kokkos v. 4.0 +// Copyright (2022) National Technology & Engineering +// Solutions of Sandia, LLC (NTESS). +// +// Under the terms of Contract DE-NA0003525 with NTESS, +// the U.S. Government retains certain rights in this software. +// +// Part of Kokkos, under the Apache License v2.0 with LLVM Exceptions. +// See https://kokkos.org/LICENSE for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//@HEADER + +#include + +#include + +namespace { + +void test_dyn_rank_view_team_scratch() { + using execution_space = TEST_EXECSPACE; + using memory_space = execution_space::scratch_memory_space; + using drv_type = Kokkos::DynRankView; + using policy_type = Kokkos::TeamPolicy; + using team_type = policy_type::member_type; + + int N0 = 10, N1 = 4, N2 = 3; + size_t shmem_size = drv_type::shmem_size(N0, N1, N2); + ASSERT_GE(shmem_size, N0 * N1 * N2 * sizeof(int)); + + Kokkos::View> + errors("errors"); + auto policy = policy_type(1, Kokkos::AUTO) + .set_scratch_size(0, Kokkos::PerTeam(shmem_size)); + Kokkos::parallel_for( + policy, KOKKOS_LAMBDA(const team_type& team) { + drv_type scr(team.team_scratch(0), N0, N1, N2); + // Control that the code ran at all + if (scr.rank() != 3) errors() |= 1u; + if (scr.extent_int(0) != N0) errors() |= 2u; + if (scr.extent_int(1) != N1) errors() |= 4u; + if (scr.extent_int(2) != N2) errors() |= 8u; + Kokkos::parallel_for( + Kokkos::TeamThreadMDRange(team, N0, N1, N2), + [=](int i, int j, int k) { scr(i, j, k) = i * 100 + j * 10 + k; }); + team.team_barrier(); + Kokkos::parallel_for(Kokkos::TeamThreadMDRange(team, N0, N1, N2), + [=](int i, int j, int k) { + if (scr(i, j, k) != i * 100 + j * 10 + k) + errors() |= 16u; + }); + errors() |= 256u; + }); + unsigned h_errors = 0; + Kokkos::deep_copy(h_errors, errors); + + ASSERT_EQ((h_errors & 1u), 0u) << "Rank mismatch"; + ASSERT_EQ((h_errors & 2u), 0u) << "extent 0 mismatch"; + ASSERT_EQ((h_errors & 4u), 0u) << "extent 1 mismatch"; + ASSERT_EQ((h_errors & 8u), 0u) << "extent 2 mismatch"; + ASSERT_EQ((h_errors & 16u), 0u) << "data access incorrect"; + ASSERT_EQ(h_errors, 256u); +} + +TEST(TEST_CATEGORY, dyn_rank_view_team_scratch) { + test_dyn_rank_view_team_scratch(); +} + +} // namespace diff --git a/lib/kokkos/containers/unit_tests/TestDynViewAPI.hpp b/lib/kokkos/containers/unit_tests/TestDynViewAPI.hpp index 4ecb6cf25c..930c76c32c 100644 --- a/lib/kokkos/containers/unit_tests/TestDynViewAPI.hpp +++ b/lib/kokkos/containers/unit_tests/TestDynViewAPI.hpp @@ -792,9 +792,8 @@ class TestDynViewAPI { int equal_ptr_h2_d = a_h2.data() == a_d.data() ? 1 : 0; int is_same_memspace = - std::is_same::value - ? 1 - : 0; + std::is_same_v ? 1 + : 0; ASSERT_EQ(equal_ptr_h_h2, 1); ASSERT_EQ(equal_ptr_h_d, is_same_memspace); ASSERT_EQ(equal_ptr_h2_d, is_same_memspace); @@ -817,9 +816,8 @@ class TestDynViewAPI { int equal_ptr_h2_d = a_h2.data() == a_d.data() ? 1 : 0; int is_same_memspace = - std::is_same::value - ? 1 - : 0; + std::is_same_v ? 1 + : 0; ASSERT_EQ(equal_ptr_h_h2, 1); ASSERT_EQ(equal_ptr_h_d, is_same_memspace); ASSERT_EQ(equal_ptr_h2_d, is_same_memspace); @@ -846,9 +844,8 @@ class TestDynViewAPI { int equal_ptr_h2_d = a_h2.data() == a_d.data() ? 1 : 0; int is_same_memspace = - std::is_same::value - ? 1 - : 0; + std::is_same_v ? 1 + : 0; ASSERT_EQ(equal_ptr_h_h2, 1); ASSERT_EQ(equal_ptr_h_d, is_same_memspace); ASSERT_EQ(equal_ptr_h2_d, is_same_memspace); @@ -879,8 +876,7 @@ class TestDynViewAPI { int equal_ptr_h3_d = a_h3.data() == a_d.data() ? 1 : 0; int is_same_memspace = - std::is_same::value + std::is_same_v ? 1 : 0; ASSERT_EQ(equal_ptr_h_h2, 1); @@ -915,8 +911,7 @@ class TestDynViewAPI { int equal_ptr_h3_d = a_h3.data() == a_d.data() ? 1 : 0; int is_same_memspace = - std::is_same::value + std::is_same_v ? 1 : 0; ASSERT_EQ(equal_ptr_h_h2, 1); @@ -943,8 +938,6 @@ class TestDynViewAPI { dView0 d("d"); -#if defined(KOKKOS_ENABLE_CXX11_DISPATCH_LAMBDA) - // Rank 0 Kokkos::resize(d); @@ -1121,8 +1114,6 @@ class TestDynViewAPI { Kokkos::deep_copy(error_flag_host, error_flag); ASSERT_EQ(error_flag_host(), 0); #endif // MDRangePolict Rank < 7 - -#endif // defined(KOKKOS_ENABLE_CXX11_DISPATCH_LAMBDA) } static void run_test_scalar() { diff --git a/lib/kokkos/containers/unit_tests/TestDynamicView.hpp b/lib/kokkos/containers/unit_tests/TestDynamicView.hpp index c8f8fed3b8..94ccea86eb 100644 --- a/lib/kokkos/containers/unit_tests/TestDynamicView.hpp +++ b/lib/kokkos/containers/unit_tests/TestDynamicView.hpp @@ -71,7 +71,6 @@ struct TestDynamicView { da.resize_serial(da_size); ASSERT_EQ(da.size(), da_size); -#if defined(KOKKOS_ENABLE_CXX11_DISPATCH_LAMBDA) Kokkos::parallel_for( Kokkos::RangePolicy(0, da_size), KOKKOS_LAMBDA(const int i) { da(i) = Scalar(i); }); @@ -85,7 +84,6 @@ struct TestDynamicView { result_sum); ASSERT_EQ(result_sum, (value_type)(da_size * (da_size - 1) / 2)); -#endif // add 3x more entries i.e. 4x larger than previous size // the first 1/4 should remain the same @@ -93,7 +91,6 @@ struct TestDynamicView { da.resize_serial(da_resize); ASSERT_EQ(da.size(), da_resize); -#if defined(KOKKOS_ENABLE_CXX11_DISPATCH_LAMBDA) Kokkos::parallel_for( Kokkos::RangePolicy(da_size, da_resize), KOKKOS_LAMBDA(const int i) { da(i) = Scalar(i); }); @@ -108,7 +105,6 @@ struct TestDynamicView { ASSERT_EQ(new_result_sum + result_sum, (value_type)(da_resize * (da_resize - 1) / 2)); -#endif } // end scope // Test: Create DynamicView, initialize size (via resize), run through @@ -123,7 +119,6 @@ struct TestDynamicView { da.resize_serial(da_size); ASSERT_EQ(da.size(), da_size); -#if defined(KOKKOS_ENABLE_CXX11_DISPATCH_LAMBDA) Kokkos::parallel_for( Kokkos::RangePolicy(0, da_size), KOKKOS_LAMBDA(const int i) { da(i) = Scalar(i); }); @@ -137,7 +132,6 @@ struct TestDynamicView { result_sum); ASSERT_EQ(result_sum, (value_type)(da_size * (da_size - 1) / 2)); -#endif // add 3x more entries i.e. 4x larger than previous size // the first 1/4 should remain the same @@ -145,7 +139,6 @@ struct TestDynamicView { da.resize_serial(da_resize); ASSERT_EQ(da.size(), da_resize); -#if defined(KOKKOS_ENABLE_CXX11_DISPATCH_LAMBDA) Kokkos::parallel_for( Kokkos::RangePolicy(da_size, da_resize), KOKKOS_LAMBDA(const int i) { da(i) = Scalar(i); }); @@ -160,7 +153,6 @@ struct TestDynamicView { ASSERT_EQ(new_result_sum + result_sum, (value_type)(da_resize * (da_resize - 1) / 2)); -#endif } // end scope // Test: Create DynamicView, initialize size (via resize), run through @@ -175,7 +167,6 @@ struct TestDynamicView { da.resize_serial(da_size); ASSERT_EQ(da.size(), da_size); -#if defined(KOKKOS_ENABLE_CXX11_DISPATCH_LAMBDA) Kokkos::parallel_for( Kokkos::RangePolicy(0, da_size), KOKKOS_LAMBDA(const int i) { da(i) = Scalar(i); }); @@ -189,14 +180,12 @@ struct TestDynamicView { result_sum); ASSERT_EQ(result_sum, (value_type)(da_size * (da_size - 1) / 2)); -#endif // remove the final 3/4 entries i.e. first 1/4 remain unsigned da_resize = arg_total_size / 8; da.resize_serial(da_resize); ASSERT_EQ(da.size(), da_resize); -#if defined(KOKKOS_ENABLE_CXX11_DISPATCH_LAMBDA) Kokkos::parallel_for( Kokkos::RangePolicy(0, da_resize), KOKKOS_LAMBDA(const int i) { da(i) = Scalar(i); }); @@ -210,7 +199,6 @@ struct TestDynamicView { new_result_sum); ASSERT_EQ(new_result_sum, (value_type)(da_resize * (da_resize - 1) / 2)); -#endif } // end scope // Test: Reproducer to demonstrate compile-time error of deep_copy @@ -229,7 +217,6 @@ struct TestDynamicView { device_dynamic_view.resize_serial(da_size); // Use parallel_for to populate device_dynamic_view and verify values -#if defined(KOKKOS_ENABLE_CXX11_DISPATCH_LAMBDA) Kokkos::parallel_for( Kokkos::RangePolicy(0, da_size), KOKKOS_LAMBDA(const int i) { device_dynamic_view(i) = Scalar(i); }); @@ -243,7 +230,6 @@ struct TestDynamicView { result_sum); ASSERT_EQ(result_sum, (value_type)(da_size * (da_size - 1) / 2)); -#endif // Use an on-device View as intermediate to deep_copy the // device_dynamic_view to host, zero out the device_dynamic_view, @@ -251,13 +237,11 @@ struct TestDynamicView { Kokkos::deep_copy(device_view, device_dynamic_view); Kokkos::deep_copy(host_view, device_view); Kokkos::deep_copy(device_view, host_view); -#if defined(KOKKOS_ENABLE_CXX11_DISPATCH_LAMBDA) Kokkos::parallel_for( Kokkos::RangePolicy(0, da_size), KOKKOS_LAMBDA(const int i) { device_dynamic_view(i) = Scalar(0); }); -#endif Kokkos::deep_copy(device_dynamic_view, device_view); -#if defined(KOKKOS_ENABLE_CXX11_DISPATCH_LAMBDA) + value_type new_result_sum = 0.0; Kokkos::parallel_reduce( Kokkos::RangePolicy(0, da_size), @@ -267,21 +251,6 @@ struct TestDynamicView { new_result_sum); ASSERT_EQ(new_result_sum, (value_type)(da_size * (da_size - 1) / 2)); -#endif - - // Try to deep_copy device_dynamic_view directly to/from host. - // host-to-device currently fails to compile because DP and SP are - // swapped in the deep_copy implementation. - // Once that's fixed, both deep_copy's will fail at runtime because the - // destination execution space cannot access the source memory space. - // Check if the memory spaces are different before testing the deep_copy. - if (!Kokkos::SpaceAccessibility::accessible) { - ASSERT_THROW(Kokkos::deep_copy(host_view, device_dynamic_view), - std::runtime_error); - ASSERT_THROW(Kokkos::deep_copy(device_dynamic_view, host_view), - std::runtime_error); - } } } }; diff --git a/lib/kokkos/containers/unit_tests/TestErrorReporter.hpp b/lib/kokkos/containers/unit_tests/TestErrorReporter.hpp index 0003a29468..4ebab889c7 100644 --- a/lib/kokkos/containers/unit_tests/TestErrorReporter.hpp +++ b/lib/kokkos/containers/unit_tests/TestErrorReporter.hpp @@ -149,7 +149,6 @@ struct ErrorReporterDriver : public ErrorReporterDriverBase { } }; -#if !defined(KOKKOS_ENABLE_CUDA) || defined(KOKKOS_ENABLE_CUDA_LAMBDA) template struct ErrorReporterDriverUseLambda : public ErrorReporterDriverBase { @@ -178,7 +177,6 @@ struct ErrorReporterDriverUseLambda driver_base::check_expectations(reporter_capacity, test_size); } }; -#endif #ifdef KOKKOS_ENABLE_OPENMP struct ErrorReporterDriverNativeOpenMP @@ -205,8 +203,7 @@ struct ErrorReporterDriverNativeOpenMP // FIXME_MSVC MSVC just gets confused when using the base class in the // KOKKOS_CLASS_LAMBDA -#if !defined(KOKKOS_COMPILER_MSVC) && \ - (!defined(KOKKOS_ENABLE_CUDA) || defined(KOKKOS_ENABLE_CUDA_LAMBDA)) +#ifndef KOKKOS_COMPILER_MSVC TEST(TEST_CATEGORY, ErrorReporterViaLambda) { TestErrorReporter>(); } diff --git a/lib/kokkos/containers/unit_tests/TestOffsetView.hpp b/lib/kokkos/containers/unit_tests/TestOffsetView.hpp index c133922e3d..706b40fff3 100644 --- a/lib/kokkos/containers/unit_tests/TestOffsetView.hpp +++ b/lib/kokkos/containers/unit_tests/TestOffsetView.hpp @@ -56,7 +56,18 @@ void test_offsetview_construction() { offset_view_type ov("firstOV", range0, range1); ASSERT_EQ("firstOV", ov.label()); - ASSERT_EQ(2, ov.Rank); + +#ifdef KOKKOS_ENABLE_DEPRECATION_WARNINGS + KOKKOS_IMPL_DISABLE_DEPRECATED_WARNINGS_PUSH() +#endif +#ifdef KOKKOS_ENABLE_DEPRECATED_CODE_4 + ASSERT_EQ(2u, ov.Rank); +#endif +#ifdef KOKKOS_ENABLE_DEPRECATION_WARNINGS + KOKKOS_IMPL_DISABLE_DEPRECATED_WARNINGS_POP() +#endif + + ASSERT_EQ(2u, ov.rank()); ASSERT_EQ(ov.begin(0), -1); ASSERT_EQ(ov.end(0), 4); @@ -67,7 +78,6 @@ void test_offsetview_construction() { ASSERT_EQ(ov.extent(0), 5u); ASSERT_EQ(ov.extent(1), 5u); -#if defined(KOKKOS_ENABLE_CUDA_LAMBDA) || !defined(KOKKOS_ENABLE_CUDA) { Kokkos::Experimental::OffsetView offsetV1("OneDOffsetView", range0); @@ -149,7 +159,6 @@ void test_offsetview_construction() { } ASSERT_EQ(OVResult, answer) << "Bad data found in OffsetView"; -#endif { offset_view_type ovCopy(ov); @@ -184,7 +193,6 @@ void test_offsetview_construction() { range3_type rangePolicy3DZero(point3_type{{0, 0, 0}}, point3_type{{extent0, extent1, extent2}}); -#if defined(KOKKOS_ENABLE_CUDA_LAMBDA) || !defined(KOKKOS_ENABLE_CUDA) int view3DSum = 0; Kokkos::parallel_reduce( rangePolicy3DZero, @@ -207,7 +215,6 @@ void test_offsetview_construction() { ASSERT_EQ(view3DSum, offsetView3DSum) << "construction of OffsetView from View and begins array broken."; -#endif } view_type viewFromOV = ov.view(); @@ -232,7 +239,6 @@ void test_offsetview_construction() { view_type aView("aView", ov.extent(0), ov.extent(1)); Kokkos::deep_copy(aView, ov); -#if defined(KOKKOS_ENABLE_CUDA_LAMBDA) || !defined(KOKKOS_ENABLE_CUDA) int sum = 0; Kokkos::parallel_reduce( rangePolicy2D, @@ -242,7 +248,6 @@ void test_offsetview_construction() { sum); ASSERT_EQ(sum, 0) << "deep_copy(view, offsetView) broken."; -#endif } { // test view to offsetview deep copy @@ -251,7 +256,6 @@ void test_offsetview_construction() { Kokkos::deep_copy(aView, 99); Kokkos::deep_copy(ov, aView); -#if defined(KOKKOS_ENABLE_CUDA_LAMBDA) || !defined(KOKKOS_ENABLE_CUDA) int sum = 0; Kokkos::parallel_reduce( rangePolicy2D, @@ -261,7 +265,6 @@ void test_offsetview_construction() { sum); ASSERT_EQ(sum, 0) << "deep_copy(offsetView, view) broken."; -#endif } } @@ -329,46 +332,131 @@ void test_offsetview_unmanaged_construction() { ASSERT_EQ(bb, ib); ASSERT_EQ(bb, ii); } +} + +template +void test_offsetview_unmanaged_construction_death() { + // Preallocated memory (Only need a valid address for this test) + Scalar s; + + // Regular expression syntax on Windows is a pain. `.` does not match `\n`. + // Feel free to make it work if you have time to spare. +#ifdef _WIN32 +#define SKIP_REGEX_ON_WINDOWS(REGEX) "" +#else +#define SKIP_REGEX_ON_WINDOWS(REGEX) REGEX +#endif { using offset_view_type = Kokkos::Experimental::OffsetView; // Range calculations must be positive - ASSERT_NO_THROW(offset_view_type(&s, {0}, {1})); - ASSERT_NO_THROW(offset_view_type(&s, {0}, {0})); - ASSERT_THROW(offset_view_type(&s, {0}, {-1}), std::runtime_error); + (void)offset_view_type(&s, {0}, {1}); + (void)offset_view_type(&s, {0}, {0}); + ASSERT_DEATH( + offset_view_type(&s, {0}, {-1}), + SKIP_REGEX_ON_WINDOWS( + "Kokkos::Experimental::OffsetView ERROR: for unmanaged OffsetView" + ".*" + "\\(ends\\[0\\] \\(-1\\) - begins\\[0\\] \\(0\\)\\) must be " + "non-negative")); } { using offset_view_type = Kokkos::Experimental::OffsetView; // Range calculations must not overflow - ASSERT_NO_THROW(offset_view_type(&s, {0}, {0x7fffffffffffffffl})); - ASSERT_THROW(offset_view_type(&s, {-1}, {0x7fffffffffffffffl}), - std::runtime_error); - ASSERT_THROW( + (void)offset_view_type(&s, {0}, {0x7fffffffffffffffl}); + ASSERT_DEATH( + offset_view_type(&s, {-1}, {0x7fffffffffffffffl}), + SKIP_REGEX_ON_WINDOWS( + "Kokkos::Experimental::OffsetView ERROR: for unmanaged OffsetView" + ".*" + "\\(ends\\[0\\] \\(9223372036854775807\\) - begins\\[0\\] " + "\\(-1\\)\\) " + "overflows")); + ASSERT_DEATH( offset_view_type(&s, {-0x7fffffffffffffffl - 1}, {0x7fffffffffffffffl}), - std::runtime_error); - ASSERT_THROW(offset_view_type(&s, {-0x7fffffffffffffffl - 1}, {0}), - std::runtime_error); + SKIP_REGEX_ON_WINDOWS( + "Kokkos::Experimental::OffsetView ERROR: for unmanaged OffsetView" + ".*" + "\\(ends\\[0\\] \\(9223372036854775807\\) - begins\\[0\\] " + "\\(-9223372036854775808\\)\\) " + "overflows")); + ASSERT_DEATH( + offset_view_type(&s, {-0x7fffffffffffffffl - 1}, {0}), + SKIP_REGEX_ON_WINDOWS( + "Kokkos::Experimental::OffsetView ERROR: for unmanaged OffsetView" + ".*" + "\\(ends\\[0\\] \\(0\\) - begins\\[0\\] " + "\\(-9223372036854775808\\)\\) " + "overflows")); } { using offset_view_type = Kokkos::Experimental::OffsetView; - // Should throw when the rank of begins and/or ends doesn't match that of - // OffsetView - ASSERT_THROW(offset_view_type(&s, {0}, {1}), std::runtime_error); - ASSERT_THROW(offset_view_type(&s, {0}, {1, 1}), std::runtime_error); - ASSERT_THROW(offset_view_type(&s, {0}, {1, 1, 1}), std::runtime_error); - ASSERT_THROW(offset_view_type(&s, {0, 0}, {1}), std::runtime_error); - ASSERT_NO_THROW(offset_view_type(&s, {0, 0}, {1, 1})); - ASSERT_THROW(offset_view_type(&s, {0, 0}, {1, 1, 1}), std::runtime_error); - ASSERT_THROW(offset_view_type(&s, {0, 0, 0}, {1}), std::runtime_error); - ASSERT_THROW(offset_view_type(&s, {0, 0, 0}, {1, 1}), std::runtime_error); - ASSERT_THROW(offset_view_type(&s, {0, 0, 0}, {1, 1, 1}), - std::runtime_error); + // Should throw when the rank of begins and/or ends doesn't match that + // of OffsetView + ASSERT_DEATH( + offset_view_type(&s, {0}, {1}), + SKIP_REGEX_ON_WINDOWS( + "Kokkos::Experimental::OffsetView ERROR: for unmanaged OffsetView" + ".*" + "begins\\.size\\(\\) \\(1\\) != Rank \\(2\\)" + ".*" + "ends\\.size\\(\\) \\(1\\) != Rank \\(2\\)")); + ASSERT_DEATH( + offset_view_type(&s, {0}, {1, 1}), + SKIP_REGEX_ON_WINDOWS( + "Kokkos::Experimental::OffsetView ERROR: for unmanaged OffsetView" + ".*" + "begins\\.size\\(\\) \\(1\\) != Rank \\(2\\)")); + ASSERT_DEATH( + offset_view_type(&s, {0}, {1, 1, 1}), + SKIP_REGEX_ON_WINDOWS( + "Kokkos::Experimental::OffsetView ERROR: for unmanaged OffsetView" + ".*" + "begins\\.size\\(\\) \\(1\\) != Rank \\(2\\)" + ".*" + "ends\\.size\\(\\) \\(3\\) != Rank \\(2\\)")); + ASSERT_DEATH( + offset_view_type(&s, {0, 0}, {1}), + SKIP_REGEX_ON_WINDOWS( + "Kokkos::Experimental::OffsetView ERROR: for unmanaged OffsetView" + ".*" + "ends\\.size\\(\\) \\(1\\) != Rank \\(2\\)")); + (void)offset_view_type(&s, {0, 0}, {1, 1}); + ASSERT_DEATH( + offset_view_type(&s, {0, 0}, {1, 1, 1}), + SKIP_REGEX_ON_WINDOWS( + "Kokkos::Experimental::OffsetView ERROR: for unmanaged OffsetView" + ".*" + "ends\\.size\\(\\) \\(3\\) != Rank \\(2\\)")); + ASSERT_DEATH( + offset_view_type(&s, {0, 0, 0}, {1}), + SKIP_REGEX_ON_WINDOWS( + "Kokkos::Experimental::OffsetView ERROR: for unmanaged OffsetView" + ".*" + "begins\\.size\\(\\) \\(3\\) != Rank \\(2\\)" + ".*" + "ends\\.size\\(\\) \\(1\\) != Rank \\(2\\)")); + ASSERT_DEATH( + offset_view_type(&s, {0, 0, 0}, {1, 1}), + SKIP_REGEX_ON_WINDOWS( + "Kokkos::Experimental::OffsetView ERROR: for unmanaged OffsetView" + ".*" + "begins\\.size\\(\\) \\(3\\) != Rank \\(2\\)")); + ASSERT_DEATH( + offset_view_type(&s, {0, 0, 0}, {1, 1, 1}), + SKIP_REGEX_ON_WINDOWS( + "Kokkos::Experimental::OffsetView ERROR: for unmanaged OffsetView" + ".*" + "begins\\.size\\(\\) \\(3\\) != Rank \\(2\\)" + ".*" + "ends\\.size\\(\\) \\(3\\) != Rank \\(2\\)")); } +#undef SKIP_REGEX_ON_WINDOWS } template @@ -377,8 +465,8 @@ void test_offsetview_subview() { Kokkos::Experimental::OffsetView sliceMe("offsetToSlice", {-10, 20}); { - auto offsetSubviewa = Kokkos::Experimental::subview(sliceMe, 0); - ASSERT_EQ(offsetSubviewa.Rank, 0) << "subview of offset is broken."; + auto offsetSubview = Kokkos::Experimental::subview(sliceMe, 0); + ASSERT_EQ(offsetSubview.rank(), 0u) << "subview of offset is broken."; } } { // test subview 2 @@ -387,13 +475,13 @@ void test_offsetview_subview() { { auto offsetSubview = Kokkos::Experimental::subview(sliceMe, Kokkos::ALL(), -2); - ASSERT_EQ(offsetSubview.Rank, 1) << "subview of offset is broken."; + ASSERT_EQ(offsetSubview.rank(), 1u) << "subview of offset is broken."; } { auto offsetSubview = Kokkos::Experimental::subview(sliceMe, 0, Kokkos::ALL()); - ASSERT_EQ(offsetSubview.Rank, 1) << "subview of offset is broken."; + ASSERT_EQ(offsetSubview.rank(), 1u) << "subview of offset is broken."; } } @@ -406,30 +494,29 @@ void test_offsetview_subview() { { auto offsetSubview = Kokkos::Experimental::subview(sliceMe, Kokkos::ALL(), Kokkos::ALL(), 0); - ASSERT_EQ(offsetSubview.Rank, 2) << "subview of offset is broken."; + ASSERT_EQ(offsetSubview.rank(), 2u) << "subview of offset is broken."; } { auto offsetSubview = Kokkos::Experimental::subview(sliceMe, Kokkos::ALL(), 0, Kokkos::ALL()); - ASSERT_EQ(offsetSubview.Rank, 2) << "subview of offset is broken."; + ASSERT_EQ(offsetSubview.rank(), 2u) << "subview of offset is broken."; } { auto offsetSubview = Kokkos::Experimental::subview( sliceMe, 0, Kokkos::ALL(), Kokkos::ALL()); - ASSERT_EQ(offsetSubview.Rank, 2) << "subview of offset is broken."; + ASSERT_EQ(offsetSubview.rank(), 2u) << "subview of offset is broken."; } { auto offsetSubview = Kokkos::Experimental::subview( sliceMe, 0, Kokkos::ALL(), Kokkos::make_pair(-30, -21)); - ASSERT_EQ(offsetSubview.Rank, 2) << "subview of offset is broken."; + ASSERT_EQ(offsetSubview.rank(), 2u) << "subview of offset is broken."; ASSERT_EQ(offsetSubview.begin(0), -20); ASSERT_EQ(offsetSubview.end(0), 31); ASSERT_EQ(offsetSubview.begin(1), 0); ASSERT_EQ(offsetSubview.end(1), 9); -#if defined(KOKKOS_ENABLE_CUDA_LAMBDA) || !defined(KOKKOS_ENABLE_CUDA) using range_type = Kokkos::MDRangePolicy, Kokkos::IndexType >; using point_type = typename range_type::point_type; @@ -455,25 +542,24 @@ void test_offsetview_subview() { sum); ASSERT_EQ(sum, 6 * (e0 - b0) * (e1 - b1)); -#endif } // slice 2 { auto offsetSubview = Kokkos::Experimental::subview(sliceMe, Kokkos::ALL(), 0, 0); - ASSERT_EQ(offsetSubview.Rank, 1) << "subview of offset is broken."; + ASSERT_EQ(offsetSubview.rank(), 1u) << "subview of offset is broken."; } { auto offsetSubview = Kokkos::Experimental::subview(sliceMe, 0, 0, Kokkos::ALL()); - ASSERT_EQ(offsetSubview.Rank, 1) << "subview of offset is broken."; + ASSERT_EQ(offsetSubview.rank(), 1u) << "subview of offset is broken."; } { auto offsetSubview = Kokkos::Experimental::subview(sliceMe, 0, Kokkos::ALL(), 0); - ASSERT_EQ(offsetSubview.Rank, 1) << "subview of offset is broken."; + ASSERT_EQ(offsetSubview.rank(), 1u) << "subview of offset is broken."; } } @@ -486,73 +572,72 @@ void test_offsetview_subview() { { auto offsetSubview = Kokkos::Experimental::subview( sliceMe, Kokkos::ALL(), Kokkos::ALL(), Kokkos::ALL(), 0); - ASSERT_EQ(offsetSubview.Rank, 3) << "subview of offset is broken."; + ASSERT_EQ(offsetSubview.rank(), 3u) << "subview of offset is broken."; } { auto offsetSubview = Kokkos::Experimental::subview( sliceMe, Kokkos::ALL(), Kokkos::ALL(), 0, Kokkos::ALL()); - ASSERT_EQ(offsetSubview.Rank, 3) << "subview of offset is broken."; + ASSERT_EQ(offsetSubview.rank(), 3u) << "subview of offset is broken."; } { auto offsetSubview = Kokkos::Experimental::subview( sliceMe, Kokkos::ALL(), 0, Kokkos::ALL(), Kokkos::ALL()); - ASSERT_EQ(offsetSubview.Rank, 3) << "subview of offset is broken."; + ASSERT_EQ(offsetSubview.rank(), 3u) << "subview of offset is broken."; } { auto offsetSubview = Kokkos::Experimental::subview( sliceMe, 0, Kokkos::ALL(), Kokkos::ALL(), Kokkos::ALL()); - ASSERT_EQ(offsetSubview.Rank, 3) << "subview of offset is broken."; + ASSERT_EQ(offsetSubview.rank(), 3u) << "subview of offset is broken."; } // slice 2 auto offsetSubview2a = Kokkos::Experimental::subview(sliceMe, Kokkos::ALL(), Kokkos::ALL(), 0, 0); - ASSERT_EQ(offsetSubview2a.Rank, 2) << "subview of offset is broken."; + ASSERT_EQ(offsetSubview2a.rank(), 2u) << "subview of offset is broken."; { auto offsetSubview2b = Kokkos::Experimental::subview( sliceMe, Kokkos::ALL(), 0, Kokkos::ALL(), 0); - ASSERT_EQ(offsetSubview2b.Rank, 2) << "subview of offset is broken."; + ASSERT_EQ(offsetSubview2b.rank(), 2u) << "subview of offset is broken."; } { auto offsetSubview2b = Kokkos::Experimental::subview( sliceMe, Kokkos::ALL(), 0, 0, Kokkos::ALL()); - ASSERT_EQ(offsetSubview2b.Rank, 2) << "subview of offset is broken."; + ASSERT_EQ(offsetSubview2b.rank(), 2u) << "subview of offset is broken."; } { auto offsetSubview2b = Kokkos::Experimental::subview( sliceMe, 0, Kokkos::ALL(), 0, Kokkos::ALL()); - ASSERT_EQ(offsetSubview2b.Rank, 2) << "subview of offset is broken."; + ASSERT_EQ(offsetSubview2b.rank(), 2u) << "subview of offset is broken."; } { auto offsetSubview2b = Kokkos::Experimental::subview( sliceMe, 0, 0, Kokkos::ALL(), Kokkos::ALL()); - ASSERT_EQ(offsetSubview2b.Rank, 2) << "subview of offset is broken."; + ASSERT_EQ(offsetSubview2b.rank(), 2u) << "subview of offset is broken."; } // slice 3 { auto offsetSubview = Kokkos::Experimental::subview(sliceMe, Kokkos::ALL(), 0, 0, 0); - ASSERT_EQ(offsetSubview.Rank, 1) << "subview of offset is broken."; + ASSERT_EQ(offsetSubview.rank(), 1u) << "subview of offset is broken."; } { auto offsetSubview = Kokkos::Experimental::subview(sliceMe, 0, Kokkos::ALL(), 0, 0); - ASSERT_EQ(offsetSubview.Rank, 1) << "subview of offset is broken."; + ASSERT_EQ(offsetSubview.rank(), 1u) << "subview of offset is broken."; } { auto offsetSubview = Kokkos::Experimental::subview(sliceMe, 0, 0, Kokkos::ALL(), 0); - ASSERT_EQ(offsetSubview.Rank, 1) << "subview of offset is broken."; + ASSERT_EQ(offsetSubview.rank(), 1u) << "subview of offset is broken."; } { auto offsetSubview = Kokkos::Experimental::subview(sliceMe, 0, 0, 0, Kokkos::ALL()); - ASSERT_EQ(offsetSubview.Rank, 1) << "subview of offset is broken."; + ASSERT_EQ(offsetSubview.rank(), 1u) << "subview of offset is broken."; } } } -#if defined(KOKKOS_ENABLE_CUDA_LAMBDA) || !defined(KOKKOS_ENABLE_CUDA) template KOKKOS_INLINE_FUNCTION T std_accumulate(InputIt first, InputIt last, T init, BinaryOperation op) { @@ -586,6 +671,7 @@ void test_offsetview_offsets_rank1() { KOKKOS_LAMBDA(const int ii, int& lerrors) { offset_view_type ov(v, {ii}); lerrors += (ov(3) != element({3 - ii})); + lerrors += (ov[3] != element({3 - ii})); }, errors); @@ -655,7 +741,6 @@ void test_offsetview_offsets_rank3() { ASSERT_EQ(0, errors); } -#endif TEST(TEST_CATEGORY, offsetview_construction) { test_offsetview_construction(); @@ -665,11 +750,15 @@ TEST(TEST_CATEGORY, offsetview_unmanaged_construction) { test_offsetview_unmanaged_construction(); } +TEST(TEST_CATEGORY_DEATH, offsetview_unmanaged_construction) { + ::testing::FLAGS_gtest_death_test_style = "threadsafe"; + test_offsetview_unmanaged_construction_death(); +} + TEST(TEST_CATEGORY, offsetview_subview) { test_offsetview_subview(); } -#if defined(KOKKOS_ENABLE_CUDA_LAMBDA) || !defined(KOKKOS_ENABLE_CUDA) TEST(TEST_CATEGORY, offsetview_offsets_rank1) { test_offsetview_offsets_rank1(); } @@ -681,7 +770,6 @@ TEST(TEST_CATEGORY, offsetview_offsets_rank2) { TEST(TEST_CATEGORY, offsetview_offsets_rank3) { test_offsetview_offsets_rank3(); } -#endif } // namespace Test diff --git a/lib/kokkos/containers/unit_tests/TestScatterView.hpp b/lib/kokkos/containers/unit_tests/TestScatterView.hpp index 733f43122c..72c1afbe96 100644 --- a/lib/kokkos/containers/unit_tests/TestScatterView.hpp +++ b/lib/kokkos/containers/unit_tests/TestScatterView.hpp @@ -33,11 +33,11 @@ struct test_scatter_view_impl_cls { public: using scatter_view_type = - Kokkos::Experimental::ScatterView; - using orig_view_type = Kokkos::View; + using orig_view_type = Kokkos::View; using size_type = typename Kokkos::HostSpace::size_type; @@ -134,11 +134,11 @@ struct test_scatter_view_impl_cls { public: using scatter_view_type = - Kokkos::Experimental::ScatterView; - using orig_view_type = Kokkos::View; + using orig_view_type = Kokkos::View; using size_type = typename Kokkos::HostSpace::size_type; @@ -235,11 +235,11 @@ struct test_scatter_view_impl_cls { public: using scatter_view_type = - Kokkos::Experimental::ScatterView; - using orig_view_type = Kokkos::View; + using orig_view_type = Kokkos::View; using size_type = typename Kokkos::HostSpace::size_type; @@ -335,11 +335,11 @@ struct test_scatter_view_impl_cls { public: using scatter_view_type = - Kokkos::Experimental::ScatterView; - using orig_view_type = Kokkos::View; + using orig_view_type = Kokkos::View; using size_type = typename Kokkos::HostSpace::size_type; @@ -714,7 +714,7 @@ void test_scatter_view(int64_t n) { test_sv_config.run_test(n); } #ifdef KOKKOS_ENABLE_SERIAL - if (!std::is_same::value) { + if (!std::is_same_v) { #endif test_scatter_view_config::value)); - ASSERT_TRUE((std::is_same::value)); - ASSERT_TRUE((std::is_same::value)); - ASSERT_TRUE((std::is_same::value)); + ASSERT_TRUE((std::is_same_v)); + ASSERT_TRUE((std::is_same_v)); + ASSERT_TRUE((std::is_same_v)); + ASSERT_TRUE((std::is_same_v)); } } /* namespace TestStaticCrsGraph */ diff --git a/lib/kokkos/containers/unit_tests/TestUnorderedMap.hpp b/lib/kokkos/containers/unit_tests/TestUnorderedMap.hpp index 4a7e826ecb..fc7435a75e 100644 --- a/lib/kokkos/containers/unit_tests/TestUnorderedMap.hpp +++ b/lib/kokkos/containers/unit_tests/TestUnorderedMap.hpp @@ -460,7 +460,7 @@ struct UnorderedMapInsert { //! Insert multiple values. template - void insert(Args &&... args) const { + void insert(Args &&...args) const { static_assert(sizeof...(Args) > 1, "Prefer the single value version"); constexpr size_t size = sizeof...(Args); Kokkos::Array values{ @@ -534,8 +534,6 @@ TEST(TEST_CATEGORY, UnorderedMap_shallow_copyable_on_device) { ASSERT_EQ(1u, test_map_copy.m_map.size()); } -#if !defined(KOKKOS_ENABLE_CUDA) || \ - (defined(KOKKOS_ENABLE_CUDA) && defined(KOKKOS_ENABLE_CUDA_LAMBDA)) void test_unordered_map_device_capture() { TestMapCopy::map_type map; @@ -549,7 +547,6 @@ void test_unordered_map_device_capture() { TEST(TEST_CATEGORY, UnorderedMap_lambda_capturable) { test_unordered_map_device_capture(); } -#endif /** * @test This test ensures that an @ref UnorderedMap can be built diff --git a/lib/kokkos/containers/unit_tests/TestViewCtorPropEmbeddedDim.hpp b/lib/kokkos/containers/unit_tests/TestViewCtorPropEmbeddedDim.hpp index 0246f11ddf..2edddcce34 100644 --- a/lib/kokkos/containers/unit_tests/TestViewCtorPropEmbeddedDim.hpp +++ b/lib/kokkos/containers/unit_tests/TestViewCtorPropEmbeddedDim.hpp @@ -48,7 +48,7 @@ struct TestViewCtorProp_EmbeddedDim { void operator()(const int i) const { v(i) = i; } }; - static void test_vcpt(const int N0, const int N1) { + static void test_vcpt(const size_t N0, const size_t N1) { // Create two views to test { using VIT = typename TestViewCtorProp_EmbeddedDim::ViewIntType; @@ -78,16 +78,16 @@ struct TestViewCtorProp_EmbeddedDim { HostCVT hcv1 = Kokkos::create_mirror_view(cv1); Kokkos::deep_copy(hcv1, cv1); - ASSERT_EQ((std::is_same::value), true); + ASSERT_EQ((std::is_same_v), true); #if 0 // debug output - for ( int i = 0; i < N0*N1; ++i ) { - printf(" Output check: hcv1(%d) = %lf\n ", i, hcv1(i) ); + for ( size_t i = 0; i < N0*N1; ++i ) { + printf(" Output check: hcv1(%zu) = %lf\n ", i, hcv1(i) ); } printf( " Common value type view: %s \n", typeid( CVT() ).name() ); printf( " Common value type: %s \n", typeid( CommonViewValueType() ).name() ); - if ( std::is_same< CommonViewValueType, double >::value == true ) { + if ( std::is_same_v< CommonViewValueType, double > == true ) { printf("Proper common value_type\n"); } else { @@ -115,7 +115,7 @@ struct TestViewCtorProp_EmbeddedDim { HostCVT hcv1 = Kokkos::create_mirror_view(cv1); Kokkos::deep_copy(hcv1, cv1); - ASSERT_EQ((std::is_same::value), true); + ASSERT_EQ((std::is_same_v), true); } } @@ -148,7 +148,7 @@ struct TestViewCtorProp_EmbeddedDim { HostCVT hcv1 = Kokkos::create_mirror_view(cv1); Kokkos::deep_copy(hcv1, cv1); - ASSERT_EQ((std::is_same::value), true); + ASSERT_EQ((std::is_same_v), true); } { @@ -169,7 +169,7 @@ struct TestViewCtorProp_EmbeddedDim { HostCVT hcv1 = Kokkos::create_mirror_view(cv1); Kokkos::deep_copy(hcv1, cv1); - ASSERT_EQ((std::is_same::value), true); + ASSERT_EQ((std::is_same_v), true); } } diff --git a/lib/kokkos/containers/unit_tests/TestWithoutInitializing.hpp b/lib/kokkos/containers/unit_tests/TestWithoutInitializing.hpp index e8558628dc..2932898554 100644 --- a/lib/kokkos/containers/unit_tests/TestWithoutInitializing.hpp +++ b/lib/kokkos/containers/unit_tests/TestWithoutInitializing.hpp @@ -44,6 +44,12 @@ Kokkos::CudaSpace>) \ GTEST_SKIP() << "skipping since unified memory requires additional " \ "fences"; +#elif defined(KOKKOS_IMPL_HIP_UNIFIED_MEMORY) +#define GTEST_SKIP_IF_UNIFIED_MEMORY_SPACE \ + if constexpr (std::is_same_v) \ + GTEST_SKIP() << "skipping since unified memory requires additional " \ + "fences"; #else #define GTEST_SKIP_IF_UNIFIED_MEMORY_SPACE #endif @@ -51,8 +57,7 @@ TEST(TEST_CATEGORY, resize_realloc_no_init_dualview) { using namespace Kokkos::Test::Tools; listen_tool_events(Config::DisableAll(), Config::EnableKernels()); - Kokkos::DualView bla("bla", 5, 6, 7, - 8); + Kokkos::DualView bla("bla", 5, 6, 7, 8); auto success = validate_absence( [&]() { @@ -82,8 +87,7 @@ TEST(TEST_CATEGORY, resize_realloc_no_alloc_dualview) { using namespace Kokkos::Test::Tools; listen_tool_events(Config::DisableAll(), Config::EnableKernels(), Config::EnableAllocs()); - Kokkos::DualView bla("bla", 8, 7, 6, - 5); + Kokkos::DualView bla("bla", 8, 7, 6, 5); auto success = validate_absence( [&]() { @@ -112,8 +116,7 @@ TEST(TEST_CATEGORY, resize_exec_space_dualview) { using namespace Kokkos::Test::Tools; listen_tool_events(Config::DisableAll(), Config::EnableFences(), Config::EnableKernels()); - Kokkos::DualView bla("bla", 8, 7, 6, - 5); + Kokkos::DualView bla("bla", 8, 7, 6, 5); auto success = validate_absence( [&]() { @@ -245,7 +248,7 @@ TEST(TEST_CATEGORY, realloc_exec_space_dynrankview) { // FIXME_THREADS The Threads backend fences every parallel_for #ifdef KOKKOS_ENABLE_THREADS - if (std::is_same::value) + if (std::is_same_v) GTEST_SKIP() << "skipping since the Threads backend isn't asynchronous"; #endif @@ -280,7 +283,7 @@ TEST(TEST_CATEGORY, resize_realloc_no_init_scatterview) { using namespace Kokkos::Test::Tools; listen_tool_events(Config::DisableAll(), Config::EnableKernels()); Kokkos::Experimental::ScatterView< - int*** * [1][2][3], typename TEST_EXECSPACE::array_layout, TEST_EXECSPACE> + int**** [1][2][3], typename TEST_EXECSPACE::array_layout, TEST_EXECSPACE> bla("bla", 4, 5, 6, 7); auto success = validate_absence( @@ -312,7 +315,7 @@ TEST(TEST_CATEGORY, resize_realloc_no_alloc_scatterview) { listen_tool_events(Config::DisableAll(), Config::EnableKernels(), Config::EnableAllocs()); Kokkos::Experimental::ScatterView< - int*** * [1][2][3], typename TEST_EXECSPACE::array_layout, TEST_EXECSPACE> + int**** [1][2][3], typename TEST_EXECSPACE::array_layout, TEST_EXECSPACE> bla("bla", 7, 6, 5, 4); auto success = validate_absence( @@ -343,7 +346,7 @@ TEST(TEST_CATEGORY, resize_exec_space_scatterview) { listen_tool_events(Config::DisableAll(), Config::EnableFences(), Config::EnableKernels()); Kokkos::Experimental::ScatterView< - int*** * [1][2][3], typename TEST_EXECSPACE::array_layout, TEST_EXECSPACE> + int**** [1][2][3], typename TEST_EXECSPACE::array_layout, TEST_EXECSPACE> bla("bla", 7, 6, 5, 4); auto success = validate_absence( @@ -384,13 +387,12 @@ TEST(TEST_CATEGORY, realloc_exec_space_scatterview) { // FIXME_THREADS The Threads backend fences every parallel_for #ifdef KOKKOS_ENABLE_THREADS - if (std::is_same::value) + if (std::is_same_v) GTEST_SKIP() << "skipping since the Threads backend isn't asynchronous"; #endif #if defined(KOKKOS_ENABLE_HPX) && \ !defined(KOKKOS_ENABLE_IMPL_HPX_ASYNC_DISPATCH) - if (std::is_same::value) + if (std::is_same_v) GTEST_SKIP() << "skipping since the HPX backend always fences with async " "dispatch disabled"; #endif diff --git a/lib/kokkos/core/CMakeLists.txt b/lib/kokkos/core/CMakeLists.txt index 0917928001..21f05f6272 100644 --- a/lib/kokkos/core/CMakeLists.txt +++ b/lib/kokkos/core/CMakeLists.txt @@ -1,22 +1,14 @@ -IF (NOT Kokkos_INSTALL_TESTING) - ADD_SUBDIRECTORY(src) -ENDIF() +if(NOT Kokkos_INSTALL_TESTING) + add_subdirectory(src) +endif() -FUNCTION(KOKKOS_ADD_BENCHMARK_DIRECTORY DIR_NAME) - IF(NOT Kokkos_ENABLE_BENCHMARKS) - RETURN() - ENDIF() +function(KOKKOS_ADD_BENCHMARK_DIRECTORY DIR_NAME) + if(NOT Kokkos_ENABLE_BENCHMARKS) + return() + endif() - IF(KOKKOS_HAS_TRILINOS) - message( - STATUS - "Benchmarks are not supported when building as part of Trilinos" - ) - RETURN() - ENDIF() + add_subdirectory(${DIR_NAME}) +endfunction() - ADD_SUBDIRECTORY(${DIR_NAME}) -ENDFUNCTION() - -KOKKOS_ADD_TEST_DIRECTORIES(unit_test) -KOKKOS_ADD_BENCHMARK_DIRECTORY(perf_test) +kokkos_add_test_directories(unit_test) +kokkos_add_benchmark_directory(perf_test) diff --git a/lib/kokkos/core/perf_test/CMakeLists.txt b/lib/kokkos/core/perf_test/CMakeLists.txt index e0dba03e1e..0cb2c804d3 100644 --- a/lib/kokkos/core/perf_test/CMakeLists.txt +++ b/lib/kokkos/core/perf_test/CMakeLists.txt @@ -1,50 +1,36 @@ # FIXME_OPENMPTARGET - the NVIDIA HPC compiler nvc++ in the OpenMPTarget backend does not pass the perf_tests. # FIXME_OPENACC - temporarily disabled due to unimplemented features -IF ((KOKKOS_ENABLE_OPENMPTARGET OR KOKKOS_ENABLE_OPENACC) AND KOKKOS_CXX_COMPILER_ID STREQUAL NVHPC) - RETURN() -ENDIF() -IF (KOKKOS_ENABLE_OPENACC AND KOKKOS_CXX_COMPILER_ID STREQUAL Clang) - RETURN() -ENDIF() +if((KOKKOS_ENABLE_OPENMPTARGET OR KOKKOS_ENABLE_OPENACC) AND KOKKOS_CXX_COMPILER_ID STREQUAL NVHPC) + return() +endif() +if(KOKKOS_ENABLE_OPENACC AND KOKKOS_CXX_COMPILER_ID STREQUAL Clang) + return() +endif() # all PerformanceTest_* executables are part of regular tests # TODO: finish converting these into benchmarks (in progress) -IF(KOKKOS_ENABLE_TESTS) - IF(KOKKOS_ENABLE_CUDA OR KOKKOS_ENABLE_HIP OR KOKKOS_ENABLE_SYCL) - KOKKOS_ADD_EXECUTABLE ( - PerformanceTest_SharedSpace - SOURCES test_sharedSpace.cpp - ) - ENDIF() +if(KOKKOS_ENABLE_TESTS) + if(KOKKOS_ENABLE_CUDA OR KOKKOS_ENABLE_HIP OR KOKKOS_ENABLE_SYCL) + kokkos_add_executable(PerformanceTest_SharedSpace SOURCES test_sharedSpace.cpp) + endif() - KOKKOS_INCLUDE_DIRECTORIES(REQUIRED_DURING_INSTALLATION_TESTING ${CMAKE_CURRENT_SOURCE_DIR}) + kokkos_include_directories(REQUIRED_DURING_INSTALLATION_TESTING ${CMAKE_CURRENT_SOURCE_DIR}) - IF(NOT Kokkos_ENABLE_OPENMPTARGET) - # FIXME OPENMPTARGET needs tasking - KOKKOS_ADD_EXECUTABLE_AND_TEST( - PerformanceTest_TaskDag - SOURCES test_taskdag.cpp - CATEGORIES PERFORMANCE - ) - ENDIF() -ENDIF() + kokkos_add_executable_and_test(PerformanceTest_TaskDag SOURCES test_taskdag.cpp CATEGORIES PERFORMANCE) +endif() -IF(NOT Kokkos_ENABLE_BENCHMARKS) - RETURN() -ENDIF() - -IF (KOKKOS_HAS_TRILINOS) - message(FATAL_ERROR "Benchmarks are not supported when building as part of Trilinos") -ENDIF() +if(NOT Kokkos_ENABLE_BENCHMARKS) + return() +endif() # Find or download google/benchmark library find_package(benchmark QUIET 1.5.6) -IF(benchmark_FOUND) - MESSAGE(STATUS "Using google benchmark found in ${benchmark_DIR}") -ELSE() +if(benchmark_FOUND) + message(STATUS "Using google benchmark found in ${benchmark_DIR}") +else() message(STATUS "No installed google benchmark found, fetching from GitHub") include(FetchContent) - SET(BENCHMARK_ENABLE_TESTING OFF) + set(BENCHMARK_ENABLE_TESTING OFF) list(APPEND CMAKE_MESSAGE_INDENT "[benchmark] ") FetchContent_Declare( @@ -57,143 +43,93 @@ ELSE() list(POP_BACK CMAKE_MESSAGE_INDENT) # Suppress clang-tidy diagnostics on code that we do not have control over - IF(CMAKE_CXX_CLANG_TIDY) - SET_TARGET_PROPERTIES(benchmark PROPERTIES CXX_CLANG_TIDY "") - ENDIF() + if(CMAKE_CXX_CLANG_TIDY) + set_target_properties(benchmark PROPERTIES CXX_CLANG_TIDY "") + endif() target_compile_options(benchmark PRIVATE -w) target_compile_options(benchmark_main PRIVATE -w) -ENDIF() +endif() +function(KOKKOS_ADD_BENCHMARK NAME) + cmake_parse_arguments(BENCHMARK "" "" "SOURCES" ${ARGN}) + if(DEFINED BENCHMARK_UNPARSED_ARGUMENTS) + message(WARNING "Unexpected arguments when adding a benchmark: " ${BENCHMARK_UNPARSED_ARGUMENTS}) + endif() -FUNCTION(KOKKOS_ADD_BENCHMARK NAME) - CMAKE_PARSE_ARGUMENTS( - BENCHMARK - "" - "" - "SOURCES" - ${ARGN} - ) - IF(DEFINED BENCHMARK_UNPARSED_ARGUMENTS) - MESSAGE( - WARNING - "Unexpected arguments when adding a benchmark: " - ${BENCHMARK_UNPARSED_ARGUMENTS} - ) - ENDIF() + set(BENCHMARK_NAME Kokkos_${NAME}) + list(APPEND BENCHMARK_SOURCES BenchmarkMain.cpp Benchmark_Context.cpp) - SET(BENCHMARK_NAME ${PACKAGE_NAME}_${NAME}) - LIST(APPEND BENCHMARK_SOURCES - BenchmarkMain.cpp - Benchmark_Context.cpp - ) + add_executable(${BENCHMARK_NAME} ${BENCHMARK_SOURCES}) + target_link_libraries(${BENCHMARK_NAME} PRIVATE benchmark::benchmark Kokkos::kokkos impl_git_version) + target_include_directories(${BENCHMARK_NAME} SYSTEM PRIVATE ${benchmark_SOURCE_DIR}/include) - ADD_EXECUTABLE( - ${BENCHMARK_NAME} - ${BENCHMARK_SOURCES} - ) - TARGET_LINK_LIBRARIES( - ${BENCHMARK_NAME} - PRIVATE benchmark::benchmark Kokkos::kokkos impl_git_version - ) - TARGET_INCLUDE_DIRECTORIES( - ${BENCHMARK_NAME} - SYSTEM PRIVATE ${benchmark_SOURCE_DIR}/include - ) + foreach(SOURCE_FILE ${BENCHMARK_SOURCES}) + set_source_files_properties(${SOURCE_FILE} PROPERTIES LANGUAGE ${KOKKOS_COMPILE_LANGUAGE}) + endforeach() - FOREACH(SOURCE_FILE ${BENCHMARK_SOURCES}) - SET_SOURCE_FILES_PROPERTIES( - ${SOURCE_FILE} - PROPERTIES LANGUAGE ${KOKKOS_COMPILE_LANGUAGE} - ) - ENDFOREACH() + string(TIMESTAMP BENCHMARK_TIME "%Y-%m-%d_T%H-%M-%S" UTC) + set(BENCHMARK_ARGS --benchmark_counters_tabular=true --benchmark_out=${BENCHMARK_NAME}_${BENCHMARK_TIME}.json) - STRING(TIMESTAMP BENCHMARK_TIME "%Y-%m-%d_T%H-%M-%S" UTC) - SET( - BENCHMARK_ARGS - --benchmark_counters_tabular=true - --benchmark_out=${BENCHMARK_NAME}_${BENCHMARK_TIME}.json - ) + add_test(NAME ${BENCHMARK_NAME} COMMAND ${BENCHMARK_NAME} ${BENCHMARK_ARGS}) +endfunction() - ADD_TEST( - NAME ${BENCHMARK_NAME} - COMMAND ${BENCHMARK_NAME} ${BENCHMARK_ARGS} - ) -ENDFUNCTION() - -SET( - BENCHMARK_SOURCES - PerfTestGramSchmidt.cpp - PerfTest_CustomReduction.cpp - PerfTest_ExecSpacePartitioning.cpp - PerfTestHexGrad.cpp - PerfTest_MallocFree.cpp - PerfTest_ViewAllocate.cpp - PerfTest_ViewCopy_a123.cpp - PerfTest_ViewCopy_b123.cpp - PerfTest_ViewCopy_c123.cpp - PerfTest_ViewCopy_d123.cpp - PerfTest_ViewCopy_a45.cpp - PerfTest_ViewCopy_b45.cpp - PerfTest_ViewCopy_c45.cpp - PerfTest_ViewCopy_d45.cpp - PerfTest_ViewCopy_a6.cpp - PerfTest_ViewCopy_b6.cpp - PerfTest_ViewCopy_c6.cpp - PerfTest_ViewCopy_d6.cpp - PerfTest_ViewCopy_a7.cpp - PerfTest_ViewCopy_b7.cpp - PerfTest_ViewCopy_c7.cpp - PerfTest_ViewCopy_d7.cpp - PerfTest_ViewCopy_a8.cpp - PerfTest_ViewCopy_b8.cpp - PerfTest_ViewCopy_c8.cpp - PerfTest_ViewCopy_d8.cpp - PerfTest_ViewCopy_Raw.cpp - PerfTest_ViewFill_123.cpp - PerfTest_ViewFill_45.cpp - PerfTest_ViewFill_6.cpp - PerfTest_ViewFill_7.cpp - PerfTest_ViewFill_8.cpp - PerfTest_ViewFill_Raw.cpp - PerfTest_ViewResize_123.cpp - PerfTest_ViewResize_45.cpp - PerfTest_ViewResize_6.cpp - PerfTest_ViewResize_7.cpp - PerfTest_ViewResize_8.cpp - PerfTest_ViewResize_Raw.cpp -) - -IF(Kokkos_ENABLE_OPENMPTARGET) -# FIXME OPENMPTARGET requires TeamPolicy Reductions and Custom Reduction - LIST(REMOVE_ITEM BENCHMARK_SOURCES +set(BENCHMARK_SOURCES PerfTestGramSchmidt.cpp PerfTest_CustomReduction.cpp PerfTest_ExecSpacePartitioning.cpp - ) -ENDIF() - -KOKKOS_ADD_BENCHMARK( - PerformanceTest_Benchmark - SOURCES ${BENCHMARK_SOURCES} + PerfTestHexGrad.cpp + PerfTest_MallocFree.cpp + PerfTest_ViewAllocate.cpp + PerfTest_ViewCopy_a123.cpp + PerfTest_ViewCopy_b123.cpp + PerfTest_ViewCopy_c123.cpp + PerfTest_ViewCopy_d123.cpp + PerfTest_ViewCopy_a45.cpp + PerfTest_ViewCopy_b45.cpp + PerfTest_ViewCopy_c45.cpp + PerfTest_ViewCopy_d45.cpp + PerfTest_ViewCopy_a6.cpp + PerfTest_ViewCopy_b6.cpp + PerfTest_ViewCopy_c6.cpp + PerfTest_ViewCopy_d6.cpp + PerfTest_ViewCopy_a7.cpp + PerfTest_ViewCopy_b7.cpp + PerfTest_ViewCopy_c7.cpp + PerfTest_ViewCopy_d7.cpp + PerfTest_ViewCopy_a8.cpp + PerfTest_ViewCopy_b8.cpp + PerfTest_ViewCopy_c8.cpp + PerfTest_ViewCopy_d8.cpp + PerfTest_ViewCopy_Raw.cpp + PerfTest_ViewFill_123.cpp + PerfTest_ViewFill_45.cpp + PerfTest_ViewFill_6.cpp + PerfTest_ViewFill_7.cpp + PerfTest_ViewFill_8.cpp + PerfTest_ViewFill_Raw.cpp + PerfTest_ViewResize_123.cpp + PerfTest_ViewResize_45.cpp + PerfTest_ViewResize_6.cpp + PerfTest_ViewResize_7.cpp + PerfTest_ViewResize_8.cpp + PerfTest_ViewResize_Raw.cpp ) -IF(NOT KOKKOS_ENABLE_CUDA OR KOKKOS_ENABLE_CUDA_LAMBDA) - KOKKOS_ADD_BENCHMARK( - Benchmark_Atomic_MinMax - SOURCES test_atomic_minmax_simple.cpp +if(Kokkos_ENABLE_OPENMPTARGET) + # FIXME OPENMPTARGET requires TeamPolicy Reductions and Custom Reduction + list(REMOVE_ITEM BENCHMARK_SOURCES PerfTestGramSchmidt.cpp PerfTest_CustomReduction.cpp + PerfTest_ExecSpacePartitioning.cpp ) -ENDIF() +endif() + +kokkos_add_benchmark(PerformanceTest_Benchmark SOURCES ${BENCHMARK_SOURCES}) + +kokkos_add_benchmark(Benchmark_Atomic_MinMax SOURCES test_atomic_minmax_simple.cpp) # FIXME_NVHPC -IF(NOT KOKKOS_CXX_COMPILER_ID STREQUAL NVHPC) - KOKKOS_ADD_BENCHMARK( - PerformanceTest_Mempool - SOURCES test_mempool.cpp - ) -ENDIF() +if(NOT KOKKOS_CXX_COMPILER_ID STREQUAL NVHPC) + kokkos_add_benchmark(PerformanceTest_Mempool SOURCES test_mempool.cpp) +endif() -KOKKOS_ADD_BENCHMARK( - PerformanceTest_Atomic - SOURCES test_atomic.cpp -) +kokkos_add_benchmark(PerformanceTest_Atomic SOURCES test_atomic.cpp) diff --git a/lib/kokkos/core/perf_test/PerfTestHexGrad.cpp b/lib/kokkos/core/perf_test/PerfTestHexGrad.cpp index 98cb246c71..1ebe750f21 100644 --- a/lib/kokkos/core/perf_test/PerfTestHexGrad.cpp +++ b/lib/kokkos/core/perf_test/PerfTestHexGrad.cpp @@ -34,10 +34,10 @@ struct HexGrad { enum { NSpace = 3, NNode = 8 }; using elem_coord_type = - Kokkos::View; + Kokkos::View; using elem_grad_type = - Kokkos::View; + Kokkos::View; elem_coord_type coords; elem_grad_type grad_op; diff --git a/lib/kokkos/core/perf_test/PerfTest_CustomReduction.cpp b/lib/kokkos/core/perf_test/PerfTest_CustomReduction.cpp index 2110f38a91..03340a5d6d 100644 --- a/lib/kokkos/core/perf_test/PerfTest_CustomReduction.cpp +++ b/lib/kokkos/core/perf_test/PerfTest_CustomReduction.cpp @@ -21,7 +21,6 @@ #include #include -#ifdef KOKKOS_ENABLE_CXX11_DISPATCH_LAMBDA namespace Test { template std::pair custom_reduction_test(int N, int R) { @@ -130,4 +129,3 @@ BENCHMARK(CustomReduction) ->UseManualTime(); } // namespace Test -#endif diff --git a/lib/kokkos/core/perf_test/PerfTest_ExecSpacePartitioning.cpp b/lib/kokkos/core/perf_test/PerfTest_ExecSpacePartitioning.cpp index d2a3d0b823..aa23ddbb60 100644 --- a/lib/kokkos/core/perf_test/PerfTest_ExecSpacePartitioning.cpp +++ b/lib/kokkos/core/perf_test/PerfTest_ExecSpacePartitioning.cpp @@ -56,8 +56,7 @@ bool is_overlapping(const Kokkos::HIP&) { #if defined(KOKKOS_ENABLE_SYCL) && defined(KOKKOS_ARCH_INTEL_GPU) template <> -bool is_overlapping( - const Kokkos::Experimental::SYCL&) { +bool is_overlapping(const Kokkos::SYCL&) { return true; } #endif diff --git a/lib/kokkos/core/perf_test/PerfTest_ViewCopy_Raw.cpp b/lib/kokkos/core/perf_test/PerfTest_ViewCopy_Raw.cpp index 67a8d7e555..e4db40e128 100644 --- a/lib/kokkos/core/perf_test/PerfTest_ViewCopy_Raw.cpp +++ b/lib/kokkos/core/perf_test/PerfTest_ViewCopy_Raw.cpp @@ -18,7 +18,6 @@ namespace Test { -#if defined(KOKKOS_ENABLE_CUDA_LAMBDA) || !defined(KOKKOS_ENABLE_CUDA) BENCHMARK(ViewDeepCopy_Raw) ->ArgName("N") ->Arg(10) @@ -38,6 +37,5 @@ BENCHMARK(ViewDeepCopy_Raw) ->ArgName("N") ->Arg(10) ->UseManualTime(); -#endif } // namespace Test diff --git a/lib/kokkos/core/perf_test/PerfTest_ViewFill_Raw.cpp b/lib/kokkos/core/perf_test/PerfTest_ViewFill_Raw.cpp index c11074d915..57bba83a9c 100644 --- a/lib/kokkos/core/perf_test/PerfTest_ViewFill_Raw.cpp +++ b/lib/kokkos/core/perf_test/PerfTest_ViewFill_Raw.cpp @@ -18,7 +18,6 @@ namespace Test { -#if defined(KOKKOS_ENABLE_CUDA_LAMBDA) || !defined(KOKKOS_ENABLE_CUDA) BENCHMARK(ViewFill_Raw) ->ArgName("N") ->Arg(N) @@ -28,6 +27,5 @@ BENCHMARK(ViewFill_Raw) ->ArgName("N") ->Arg(N) ->UseManualTime(); -#endif } // namespace Test diff --git a/lib/kokkos/core/perf_test/PerfTest_ViewResize_Raw.cpp b/lib/kokkos/core/perf_test/PerfTest_ViewResize_Raw.cpp index 2d1bcbb3ca..ab469cb647 100644 --- a/lib/kokkos/core/perf_test/PerfTest_ViewResize_Raw.cpp +++ b/lib/kokkos/core/perf_test/PerfTest_ViewResize_Raw.cpp @@ -18,7 +18,6 @@ namespace Test { -#if defined(KOKKOS_ENABLE_CUDA_LAMBDA) || !defined(KOKKOS_ENABLE_CUDA) BENCHMARK(ViewResize_NoInit_Raw) ->ArgName("N") ->Arg(N) @@ -30,6 +29,5 @@ BENCHMARK(ViewResize_NoInit_Raw) ->Arg(N) ->UseManualTime() ->Iterations(R); -#endif } // namespace Test diff --git a/lib/kokkos/core/perf_test/test_mempool.cpp b/lib/kokkos/core/perf_test/test_mempool.cpp index 9905740afb..bdfe59b0b3 100644 --- a/lib/kokkos/core/perf_test/test_mempool.cpp +++ b/lib/kokkos/core/perf_test/test_mempool.cpp @@ -198,7 +198,7 @@ static void Mempool_Fill(benchmark::State& state) { int fill_level = get_parameter("--fill_level=", state.range(4)); int repeat_inner = get_parameter("--repeat_inner=", state.range(5)); int number_alloc = get_number_alloc(chunk_span, min_superblock_size, - total_alloc_size, fill_level); + total_alloc_size, fill_level); for (auto _ : state) { TestFunctor functor(total_alloc_size, min_superblock_size, number_alloc, @@ -225,7 +225,7 @@ static void Mempool_Alloc_Dealloc(benchmark::State& state) { int fill_level = get_parameter("--fill_level=", state.range(4)); int repeat_inner = get_parameter("--repeat_inner=", state.range(5)); int number_alloc = get_number_alloc(chunk_span, min_superblock_size, - total_alloc_size, fill_level); + total_alloc_size, fill_level); for (auto _ : state) { TestFunctor functor(total_alloc_size, min_superblock_size, number_alloc, diff --git a/lib/kokkos/core/perf_test/test_sharedSpace.cpp b/lib/kokkos/core/perf_test/test_sharedSpace.cpp index 4f140c9409..3c06770e28 100644 --- a/lib/kokkos/core/perf_test/test_sharedSpace.cpp +++ b/lib/kokkos/core/perf_test/test_sharedSpace.cpp @@ -103,7 +103,7 @@ size_t getDeviceMemorySize() { #elif defined KOKKOS_ENABLE_HIP return Kokkos::HIP{}.hip_device_prop().totalGlobalMem; #elif defined KOKKOS_ENABLE_SYCL - auto device = Kokkos::Experimental::SYCL{}.sycl_queue().get_device(); + auto device = Kokkos::SYCL{}.sycl_queue().get_device(); return device.get_info(); #else #error \ diff --git a/lib/kokkos/core/perf_test/test_taskdag.cpp b/lib/kokkos/core/perf_test/test_taskdag.cpp index fccaab64dd..347d9748b5 100644 --- a/lib/kokkos/core/perf_test/test_taskdag.cpp +++ b/lib/kokkos/core/perf_test/test_taskdag.cpp @@ -32,6 +32,11 @@ int main() { return 0; } #include +#ifdef KOKKOS_ENABLE_DEPRECATION_WARNINGS +// We allow using deprecated classes in this file +KOKKOS_IMPL_DISABLE_DEPRECATED_WARNINGS_PUSH() +#endif + using ExecSpace = Kokkos::DefaultExecutionSpace; inline long eval_fib(long n) { @@ -223,4 +228,8 @@ int main(int argc, char* argv[]) { return 0; } +#ifdef KOKKOS_ENABLE_DEPRECATION_WARNINGS +KOKKOS_IMPL_DISABLE_DEPRECATED_WARNINGS_POP() +#endif + #endif diff --git a/lib/kokkos/core/src/CMakeLists.txt b/lib/kokkos/core/src/CMakeLists.txt index b84677e61b..72663739a1 100644 --- a/lib/kokkos/core/src/CMakeLists.txt +++ b/lib/kokkos/core/src/CMakeLists.txt @@ -1,118 +1,125 @@ -KOKKOS_INCLUDE_DIRECTORIES( - ${CMAKE_CURRENT_BINARY_DIR} - ${CMAKE_CURRENT_SOURCE_DIR} - ${KOKKOS_TOP_BUILD_DIR} -) -IF (NOT desul_FOUND) - IF(KOKKOS_ENABLE_CUDA) - SET(DESUL_ATOMICS_ENABLE_CUDA ON) - ENDIF() - IF(KOKKOS_ENABLE_CUDA_RELOCATABLE_DEVICE_CODE) - SET(DESUL_ATOMICS_ENABLE_CUDA_SEPARABLE_COMPILATION ON) - ENDIF() - IF(KOKKOS_ENABLE_HIP) - SET(DESUL_ATOMICS_ENABLE_HIP ON) - ENDIF() - IF(KOKKOS_ENABLE_HIP_RELOCATABLE_DEVICE_CODE) - SET(DESUL_ATOMICS_ENABLE_HIP_SEPARABLE_COMPILATION ON) - ENDIF() - IF(KOKKOS_ENABLE_SYCL) - SET(DESUL_ATOMICS_ENABLE_SYCL ON) - IF(KOKKOS_IMPL_SYCL_DEVICE_GLOBAL_SUPPORTED AND NOT KOKKOS_IMPL_HAVE_SYCL_EXT_ONEAPI_DEVICE_GLOBAL) - SET(DESUL_ATOMICS_ENABLE_SYCL_SEPARABLE_COMPILATION ON) - ENDIF() - ENDIF() - IF(KOKKOS_ENABLE_OPENMPTARGET) - SET(DESUL_ATOMICS_ENABLE_OPENMP ON) # not a typo Kokkos OpenMPTarget -> Desul OpenMP - ENDIF() - IF(KOKKOS_ENABLE_OPENACC) - SET(DESUL_ATOMICS_ENABLE_OPENACC ON) - ENDIF() - CONFIGURE_FILE( - ${CMAKE_CURRENT_SOURCE_DIR}/../../tpls/desul/Config.hpp.cmake.in - ${CMAKE_CURRENT_BINARY_DIR}/desul/atomics/Config.hpp +kokkos_include_directories(${CMAKE_CURRENT_BINARY_DIR} ${CMAKE_CURRENT_SOURCE_DIR} ${KOKKOS_TOP_BUILD_DIR}) +if(NOT desul_FOUND) + if(KOKKOS_ENABLE_CUDA) + set(DESUL_ATOMICS_ENABLE_CUDA ON) + endif() + if(KOKKOS_ENABLE_CUDA_RELOCATABLE_DEVICE_CODE) + set(DESUL_ATOMICS_ENABLE_CUDA_SEPARABLE_COMPILATION ON) + endif() + if(KOKKOS_ENABLE_HIP) + set(DESUL_ATOMICS_ENABLE_HIP ON) + endif() + if(KOKKOS_ENABLE_HIP_RELOCATABLE_DEVICE_CODE) + set(DESUL_ATOMICS_ENABLE_HIP_SEPARABLE_COMPILATION ON) + endif() + if(KOKKOS_ENABLE_SYCL) + set(DESUL_ATOMICS_ENABLE_SYCL ON) + if(KOKKOS_IMPL_SYCL_DEVICE_GLOBAL_SUPPORTED AND NOT KOKKOS_IMPL_HAVE_SYCL_EXT_ONEAPI_DEVICE_GLOBAL) + set(DESUL_ATOMICS_ENABLE_SYCL_SEPARABLE_COMPILATION ON) + endif() + endif() + if(KOKKOS_ENABLE_OPENMPTARGET) + set(DESUL_ATOMICS_ENABLE_OPENMP ON) # not a typo Kokkos OpenMPTarget -> Desul OpenMP + endif() + if(KOKKOS_ENABLE_OPENACC) + # FIXME_OPENACC FIXME_CLACC - Below condition will be removed if Clacc can compile atomics. + if(KOKKOS_CXX_COMPILER_ID STREQUAL NVHPC) + set(DESUL_ATOMICS_ENABLE_OPENACC ON) + endif() + endif() + configure_file( + ${KOKKOS_SOURCE_DIR}/tpls/desul/Config.hpp.cmake.in ${CMAKE_CURRENT_BINARY_DIR}/desul/atomics/Config.hpp ) - KOKKOS_INCLUDE_DIRECTORIES( - ${CMAKE_CURRENT_SOURCE_DIR}/../../tpls/desul/include - ) -ENDIF() + kokkos_include_directories(${KOKKOS_SOURCE_DIR}/tpls/desul/include) +endif() -INSTALL (DIRECTORY - "${CMAKE_CURRENT_SOURCE_DIR}/" +install( + DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/" DESTINATION ${KOKKOS_HEADER_DIR} FILES_MATCHING PATTERN "*.hpp" PATTERN "*.h" ) -SET(KOKKOS_CORE_SRCS) -APPEND_GLOB(KOKKOS_CORE_SRCS ${CMAKE_CURRENT_SOURCE_DIR}/impl/*.cpp) -SET(KOKKOS_CORE_HEADERS) -APPEND_GLOB(KOKKOS_CORE_HEADERS ${CMAKE_CURRENT_SOURCE_DIR}/*.hpp) -APPEND_GLOB(KOKKOS_CORE_HEADERS ${CMAKE_CURRENT_SOURCE_DIR}/impl/*.hpp) +set(KOKKOS_CORE_SRCS) +append_glob(KOKKOS_CORE_SRCS ${CMAKE_CURRENT_SOURCE_DIR}/impl/*.cpp) +set(KOKKOS_CORE_HEADERS) +append_glob(KOKKOS_CORE_HEADERS ${CMAKE_CURRENT_SOURCE_DIR}/*.hpp) +append_glob(KOKKOS_CORE_HEADERS ${CMAKE_CURRENT_SOURCE_DIR}/impl/*.hpp) -IF (KOKKOS_ENABLE_CUDA) - APPEND_GLOB(KOKKOS_CORE_SRCS ${CMAKE_CURRENT_SOURCE_DIR}/Cuda/*.cpp) - APPEND_GLOB(KOKKOS_CORE_HEADERS ${CMAKE_CURRENT_SOURCE_DIR}/Cuda/*.hpp) -ENDIF() +if(KOKKOS_ENABLE_CUDA) + append_glob(KOKKOS_CORE_SRCS ${CMAKE_CURRENT_SOURCE_DIR}/Cuda/*.cpp) + if(NOT Kokkos_ENABLE_DEPRECATED_CODE_4) + list(REMOVE_ITEM KOKKOS_CORE_SRCS ${CMAKE_CURRENT_SOURCE_DIR}/Cuda/Kokkos_Cuda_Task.cpp) + endif() + append_glob(KOKKOS_CORE_HEADERS ${CMAKE_CURRENT_SOURCE_DIR}/Cuda/*.hpp) +endif() -IF (KOKKOS_ENABLE_OPENMP) - APPEND_GLOB(KOKKOS_CORE_SRCS ${CMAKE_CURRENT_SOURCE_DIR}/OpenMP/*.cpp) - APPEND_GLOB(KOKKOS_CORE_HEADERS ${CMAKE_CURRENT_SOURCE_DIR}/OpenMP/*.hpp) -ENDIF() +if(KOKKOS_ENABLE_OPENMP) + append_glob(KOKKOS_CORE_SRCS ${CMAKE_CURRENT_SOURCE_DIR}/OpenMP/*.cpp) + if(NOT Kokkos_ENABLE_DEPRECATED_CODE_4) + list(REMOVE_ITEM KOKKOS_CORE_SRCS ${CMAKE_CURRENT_SOURCE_DIR}/OpenMP/Kokkos_OpenMP_Task.cpp) + endif() + append_glob(KOKKOS_CORE_HEADERS ${CMAKE_CURRENT_SOURCE_DIR}/OpenMP/*.hpp) +endif() -IF (KOKKOS_ENABLE_OPENMPTARGET) - APPEND_GLOB(KOKKOS_CORE_SRCS ${CMAKE_CURRENT_SOURCE_DIR}/OpenMPTarget/*.cpp) - APPEND_GLOB(KOKKOS_CORE_HEADERS ${CMAKE_CURRENT_SOURCE_DIR}/OpenMPTarget/*.hpp) -ENDIF() +if(KOKKOS_ENABLE_OPENMPTARGET) + append_glob(KOKKOS_CORE_SRCS ${CMAKE_CURRENT_SOURCE_DIR}/OpenMPTarget/*.cpp) + append_glob(KOKKOS_CORE_HEADERS ${CMAKE_CURRENT_SOURCE_DIR}/OpenMPTarget/*.hpp) +endif() -IF (KOKKOS_ENABLE_OPENACC) - APPEND_GLOB(KOKKOS_CORE_SRCS ${CMAKE_CURRENT_SOURCE_DIR}/OpenACC/*.cpp) - APPEND_GLOB(KOKKOS_CORE_HEADERS ${CMAKE_CURRENT_SOURCE_DIR}/OpenACC/*.hpp) -ENDIF() +if(KOKKOS_ENABLE_OPENACC) + append_glob(KOKKOS_CORE_SRCS ${CMAKE_CURRENT_SOURCE_DIR}/OpenACC/*.cpp) + append_glob(KOKKOS_CORE_HEADERS ${CMAKE_CURRENT_SOURCE_DIR}/OpenACC/*.hpp) +endif() -IF (KOKKOS_ENABLE_THREADS) - APPEND_GLOB(KOKKOS_CORE_SRCS ${CMAKE_CURRENT_SOURCE_DIR}/Threads/*.cpp) - APPEND_GLOB(KOKKOS_CORE_HEADERS ${CMAKE_CURRENT_SOURCE_DIR}/Threads/*.hpp) -ENDIF() +if(KOKKOS_ENABLE_THREADS) + append_glob(KOKKOS_CORE_SRCS ${CMAKE_CURRENT_SOURCE_DIR}/Threads/*.cpp) + append_glob(KOKKOS_CORE_HEADERS ${CMAKE_CURRENT_SOURCE_DIR}/Threads/*.hpp) +endif() -IF (KOKKOS_ENABLE_HIP) - APPEND_GLOB(KOKKOS_CORE_SRCS ${CMAKE_CURRENT_SOURCE_DIR}/HIP/*.cpp) - APPEND_GLOB(KOKKOS_CORE_HEADERS ${CMAKE_CURRENT_SOURCE_DIR}/HIP/*.hpp) -ENDIF() +if(KOKKOS_ENABLE_HIP) + append_glob(KOKKOS_CORE_SRCS ${CMAKE_CURRENT_SOURCE_DIR}/HIP/*.cpp) + append_glob(KOKKOS_CORE_HEADERS ${CMAKE_CURRENT_SOURCE_DIR}/HIP/*.hpp) +endif() -IF (KOKKOS_ENABLE_HPX) - APPEND_GLOB(KOKKOS_CORE_SRCS ${CMAKE_CURRENT_SOURCE_DIR}/HPX/*.cpp) - APPEND_GLOB(KOKKOS_CORE_HEADERS ${CMAKE_CURRENT_SOURCE_DIR}/HPX/*.hpp) -ENDIF() +if(KOKKOS_ENABLE_HPX) + append_glob(KOKKOS_CORE_SRCS ${CMAKE_CURRENT_SOURCE_DIR}/HPX/*.cpp) + if(NOT Kokkos_ENABLE_DEPRECATED_CODE_4) + list(REMOVE_ITEM KOKKOS_CORE_SRCS ${CMAKE_CURRENT_SOURCE_DIR}/HPX/Kokkos_HPX_Task.cpp) + endif() + append_glob(KOKKOS_CORE_HEADERS ${CMAKE_CURRENT_SOURCE_DIR}/HPX/*.hpp) +endif() -IF (KOKKOS_ENABLE_SERIAL) - APPEND_GLOB(KOKKOS_CORE_SRCS ${CMAKE_CURRENT_SOURCE_DIR}/Serial/*.cpp) - APPEND_GLOB(KOKKOS_CORE_HEADERS ${CMAKE_CURRENT_SOURCE_DIR}/Serial/*.hpp) -ENDIF() +if(KOKKOS_ENABLE_SERIAL) + append_glob(KOKKOS_CORE_SRCS ${CMAKE_CURRENT_SOURCE_DIR}/Serial/*.cpp) + if(NOT Kokkos_ENABLE_DEPRECATED_CODE_4) + list(REMOVE_ITEM KOKKOS_CORE_SRCS ${CMAKE_CURRENT_SOURCE_DIR}/Serial/Kokkos_Serial_Task.cpp) + endif() + append_glob(KOKKOS_CORE_HEADERS ${CMAKE_CURRENT_SOURCE_DIR}/Serial/*.hpp) +endif() -IF (KOKKOS_ENABLE_SYCL) - APPEND_GLOB(KOKKOS_CORE_SRCS ${CMAKE_CURRENT_SOURCE_DIR}/SYCL/*.cpp) - APPEND_GLOB(KOKKOS_CORE_HEADERS ${CMAKE_CURRENT_SOURCE_DIR}/SYCL/*.hpp) -ENDIF() +if(KOKKOS_ENABLE_SYCL) + append_glob(KOKKOS_CORE_SRCS ${CMAKE_CURRENT_SOURCE_DIR}/SYCL/*.cpp) + append_glob(KOKKOS_CORE_HEADERS ${CMAKE_CURRENT_SOURCE_DIR}/SYCL/*.hpp) +endif() -IF (NOT desul_FOUND) - IF (KOKKOS_ENABLE_CUDA) - APPEND_GLOB(KOKKOS_CORE_SRCS ${CMAKE_CURRENT_SOURCE_DIR}/../../tpls/desul/src/Lock_Array_CUDA.cpp) - ELSEIF (KOKKOS_ENABLE_HIP) - APPEND_GLOB(KOKKOS_CORE_SRCS ${CMAKE_CURRENT_SOURCE_DIR}/../../tpls/desul/src/Lock_Array_HIP.cpp) - ELSEIF (KOKKOS_ENABLE_SYCL) - APPEND_GLOB(KOKKOS_CORE_SRCS ${CMAKE_CURRENT_SOURCE_DIR}/../../tpls/desul/src/Lock_Array_SYCL.cpp) - ENDIF() - APPEND_GLOB(KOKKOS_CORE_HEADERS ${CMAKE_CURRENT_SOURCE_DIR}/../../tpls/desul/include/desul/*.hpp) - APPEND_GLOB(KOKKOS_CORE_HEADERS ${CMAKE_CURRENT_SOURCE_DIR}/../../tpls/desul/include/desul/*/*.hpp) - APPEND_GLOB(KOKKOS_CORE_HEADERS ${CMAKE_CURRENT_SOURCE_DIR}/../../tpls/desul/include/desul/*/*/*.hpp) - APPEND_GLOB(KOKKOS_CORE_HEADERS ${CMAKE_CURRENT_SOURCE_DIR}/../../tpls/desul/include/*/*/*.inc*) - APPEND_GLOB(KOKKOS_CORE_HEADERS ${CMAKE_CURRENT_BINARY_DIR}/desul/*.hpp) +if(NOT desul_FOUND) + if(KOKKOS_ENABLE_CUDA) + append_glob(KOKKOS_CORE_SRCS ${KOKKOS_SOURCE_DIR}/tpls/desul/src/Lock_Array_CUDA.cpp) + elseif(KOKKOS_ENABLE_HIP) + append_glob(KOKKOS_CORE_SRCS ${KOKKOS_SOURCE_DIR}/tpls/desul/src/Lock_Array_HIP.cpp) + elseif(KOKKOS_ENABLE_SYCL) + append_glob(KOKKOS_CORE_SRCS ${KOKKOS_SOURCE_DIR}/tpls/desul/src/Lock_Array_SYCL.cpp) + endif() + append_glob(KOKKOS_CORE_HEADERS ${KOKKOS_SOURCE_DIR}/tpls/desul/include/desul/*.hpp) + append_glob(KOKKOS_CORE_HEADERS ${KOKKOS_SOURCE_DIR}/tpls/desul/include/desul/*/*.hpp) + append_glob(KOKKOS_CORE_HEADERS ${KOKKOS_SOURCE_DIR}/tpls/desul/include/desul/*/*/*.hpp) + append_glob(KOKKOS_CORE_HEADERS ${KOKKOS_SOURCE_DIR}/tpls/desul/include/*/*/*.inc*) + append_glob(KOKKOS_CORE_HEADERS ${CMAKE_CURRENT_BINARY_DIR}/desul/*.hpp) - INSTALL (DIRECTORY - "${CMAKE_CURRENT_SOURCE_DIR}/../../tpls/desul/include/desul" - "${CMAKE_CURRENT_BINARY_DIR}/desul" + install( + DIRECTORY "${KOKKOS_SOURCE_DIR}/tpls/desul/include/desul" "${CMAKE_CURRENT_BINARY_DIR}/desul" DESTINATION ${KOKKOS_HEADER_DIR} FILES_MATCHING PATTERN "*.inc" @@ -120,33 +127,26 @@ IF (NOT desul_FOUND) PATTERN "*.hpp" ) - MESSAGE(STATUS "Using internal desul_atomics copy") -ELSE() - MESSAGE(STATUS "Using external desul_atomics install found at:") - MESSAGE(STATUS " " ${desul_DIR}) -ENDIF() + message(STATUS "Using internal desul_atomics copy") +else() + message(STATUS "Using external desul_atomics install found at:") + message(STATUS " " ${desul_DIR}) +endif() - -KOKKOS_ADD_LIBRARY( - kokkoscore - SOURCES ${KOKKOS_CORE_SRCS} - HEADERS ${KOKKOS_CORE_HEADERS} +kokkos_add_library( + kokkoscore SOURCES ${KOKKOS_CORE_SRCS} HEADERS ${KOKKOS_CORE_HEADERS} ADD_BUILD_OPTIONS # core should be given all the necessary compiler/linker flags ) -KOKKOS_LIB_INCLUDE_DIRECTORIES(kokkoscore - ${KOKKOS_TOP_BUILD_DIR} - ${CMAKE_CURRENT_BINARY_DIR} - ${CMAKE_CURRENT_SOURCE_DIR} +kokkos_lib_include_directories( + kokkoscore ${KOKKOS_TOP_BUILD_DIR} ${CMAKE_CURRENT_BINARY_DIR} ${CMAKE_CURRENT_SOURCE_DIR} ) -IF (NOT desul_FOUND) - KOKKOS_LIB_INCLUDE_DIRECTORIES(kokkoscore - ${CMAKE_CURRENT_SOURCE_DIR}/../../tpls/desul/include - ) -ENDIF() +if(NOT desul_FOUND) + kokkos_lib_include_directories(kokkoscore ${KOKKOS_SOURCE_DIR}/tpls/desul/include) +endif() -IF (Kokkos_ENABLE_IMPL_MDSPAN) - MESSAGE(STATUS "Experimental mdspan support is enabled") +if(Kokkos_ENABLE_IMPL_MDSPAN) + message(STATUS "Experimental mdspan support is enabled") # Some compilers now include mdspan... we just flag on their version # for now until we can get some compiler detection support @@ -154,62 +154,56 @@ IF (Kokkos_ENABLE_IMPL_MDSPAN) check_include_file_cxx(experimental/mdspan KOKKOS_COMPILER_SUPPORTS_EXPERIMENTAL_MDSPAN) check_include_file_cxx(mdspan KOKKOS_COMPILER_SUPPORTS_MDSPAN) - if (Kokkos_ENABLE_MDSPAN_EXTERNAL) - MESSAGE(STATUS "Using external mdspan") + if(Kokkos_ENABLE_MDSPAN_EXTERNAL) + message(STATUS "Using external mdspan") target_link_libraries(kokkoscore PUBLIC std::mdspan) elseif(KOKKOS_COMPILER_SUPPORTS_MDSPAN AND NOT Kokkos_ENABLE_IMPL_SKIP_COMPILER_MDSPAN) message(STATUS "Using compiler-supplied mdspan") elseif(KOKKOS_COMPILER_SUPPORTS_EXPERIMENTAL_MDSPAN AND NOT Kokkos_ENABLE_IMPL_SKIP_COMPILER_MDSPAN) message(STATUS "Using compiler-supplied experimental/mdspan") else() - KOKKOS_LIB_INCLUDE_DIRECTORIES( - kokkoscore - ${CMAKE_CURRENT_SOURCE_DIR}/../../tpls/mdspan/include - ) + kokkos_lib_include_directories(kokkoscore ${KOKKOS_SOURCE_DIR}/tpls/mdspan/include) - APPEND_GLOB(KOKKOS_CORE_HEADERS ${CMAKE_CURRENT_SOURCE_DIR}/../../tpls/mdspan/include/experimental/__p0009_bits/*.hpp) - APPEND_GLOB(KOKKOS_CORE_HEADERS ${CMAKE_CURRENT_SOURCE_DIR}/../../tpls/mdspan/include/experimental/mdspan) + append_glob(KOKKOS_CORE_HEADERS ${KOKKOS_SOURCE_DIR}/tpls/mdspan/include/experimental/__p0009_bits/*.hpp) + append_glob(KOKKOS_CORE_HEADERS ${KOKKOS_SOURCE_DIR}/tpls/mdspan/include/experimental/mdspan) - INSTALL (DIRECTORY - "${CMAKE_CURRENT_SOURCE_DIR}/../../tpls/mdspan/include/" + install( + DIRECTORY "${KOKKOS_SOURCE_DIR}/tpls/mdspan/include/" DESTINATION ${KOKKOS_HEADER_DIR} FILES_MATCHING PATTERN "mdspan" PATTERN "*.hpp" ) - MESSAGE(STATUS "Using internal mdspan directory ${CMAKE_CURRENT_SOURCE_DIR}/../../tpls/mdspan/include") + message(STATUS "Using internal mdspan directory ${KOKKOS_SOURCE_DIR}/tpls/mdspan/include") endif() -ENDIF() +endif() -KOKKOS_LINK_TPL(kokkoscore PUBLIC HWLOC) -KOKKOS_LINK_TPL(kokkoscore PUBLIC CUDA) -KOKKOS_LINK_TPL(kokkoscore PUBLIC HPX) -KOKKOS_LINK_TPL(kokkoscore PUBLIC LIBDL) +kokkos_link_tpl(kokkoscore PUBLIC HWLOC) +kokkos_link_tpl(kokkoscore PUBLIC CUDA) +kokkos_link_tpl(kokkoscore PUBLIC HPX) +kokkos_link_tpl(kokkoscore PUBLIC LIBDL) # On *nix-like systems (Linux, macOS) we need pthread for C++ std::thread -IF (NOT WIN32) - KOKKOS_LINK_TPL(kokkoscore PUBLIC THREADS) -ENDIF() -IF (NOT KOKKOS_ENABLE_COMPILE_AS_CMAKE_LANGUAGE) - KOKKOS_LINK_TPL(kokkoscore PUBLIC ROCM) -ENDIF() +if(NOT WIN32) + kokkos_link_tpl(kokkoscore PUBLIC THREADS) +endif() +if(NOT KOKKOS_ENABLE_COMPILE_AS_CMAKE_LANGUAGE) + kokkos_link_tpl(kokkoscore PUBLIC ROCM) +endif() # FIXME: We need a proper solution to figure out whether to enable # libatomic # Most compilers only require libatomic for 128-bit CAS # I (CT) had removed 128bit CAS from desul to not need libatomic. -IF (KOKKOS_ENABLE_OPENMPTARGET) +if(KOKKOS_ENABLE_OPENMPTARGET) target_link_libraries(kokkoscore PUBLIC atomic) -ENDIF() +endif() -IF (desul_FOUND) +if(desul_FOUND) target_link_libraries(kokkoscore PUBLIC desul_atomics) -ENDIF() +endif() -# FIXME_TRILINOS Trilinos doesn't allow for Kokkos to use find_dependency so we -# just append the flags in cmake/kokkos_tpls.cmake instead of linking with the -# OpenMP target. -IF(Kokkos_ENABLE_OPENMP AND NOT KOKKOS_HAS_TRILINOS) +if(Kokkos_ENABLE_OPENMP) target_link_libraries(kokkoscore PUBLIC OpenMP::OpenMP_CXX) -ENDIF() +endif() -KOKKOS_LINK_TPL(kokkoscore PUBLIC LIBQUADMATH) +kokkos_link_tpl(kokkoscore PUBLIC LIBQUADMATH) diff --git a/lib/kokkos/core/src/Cuda/Kokkos_Cuda.hpp b/lib/kokkos/core/src/Cuda/Kokkos_Cuda.hpp index fd86976d3b..07c35e6611 100644 --- a/lib/kokkos/core/src/Cuda/Kokkos_Cuda.hpp +++ b/lib/kokkos/core/src/Cuda/Kokkos_Cuda.hpp @@ -35,7 +35,6 @@ static_assert(false, #include // CUDA_SAFE_CALL #include -#include #include #include #include diff --git a/lib/kokkos/core/src/Cuda/Kokkos_CudaSpace.cpp b/lib/kokkos/core/src/Cuda/Kokkos_CudaSpace.cpp index 6ae24022c8..8bcd6525c9 100644 --- a/lib/kokkos/core/src/Cuda/Kokkos_CudaSpace.cpp +++ b/lib/kokkos/core/src/Cuda/Kokkos_CudaSpace.cpp @@ -201,7 +201,14 @@ void *impl_allocate_common(const int device_id, } } #elif (defined(KOKKOS_ENABLE_IMPL_CUDA_MALLOC_ASYNC) && CUDART_VERSION >= 11020) - if (arg_alloc_size >= memory_threshold_g) { + // FIXME_KEPLER Everything after Kepler should support cudaMallocAsync + int device_supports_cuda_malloc_async; + KOKKOS_IMPL_CUDA_SAFE_CALL( + cudaDeviceGetAttribute(&device_supports_cuda_malloc_async, + cudaDevAttrMemoryPoolsSupported, device_id)); + + if (arg_alloc_size >= memory_threshold_g && + device_supports_cuda_malloc_async == 1) { error_code = cudaMallocAsync(&ptr, arg_alloc_size, stream); if (error_code == cudaSuccess) { diff --git a/lib/kokkos/core/src/Cuda/Kokkos_CudaSpace.hpp b/lib/kokkos/core/src/Cuda/Kokkos_CudaSpace.hpp index e1d062d72d..1ccf38a4a1 100644 --- a/lib/kokkos/core/src/Cuda/Kokkos_CudaSpace.hpp +++ b/lib/kokkos/core/src/Cuda/Kokkos_CudaSpace.hpp @@ -73,9 +73,9 @@ class CudaSpace { CudaSpace(int device_id, cudaStream_t stream); public: - CudaSpace(CudaSpace&& rhs) = default; - CudaSpace(const CudaSpace& rhs) = default; - CudaSpace& operator=(CudaSpace&& rhs) = default; + CudaSpace(CudaSpace&& rhs) = default; + CudaSpace(const CudaSpace& rhs) = default; + CudaSpace& operator=(CudaSpace&& rhs) = default; CudaSpace& operator=(const CudaSpace& rhs) = default; ~CudaSpace() = default; @@ -174,9 +174,9 @@ class CudaUVMSpace { CudaUVMSpace(int device_id, cudaStream_t stream); public: - CudaUVMSpace(CudaUVMSpace&& rhs) = default; - CudaUVMSpace(const CudaUVMSpace& rhs) = default; - CudaUVMSpace& operator=(CudaUVMSpace&& rhs) = default; + CudaUVMSpace(CudaUVMSpace&& rhs) = default; + CudaUVMSpace(const CudaUVMSpace& rhs) = default; + CudaUVMSpace& operator=(CudaUVMSpace&& rhs) = default; CudaUVMSpace& operator=(const CudaUVMSpace& rhs) = default; ~CudaUVMSpace() = default; @@ -266,9 +266,9 @@ class CudaHostPinnedSpace { CudaHostPinnedSpace(int device_id, cudaStream_t stream); public: - CudaHostPinnedSpace(CudaHostPinnedSpace&& rhs) = default; - CudaHostPinnedSpace(const CudaHostPinnedSpace& rhs) = default; - CudaHostPinnedSpace& operator=(CudaHostPinnedSpace&& rhs) = default; + CudaHostPinnedSpace(CudaHostPinnedSpace&& rhs) = default; + CudaHostPinnedSpace(const CudaHostPinnedSpace& rhs) = default; + CudaHostPinnedSpace& operator=(CudaHostPinnedSpace&& rhs) = default; CudaHostPinnedSpace& operator=(const CudaHostPinnedSpace& rhs) = default; ~CudaHostPinnedSpace() = default; diff --git a/lib/kokkos/core/src/Cuda/Kokkos_Cuda_GraphNodeKernel.hpp b/lib/kokkos/core/src/Cuda/Kokkos_Cuda_GraphNodeKernel.hpp index 5a821ab64a..058b1f538d 100644 --- a/lib/kokkos/core/src/Cuda/Kokkos_Cuda_GraphNodeKernel.hpp +++ b/lib/kokkos/core/src/Cuda/Kokkos_Cuda_GraphNodeKernel.hpp @@ -51,7 +51,8 @@ class GraphNodeKernelImpl m_graph_node_ptr = nullptr; // Basically, we have to make this mutable for the same reasons that the // global kernel buffers in the Cuda instance are mutable... - mutable Kokkos::OwningRawPtr m_driver_storage = nullptr; + mutable std::shared_ptr m_driver_storage = nullptr; + std::string label; public: using Policy = PolicyType; @@ -61,25 +62,20 @@ class GraphNodeKernelImpl - GraphNodeKernelImpl(std::string, Kokkos::Cuda const&, Functor arg_functor, + GraphNodeKernelImpl(std::string label_, Cuda const&, Functor arg_functor, PolicyDeduced&& arg_policy, ArgsDeduced&&... args) // This is super ugly, but it works for now and is the most minimal change // to the codebase for now... - : base_t(std::move(arg_functor), (PolicyDeduced &&) arg_policy, - (ArgsDeduced &&) args...) {} + : base_t(std::move(arg_functor), (PolicyDeduced&&)arg_policy, + (ArgsDeduced&&)args...), + label(std::move(label_)) {} // FIXME @graph Forward through the instance once that works in the backends template GraphNodeKernelImpl(Kokkos::Cuda const& ex, Functor arg_functor, PolicyDeduced&& arg_policy) - : GraphNodeKernelImpl("", ex, std::move(arg_functor), - (PolicyDeduced &&) arg_policy) {} - - ~GraphNodeKernelImpl() { - if (m_driver_storage) { - Kokkos::CudaSpace().deallocate(m_driver_storage, sizeof(base_t)); - } - } + : GraphNodeKernelImpl("[unlabeled]", ex, std::move(arg_functor), + (PolicyDeduced&&)arg_policy) {} void set_cuda_graph_ptr(cudaGraph_t* arg_graph_ptr) { m_graph_ptr = arg_graph_ptr; @@ -90,13 +86,21 @@ class GraphNodeKernelImpl allocate_driver_memory_buffer() const { + Kokkos::ObservingRawPtr allocate_driver_memory_buffer( + const CudaSpace& mem) const { KOKKOS_EXPECTS(m_driver_storage == nullptr) - m_driver_storage = static_cast(Kokkos::CudaSpace().allocate( - "GraphNodeKernel global memory functor storage", sizeof(base_t))); + std::string alloc_label = + label + " - GraphNodeKernel global memory functor storage"; + m_driver_storage = std::shared_ptr( + static_cast(mem.allocate(alloc_label.c_str(), sizeof(base_t))), + [alloc_label, mem](base_t* ptr) { + mem.deallocate(alloc_label.c_str(), ptr, sizeof(base_t)); + }); KOKKOS_ENSURES(m_driver_storage != nullptr) - return m_driver_storage; + return m_driver_storage.get(); } + + auto get_driver_storage() const { return m_driver_storage; } }; struct CudaGraphNodeAggregateKernel { @@ -128,7 +132,8 @@ struct get_graph_node_kernel_type // {{{1 template -auto* allocate_driver_storage_for_kernel(KernelType const& kernel) { +auto* allocate_driver_storage_for_kernel(const CudaSpace& mem, + KernelType const& kernel) { using graph_node_kernel_t = typename get_graph_node_kernel_type::type; auto const& kernel_as_graph_kernel = @@ -136,7 +141,7 @@ auto* allocate_driver_storage_for_kernel(KernelType const& kernel) { // TODO @graphs we need to somehow indicate the need for a fence in the // destructor of the GraphImpl object (so that we don't have to // just always do it) - return kernel_as_graph_kernel.allocate_driver_memory_buffer(); + return kernel_as_graph_kernel.allocate_driver_memory_buffer(mem); } template diff --git a/lib/kokkos/core/src/Cuda/Kokkos_Cuda_Graph_Impl.hpp b/lib/kokkos/core/src/Cuda/Kokkos_Cuda_Graph_Impl.hpp index 625d8c317a..8e800e756d 100644 --- a/lib/kokkos/core/src/Cuda/Kokkos_Cuda_Graph_Impl.hpp +++ b/lib/kokkos/core/src/Cuda/Kokkos_Cuda_Graph_Impl.hpp @@ -51,7 +51,14 @@ struct GraphImpl { using node_details_t = GraphNodeBackendSpecificDetails; - void _instantiate_graph() { + // Store drivers for the kernel nodes that launch in global memory. + // This is required as lifetime of drivers must be bounded to this instance's + // lifetime. + std::vector> m_driver_storage; + + public: + void instantiate() { + KOKKOS_EXPECTS(!m_graph_exec); constexpr size_t error_log_size = 256; cudaGraphNode_t error_node = nullptr; char error_log[error_log_size]; @@ -60,10 +67,10 @@ struct GraphImpl { ->cuda_graph_instantiate_wrapper(&m_graph_exec, m_graph, &error_node, error_log, error_log_size))); + KOKKOS_ENSURES(m_graph_exec); // TODO @graphs print out errors } - public: using root_node_impl_t = GraphNodeImpl; @@ -74,11 +81,11 @@ struct GraphImpl { // Not movable or copyable; it spends its whole life as a shared_ptr in the // Graph object - GraphImpl() = delete; - GraphImpl(GraphImpl const&) = delete; - GraphImpl(GraphImpl&&) = delete; + GraphImpl() = delete; + GraphImpl(GraphImpl const&) = delete; + GraphImpl(GraphImpl&&) = delete; GraphImpl& operator=(GraphImpl const&) = delete; - GraphImpl& operator=(GraphImpl&&) = delete; + GraphImpl& operator=(GraphImpl&&) = delete; ~GraphImpl() { // TODO @graphs we need to somehow indicate the need for a fence in the // destructor of the GraphImpl object (so that we don't have to @@ -129,6 +136,8 @@ struct GraphImpl { kernel.set_cuda_graph_node_ptr(&cuda_node); kernel.execute(); KOKKOS_ENSURES(bool(cuda_node)); + if (std::shared_ptr tmp = kernel.get_driver_storage()) + m_driver_storage.push_back(std::move(tmp)); } template @@ -158,13 +167,13 @@ struct GraphImpl { &cuda_node, 1))); } - void submit() { + void submit(const execution_space& exec) { if (!bool(m_graph_exec)) { - _instantiate_graph(); + instantiate(); } KOKKOS_IMPL_CUDA_SAFE_CALL( - (m_execution_space.impl_internal_space_instance() - ->cuda_graph_launch_wrapper(m_graph_exec))); + (exec.impl_internal_space_instance()->cuda_graph_launch_wrapper( + m_graph_exec))); } execution_space const& get_execution_space() const noexcept { @@ -197,6 +206,9 @@ struct GraphImpl { m_execution_space, _graph_node_kernel_ctor_tag{}, aggregate_kernel_impl_t{}); } + + cudaGraph_t cuda_graph() { return m_graph; } + cudaGraphExec_t cuda_graph_exec() { return m_graph_exec; } }; } // end namespace Impl diff --git a/lib/kokkos/core/src/Cuda/Kokkos_Cuda_Instance.cpp b/lib/kokkos/core/src/Cuda/Kokkos_Cuda_Instance.cpp index 158c8acdda..ec5768a7f0 100644 --- a/lib/kokkos/core/src/Cuda/Kokkos_Cuda_Instance.cpp +++ b/lib/kokkos/core/src/Cuda/Kokkos_Cuda_Instance.cpp @@ -26,10 +26,10 @@ #include -//#include -//#include -//#include -//#include +// #include +// #include +// #include +// #include #include #include #include @@ -687,16 +687,6 @@ void Cuda::print_configuration(std::ostream &os, bool /*verbose*/) const { os << " KOKKOS_ENABLE_CUDA: yes\n"; os << "Cuda Options:\n"; - os << " KOKKOS_ENABLE_CUDA_LAMBDA: "; -#ifdef KOKKOS_ENABLE_CUDA_LAMBDA - os << "yes\n"; -#else - os << "no\n"; -#endif -#ifdef KOKKOS_ENABLE_DEPRECATED_CODE_4 - os << " KOKKOS_ENABLE_CUDA_LDG_INTRINSIC: "; - os << "yes\n"; -#endif os << " KOKKOS_ENABLE_CUDA_RELOCATABLE_DEVICE_CODE: "; #ifdef KOKKOS_ENABLE_CUDA_RELOCATABLE_DEVICE_CODE os << "yes\n"; @@ -708,12 +698,6 @@ void Cuda::print_configuration(std::ostream &os, bool /*verbose*/) const { os << "yes\n"; #else os << "no\n"; -#endif - os << " KOKKOS_ENABLE_CXX11_DISPATCH_LAMBDA: "; -#ifdef KOKKOS_ENABLE_CXX11_DISPATCH_LAMBDA - os << "yes\n"; -#else - os << "no\n"; #endif os << " KOKKOS_ENABLE_IMPL_CUDA_MALLOC_ASYNC: "; #ifdef KOKKOS_ENABLE_IMPL_CUDA_MALLOC_ASYNC diff --git a/lib/kokkos/core/src/Cuda/Kokkos_Cuda_KernelLaunch.hpp b/lib/kokkos/core/src/Cuda/Kokkos_Cuda_KernelLaunch.hpp index b0dadb45f7..2d00e735cb 100644 --- a/lib/kokkos/core/src/Cuda/Kokkos_Cuda_KernelLaunch.hpp +++ b/lib/kokkos/core/src/Cuda/Kokkos_Cuda_KernelLaunch.hpp @@ -209,8 +209,8 @@ inline void configure_shmem_preference(const int cuda_device, // Use multiples of 8kB const size_t max_shmem_per_sm = device_props.sharedMemPerMultiprocessor; size_t carveout = shmem_per_block == 0 - ? 0 - : 100 * + ? 0 + : 100 * (((num_blocks_desired * shmem_per_block + min_shmem_size_per_sm - 1) / min_shmem_size_per_sm) * @@ -491,7 +491,10 @@ struct CudaParallelLaunchKernelInvoker< cuda_instance->m_deviceProp, block_size, shmem, desired_occupancy); } - auto* driver_ptr = Impl::allocate_driver_storage_for_kernel(driver); + auto* driver_ptr = Impl::allocate_driver_storage_for_kernel( + CudaSpace::impl_create(cuda_instance->m_cudaDev, + cuda_instance->m_stream), + driver); // Unlike in the non-graph case, we can get away with doing an async copy // here because the `DriverType` instance is held in the GraphNodeImpl @@ -714,7 +717,7 @@ struct CudaParallelLaunch; template CudaParallelLaunch(Args&&... args) { - base_t::launch_kernel((Args &&) args...); + base_t::launch_kernel((Args&&)args...); } }; @@ -728,7 +731,7 @@ struct CudaParallelLaunch; template CudaParallelLaunch(Args&&... args) { - base_t::create_parallel_launch_graph_node((Args &&) args...); + base_t::create_parallel_launch_graph_node((Args&&)args...); } }; diff --git a/lib/kokkos/core/src/Cuda/Kokkos_Cuda_Parallel_MDRange.hpp b/lib/kokkos/core/src/Cuda/Kokkos_Cuda_Parallel_MDRange.hpp index 6303898400..c50ff43034 100644 --- a/lib/kokkos/core/src/Cuda/Kokkos_Cuda_Parallel_MDRange.hpp +++ b/lib/kokkos/core/src/Cuda/Kokkos_Cuda_Parallel_MDRange.hpp @@ -95,11 +95,39 @@ class ParallelFor, Kokkos::Cuda> { inline void execute() const { if (m_rp.m_num_tiles == 0) return; - const auto maxblocks = m_rp.space().cuda_device_prop().maxGridSize; + const auto maxblocks = m_rp.space().cuda_device_prop().maxGridSize; + const auto maxthreads = m_rp.space().cuda_device_prop().maxThreadsDim; + [[maybe_unused]] const auto maxThreadsPerBlock = + m_rp.space().cuda_device_prop().maxThreadsPerBlock; + // make sure the Z dimension (it is less than x,y limits) isn't exceeded + const auto clampZ = [&](const int input) { + return (input > maxthreads[2] ? maxthreads[2] : input); + }; + // make sure the block dimensions don't exceed the max number of threads + // allowed + const auto check_block_sizes = [&]([[maybe_unused]] const dim3& block) { + KOKKOS_ASSERT(block.x > 0 && + block.x <= static_cast(maxthreads[0])); + KOKKOS_ASSERT(block.y > 0 && + block.y <= static_cast(maxthreads[1])); + KOKKOS_ASSERT(block.z > 0 && + block.z <= static_cast(maxthreads[2])); + KOKKOS_ASSERT(block.x * block.y * block.z <= + static_cast(maxThreadsPerBlock)); + }; + // make sure the grid dimensions don't exceed the max number of blocks + // allowed + const auto check_grid_sizes = [&]([[maybe_unused]] const dim3& grid) { + KOKKOS_ASSERT(grid.x > 0 && + grid.x <= static_cast(maxblocks[0])); + KOKKOS_ASSERT(grid.y > 0 && + grid.y <= static_cast(maxblocks[1])); + KOKKOS_ASSERT(grid.z > 0 && + grid.z <= static_cast(maxblocks[2])); + }; if (RP::rank == 2) { const dim3 block(m_rp.m_tile[0], m_rp.m_tile[1], 1); - KOKKOS_ASSERT(block.x > 0); - KOKKOS_ASSERT(block.y > 0); + check_block_sizes(block); const dim3 grid( std::min( (m_rp.m_upper[0] - m_rp.m_lower[0] + block.x - 1) / block.x, @@ -108,13 +136,12 @@ class ParallelFor, Kokkos::Cuda> { (m_rp.m_upper[1] - m_rp.m_lower[1] + block.y - 1) / block.y, maxblocks[1]), 1); + check_grid_sizes(grid); CudaParallelLaunch( *this, grid, block, 0, m_rp.space().impl_internal_space_instance()); } else if (RP::rank == 3) { - const dim3 block(m_rp.m_tile[0], m_rp.m_tile[1], m_rp.m_tile[2]); - KOKKOS_ASSERT(block.x > 0); - KOKKOS_ASSERT(block.y > 0); - KOKKOS_ASSERT(block.z > 0); + const dim3 block(m_rp.m_tile[0], m_rp.m_tile[1], clampZ(m_rp.m_tile[2])); + check_block_sizes(block); const dim3 grid( std::min( (m_rp.m_upper[0] - m_rp.m_lower[0] + block.x - 1) / block.x, @@ -125,15 +152,16 @@ class ParallelFor, Kokkos::Cuda> { std::min( (m_rp.m_upper[2] - m_rp.m_lower[2] + block.z - 1) / block.z, maxblocks[2])); + // ensure we don't exceed the capability of the device + check_grid_sizes(grid); CudaParallelLaunch( *this, grid, block, 0, m_rp.space().impl_internal_space_instance()); } else if (RP::rank == 4) { // id0,id1 encoded within threadIdx.x; id2 to threadIdx.y; id3 to // threadIdx.z const dim3 block(m_rp.m_tile[0] * m_rp.m_tile[1], m_rp.m_tile[2], - m_rp.m_tile[3]); - KOKKOS_ASSERT(block.y > 0); - KOKKOS_ASSERT(block.z > 0); + clampZ(m_rp.m_tile[3])); + check_block_sizes(block); const dim3 grid( std::min(m_rp.m_tile_end[0] * m_rp.m_tile_end[1], maxblocks[0]), @@ -143,14 +171,15 @@ class ParallelFor, Kokkos::Cuda> { std::min( (m_rp.m_upper[3] - m_rp.m_lower[3] + block.z - 1) / block.z, maxblocks[2])); + check_grid_sizes(grid); CudaParallelLaunch( *this, grid, block, 0, m_rp.space().impl_internal_space_instance()); } else if (RP::rank == 5) { // id0,id1 encoded within threadIdx.x; id2,id3 to threadIdx.y; id4 to // threadIdx.z const dim3 block(m_rp.m_tile[0] * m_rp.m_tile[1], - m_rp.m_tile[2] * m_rp.m_tile[3], m_rp.m_tile[4]); - KOKKOS_ASSERT(block.z > 0); + m_rp.m_tile[2] * m_rp.m_tile[3], clampZ(m_rp.m_tile[4])); + check_block_sizes(block); const dim3 grid( std::min(m_rp.m_tile_end[0] * m_rp.m_tile_end[1], maxblocks[0]), @@ -159,6 +188,7 @@ class ParallelFor, Kokkos::Cuda> { std::min( (m_rp.m_upper[4] - m_rp.m_lower[4] + block.z - 1) / block.z, maxblocks[2])); + check_grid_sizes(grid); CudaParallelLaunch( *this, grid, block, 0, m_rp.space().impl_internal_space_instance()); } else if (RP::rank == 6) { @@ -166,7 +196,8 @@ class ParallelFor, Kokkos::Cuda> { // threadIdx.z const dim3 block(m_rp.m_tile[0] * m_rp.m_tile[1], m_rp.m_tile[2] * m_rp.m_tile[3], - m_rp.m_tile[4] * m_rp.m_tile[5]); + clampZ(m_rp.m_tile[4] * m_rp.m_tile[5])); + check_block_sizes(block); const dim3 grid( std::min(m_rp.m_tile_end[0] * m_rp.m_tile_end[1], maxblocks[0]), @@ -174,6 +205,7 @@ class ParallelFor, Kokkos::Cuda> { maxblocks[1]), std::min(m_rp.m_tile_end[4] * m_rp.m_tile_end[5], maxblocks[2])); + check_grid_sizes(grid); CudaParallelLaunch( *this, grid, block, 0, m_rp.space().impl_internal_space_instance()); } else { diff --git a/lib/kokkos/core/src/Cuda/Kokkos_Cuda_Parallel_Range.hpp b/lib/kokkos/core/src/Cuda/Kokkos_Cuda_Parallel_Range.hpp index 334834938a..8251fcb248 100644 --- a/lib/kokkos/core/src/Cuda/Kokkos_Cuda_Parallel_Range.hpp +++ b/lib/kokkos/core/src/Cuda/Kokkos_Cuda_Parallel_Range.hpp @@ -48,7 +48,7 @@ class ParallelFor, Kokkos::Cuda> { const FunctorType m_functor; const Policy m_policy; - ParallelFor() = delete; + ParallelFor() = delete; ParallelFor& operator=(const ParallelFor&) = delete; template diff --git a/lib/kokkos/core/src/Cuda/Kokkos_Cuda_Parallel_Team.hpp b/lib/kokkos/core/src/Cuda/Kokkos_Cuda_Parallel_Team.hpp index 71e7751821..a2955e3ab6 100644 --- a/lib/kokkos/core/src/Cuda/Kokkos_Cuda_Parallel_Team.hpp +++ b/lib/kokkos/core/src/Cuda/Kokkos_Cuda_Parallel_Team.hpp @@ -539,9 +539,14 @@ class ParallelFor, m_vector_size(arg_policy.impl_vector_length()) { auto internal_space_instance = m_policy.space().impl_internal_space_instance(); - m_team_size = m_team_size >= 0 ? m_team_size - : arg_policy.team_size_recommended( - arg_functor, ParallelForTag()); + if (m_team_size < 0) { + m_team_size = + arg_policy.team_size_recommended(arg_functor, ParallelForTag()); + if (m_team_size <= 0) + Kokkos::Impl::throw_runtime_exception( + "Kokkos::Impl::ParallelFor could not find a " + "valid execution configuration."); + } m_shmem_begin = (sizeof(double) * (m_team_size + 2)); m_shmem_size = @@ -631,7 +636,7 @@ class ParallelReduce word_count(m_functor_reducer.get_reducer().value_size() / sizeof(word_size_type)); - reference_type value = m_functor_reducer.get_reducer().init( - kokkos_impl_cuda_shared_memory() + - threadIdx.y * word_count.value); + reference_type value = + m_functor_reducer.get_reducer().init(reinterpret_cast( + kokkos_impl_cuda_shared_memory() + + threadIdx.y * word_count.value)); // Iterate this block through the league const int int_league_size = (int)m_league_size; @@ -895,11 +901,16 @@ class ParallelReduce= 0 ? m_team_size - : arg_policy.team_size_recommended( - arg_functor_reducer.get_functor(), - arg_functor_reducer.get_reducer(), - ParallelReduceTag()); + + if (m_team_size < 0) { + m_team_size = arg_policy.team_size_recommended( + arg_functor_reducer.get_functor(), arg_functor_reducer.get_reducer(), + ParallelReduceTag()); + if (m_team_size <= 0) + Kokkos::Impl::throw_runtime_exception( + "Kokkos::Impl::ParallelReduce could not find a " + "valid execution configuration."); + } m_team_begin = UseShflReduction diff --git a/lib/kokkos/core/src/Cuda/Kokkos_Cuda_Task.hpp b/lib/kokkos/core/src/Cuda/Kokkos_Cuda_Task.hpp index 86d6d91bbe..5090e84c38 100644 --- a/lib/kokkos/core/src/Cuda/Kokkos_Cuda_Task.hpp +++ b/lib/kokkos/core/src/Cuda/Kokkos_Cuda_Task.hpp @@ -31,6 +31,9 @@ //---------------------------------------------------------------------------- +// We allow using deprecated classes in this file +KOKKOS_IMPL_DISABLE_DEPRECATED_WARNINGS_PUSH() + #if defined(__CUDA_ARCH__) #define KOKKOS_IMPL_CUDA_SYNCWARP_OR_RETURN(MSG) \ { \ @@ -584,9 +587,9 @@ class TaskExec { private: enum : int { WarpSize = Kokkos::Impl::CudaTraits::WarpSize }; - TaskExec(TaskExec&&) = delete; - TaskExec(TaskExec const&) = delete; - TaskExec& operator=(TaskExec&&) = delete; + TaskExec(TaskExec&&) = delete; + TaskExec(TaskExec const&) = delete; + TaskExec& operator=(TaskExec&&) = delete; TaskExec& operator=(TaskExec const&) = delete; friend class Kokkos::Impl::TaskQueue< @@ -1224,5 +1227,7 @@ KOKKOS_INLINE_FUNCTION void single( #undef KOKKOS_IMPL_CUDA_SYNCWARP_OR_RETURN +KOKKOS_IMPL_DISABLE_DEPRECATED_WARNINGS_POP() + #endif /* #if defined( KOKKOS_ENABLE_TASKDAG ) */ #endif /* #ifndef KOKKOS_IMPL_CUDA_TASK_HPP */ diff --git a/lib/kokkos/core/src/Cuda/Kokkos_Cuda_Team.hpp b/lib/kokkos/core/src/Cuda/Kokkos_Cuda_Team.hpp index c2b5f1fa78..aec692c2c3 100644 --- a/lib/kokkos/core/src/Cuda/Kokkos_Cuda_Team.hpp +++ b/lib/kokkos/core/src/Cuda/Kokkos_Cuda_Team.hpp @@ -184,24 +184,37 @@ class CudaTeamMember { * ( 1 == blockDim.z ) */ template - KOKKOS_INLINE_FUNCTION std::enable_if_t::value> + KOKKOS_INLINE_FUNCTION std::enable_if_t> team_reduce(ReducerType const& reducer) const noexcept { team_reduce(reducer, reducer.reference()); } template - KOKKOS_INLINE_FUNCTION std::enable_if_t::value> + KOKKOS_INLINE_FUNCTION std::enable_if_t> team_reduce(ReducerType const& reducer, typename ReducerType::value_type& value) const noexcept { (void)reducer; (void)value; + + KOKKOS_IF_ON_DEVICE(( + typename Impl::FunctorAnalysis< + Impl::FunctorPatternInterface::REDUCE, TeamPolicy, + ReducerType, typename ReducerType::value_type>::Reducer + wrapped_reducer(reducer); + + impl_team_reduce(wrapped_reducer, value); reducer.reference() = value;)) + } + + template + KOKKOS_INLINE_FUNCTION std::enable_if_t> + impl_team_reduce( + WrappedReducerType const& wrapped_reducer, + typename WrappedReducerType::value_type& value) const noexcept { + (void)wrapped_reducer; + (void)value; + KOKKOS_IF_ON_DEVICE( - (typename Impl::FunctorAnalysis< - Impl::FunctorPatternInterface::REDUCE, TeamPolicy, - ReducerType, typename ReducerType::value_type>::Reducer - wrapped_reducer(reducer); - cuda_intra_block_reduction(value, wrapped_reducer, blockDim.y); - reducer.reference() = value;)) + (cuda_intra_block_reduction(value, wrapped_reducer, blockDim.y);)) } //-------------------------------------------------------------------------- @@ -260,23 +273,42 @@ class CudaTeamMember { //---------------------------------------- template - KOKKOS_INLINE_FUNCTION static std::enable_if_t::value> + KOKKOS_INLINE_FUNCTION static std::enable_if_t> vector_reduce(ReducerType const& reducer) { vector_reduce(reducer, reducer.reference()); } template - KOKKOS_INLINE_FUNCTION static std::enable_if_t::value> + KOKKOS_INLINE_FUNCTION static std::enable_if_t> vector_reduce(ReducerType const& reducer, typename ReducerType::value_type& value) { (void)reducer; (void)value; + + KOKKOS_IF_ON_DEVICE( + (typename Impl::FunctorAnalysis< + Impl::FunctorPatternInterface::REDUCE, TeamPolicy, + ReducerType, typename ReducerType::value_type>::Reducer + wrapped_reducer(reducer); + + impl_vector_reduce(wrapped_reducer, value); + reducer.reference() = value;)) + } + + template + KOKKOS_INLINE_FUNCTION static std::enable_if_t< + is_reducer_v> + impl_vector_reduce(WrappedReducerType const& wrapped_reducer, + typename WrappedReducerType::value_type& value) { + (void)wrapped_reducer; + (void)value; + KOKKOS_IF_ON_DEVICE( (if (blockDim.x == 1) return; // Intra vector lane shuffle reduction: - typename ReducerType::value_type tmp(value); - typename ReducerType::value_type tmp2 = tmp; + typename WrappedReducerType::value_type tmp(value); + typename WrappedReducerType::value_type tmp2 = tmp; unsigned mask = blockDim.x == 32 @@ -287,7 +319,7 @@ class CudaTeamMember { for (int i = blockDim.x; (i >>= 1);) { Impl::in_place_shfl_down(tmp2, tmp, i, blockDim.x, mask); if ((int)threadIdx.x < i) { - reducer.join(tmp, tmp2); + wrapped_reducer.join(&tmp, &tmp2); } } @@ -297,7 +329,7 @@ class CudaTeamMember { // and thus different threads could have different results. Impl::in_place_shfl(tmp2, tmp, 0, blockDim.x, mask); - value = tmp2; reducer.reference() = tmp2;)) + value = tmp2;)) } //---------------------------------------- @@ -487,14 +519,21 @@ parallel_reduce(const Impl::TeamThreadRangeBoundariesStruct< iType, Impl::CudaTeamMember>& loop_boundaries, const Closure& closure, const ReducerType& reducer) { KOKKOS_IF_ON_DEVICE( - (typename ReducerType::value_type value; + (using value_type = typename ReducerType::value_type; + using functor_analysis_type = typename Impl::FunctorAnalysis< + Impl::FunctorPatternInterface::REDUCE, + TeamPolicy, + ReducerType, value_type>; + using wrapped_reducer_type = typename functor_analysis_type::Reducer; - reducer.init(value); + wrapped_reducer_type wrapped_reducer(reducer); value_type value; + wrapped_reducer.init(&value); for (iType i = loop_boundaries.start + threadIdx.y; i < loop_boundaries.end; i += blockDim.y) { closure(i, value); } - loop_boundaries.member.team_reduce(reducer, value);)) + loop_boundaries.member.impl_team_reduce(wrapped_reducer, value); + wrapped_reducer.final(&value); reducer.reference() = value;)) // Avoid bogus warning about reducer value being uninitialized with combined // reducers KOKKOS_IF_ON_HOST(((void)loop_boundaries; (void)closure; @@ -518,16 +557,25 @@ parallel_reduce(const Impl::TeamThreadRangeBoundariesStruct< (void)loop_boundaries; (void)closure; (void)result; - KOKKOS_IF_ON_DEVICE( - (ValueType val; Kokkos::Sum reducer(val); - reducer.init(reducer.reference()); + KOKKOS_IF_ON_DEVICE( + (using functor_analysis_type = typename Impl::FunctorAnalysis< + Impl::FunctorPatternInterface::REDUCE, + TeamPolicy, Closure, + ValueType>; + using wrapped_reducer_type = typename functor_analysis_type::Reducer; + using value_type = typename wrapped_reducer_type::value_type; + + wrapped_reducer_type wrapped_reducer(closure); value_type value{}; + wrapped_reducer.init(&value); for (iType i = loop_boundaries.start + threadIdx.y; - i < loop_boundaries.end; i += blockDim.y) { closure(i, val); } + i < loop_boundaries.end; i += blockDim.y) { closure(i, value); } - loop_boundaries.member.team_reduce(reducer, val); - result = reducer.reference();)) + loop_boundaries.member.impl_team_reduce(wrapped_reducer, value); + wrapped_reducer.final(&value); result = value; + + )) } template @@ -548,16 +596,27 @@ KOKKOS_INLINE_FUNCTION std::enable_if_t::value> parallel_reduce(const Impl::TeamVectorRangeBoundariesStruct< iType, Impl::CudaTeamMember>& loop_boundaries, const Closure& closure, const ReducerType& reducer) { - KOKKOS_IF_ON_DEVICE((typename ReducerType::value_type value; - reducer.init(value); + KOKKOS_IF_ON_DEVICE( + (using value_type = typename ReducerType::value_type; + using functor_analysis_type = typename Impl::FunctorAnalysis< + Impl::FunctorPatternInterface::REDUCE, + TeamPolicy, + ReducerType, value_type>; + using wrapped_reducer_type = typename functor_analysis_type::Reducer; - for (iType i = loop_boundaries.start + - threadIdx.y * blockDim.x + threadIdx.x; - i < loop_boundaries.end; - i += blockDim.y * blockDim.x) { closure(i, value); } + wrapped_reducer_type wrapped_reducer(reducer); value_type value; + wrapped_reducer.init(&value); + + for (iType i = + loop_boundaries.start + threadIdx.y * blockDim.x + threadIdx.x; + i < loop_boundaries.end; + i += blockDim.y * blockDim.x) { closure(i, value); } + + loop_boundaries.member.impl_vector_reduce(wrapped_reducer, value); + loop_boundaries.member.impl_team_reduce(wrapped_reducer, value); + + wrapped_reducer.final(&value); reducer.reference() = value;)) - loop_boundaries.member.vector_reduce(reducer, value); - loop_boundaries.member.team_reduce(reducer, value);)) // Avoid bogus warning about reducer value being uninitialized with combined // reducers KOKKOS_IF_ON_HOST(((void)loop_boundaries; (void)closure; @@ -573,18 +632,27 @@ parallel_reduce(const Impl::TeamVectorRangeBoundariesStruct< (void)loop_boundaries; (void)closure; (void)result; - KOKKOS_IF_ON_DEVICE((ValueType val; Kokkos::Sum reducer(val); - reducer.init(reducer.reference()); + KOKKOS_IF_ON_DEVICE( + (using functor_analysis_type = typename Impl::FunctorAnalysis< + Impl::FunctorPatternInterface::REDUCE, + TeamPolicy, Closure, + ValueType>; + using wrapped_reducer_type = typename functor_analysis_type::Reducer; + using value_type = typename wrapped_reducer_type::value_type; - for (iType i = loop_boundaries.start + - threadIdx.y * blockDim.x + threadIdx.x; - i < loop_boundaries.end; - i += blockDim.y * blockDim.x) { closure(i, val); } + wrapped_reducer_type wrapped_reducer(closure); value_type value; + wrapped_reducer.init(&value); - loop_boundaries.member.vector_reduce(reducer); - loop_boundaries.member.team_reduce(reducer); - result = reducer.reference();)) + for (iType i = + loop_boundaries.start + threadIdx.y * blockDim.x + threadIdx.x; + i < loop_boundaries.end; + i += blockDim.y * blockDim.x) { closure(i, value); } + + loop_boundaries.member.impl_vector_reduce(wrapped_reducer, value); + loop_boundaries.member.impl_team_reduce(wrapped_reducer, value); + + wrapped_reducer.final(&value); result = value;)) } //---------------------------------------------------------------------------- @@ -632,13 +700,22 @@ parallel_reduce(Impl::ThreadVectorRangeBoundariesStruct< Closure const& closure, ReducerType const& reducer) { KOKKOS_IF_ON_DEVICE(( - reducer.init(reducer.reference()); + using value_type = typename ReducerType::value_type; + using functor_analysis_type = typename Impl::FunctorAnalysis< + Impl::FunctorPatternInterface::REDUCE, + TeamPolicy, + ReducerType, value_type>; + using wrapped_reducer_type = typename functor_analysis_type::Reducer; + + wrapped_reducer_type wrapped_reducer(reducer); value_type value; + wrapped_reducer.init(&value); for (iType i = loop_boundaries.start + threadIdx.x; - i < loop_boundaries.end; - i += blockDim.x) { closure(i, reducer.reference()); } + i < loop_boundaries.end; i += blockDim.x) { closure(i, value); } - Impl::CudaTeamMember::vector_reduce(reducer); + Impl::CudaTeamMember::impl_vector_reduce(wrapped_reducer, value); + + wrapped_reducer.final(&value); reducer.reference() = value; )) // Avoid bogus warning about reducer value being uninitialized with combined @@ -667,15 +744,26 @@ parallel_reduce(Impl::ThreadVectorRangeBoundariesStruct< (void)loop_boundaries; (void)closure; (void)result; - KOKKOS_IF_ON_DEVICE( - (result = ValueType(); - for (iType i = loop_boundaries.start + threadIdx.x; - i < loop_boundaries.end; i += blockDim.x) { closure(i, result); } + KOKKOS_IF_ON_DEVICE(( - Impl::CudaTeamMember::vector_reduce(Kokkos::Sum(result)); + using functor_analysis_type = typename Impl::FunctorAnalysis< + Impl::FunctorPatternInterface::REDUCE, + TeamPolicy, Closure, + ValueType>; + using wrapped_reducer_type = typename functor_analysis_type::Reducer; + using value_type = typename wrapped_reducer_type::value_type; - )) + wrapped_reducer_type wrapped_reducer(closure); value_type value; + wrapped_reducer.init(&value); + + for (iType i = loop_boundaries.start + threadIdx.x; + i < loop_boundaries.end; i += blockDim.x) { closure(i, value); } + + Impl::CudaTeamMember::impl_vector_reduce(wrapped_reducer, value); + wrapped_reducer.final(&value); result = value; + + )) } //---------------------------------------------------------------------------- diff --git a/lib/kokkos/core/src/Cuda/Kokkos_Cuda_Vectorization.hpp b/lib/kokkos/core/src/Cuda/Kokkos_Cuda_Vectorization.hpp index a3f4f2f4cc..9e0c5819f7 100644 --- a/lib/kokkos/core/src/Cuda/Kokkos_Cuda_Vectorization.hpp +++ b/lib/kokkos/core/src/Cuda/Kokkos_Cuda_Vectorization.hpp @@ -125,8 +125,8 @@ struct in_place_shfl_op { struct in_place_shfl_fn : in_place_shfl_op { template __device__ KOKKOS_IMPL_FORCEINLINE T do_shfl_op(unsigned mask, T& val, - int lane, int width) const - noexcept { + int lane, + int width) const noexcept { (void)mask; (void)val; (void)lane; @@ -136,28 +136,28 @@ struct in_place_shfl_fn : in_place_shfl_op { }; template __device__ KOKKOS_IMPL_FORCEINLINE void in_place_shfl(Args&&... args) noexcept { - in_place_shfl_fn{}((Args &&) args...); + in_place_shfl_fn{}((Args&&)args...); } struct in_place_shfl_up_fn : in_place_shfl_op { template __device__ KOKKOS_IMPL_FORCEINLINE T do_shfl_op(unsigned mask, T& val, - int lane, int width) const - noexcept { + int lane, + int width) const noexcept { return __shfl_up_sync(mask, val, lane, width); } }; template __device__ KOKKOS_IMPL_FORCEINLINE void in_place_shfl_up( Args&&... args) noexcept { - in_place_shfl_up_fn{}((Args &&) args...); + in_place_shfl_up_fn{}((Args&&)args...); } struct in_place_shfl_down_fn : in_place_shfl_op { template __device__ KOKKOS_IMPL_FORCEINLINE T do_shfl_op(unsigned mask, T& val, - int lane, int width) const - noexcept { + int lane, + int width) const noexcept { (void)mask; (void)val; (void)lane; @@ -168,7 +168,7 @@ struct in_place_shfl_down_fn : in_place_shfl_op { template __device__ KOKKOS_IMPL_FORCEINLINE void in_place_shfl_down( Args&&... args) noexcept { - in_place_shfl_down_fn{}((Args &&) args...); + in_place_shfl_down_fn{}((Args&&)args...); } } // namespace Impl diff --git a/lib/kokkos/core/src/Cuda/Kokkos_Cuda_ZeroMemset.hpp b/lib/kokkos/core/src/Cuda/Kokkos_Cuda_ZeroMemset.hpp index 517c592af7..0ac2d4052d 100644 --- a/lib/kokkos/core/src/Cuda/Kokkos_Cuda_ZeroMemset.hpp +++ b/lib/kokkos/core/src/Cuda/Kokkos_Cuda_ZeroMemset.hpp @@ -23,15 +23,12 @@ namespace Kokkos { namespace Impl { -template -struct ZeroMemset> { - ZeroMemset(const Kokkos::Cuda& exec_space_instance, - const View& dst) { +template <> +struct ZeroMemset { + ZeroMemset(const Kokkos::Cuda& exec_space_instance, void* dst, size_t cnt) { KOKKOS_IMPL_CUDA_SAFE_CALL( (exec_space_instance.impl_internal_space_instance() - ->cuda_memset_async_wrapper( - dst.data(), 0, - dst.size() * sizeof(typename View::value_type)))); + ->cuda_memset_async_wrapper(dst, 0, cnt))); } }; diff --git a/lib/kokkos/core/src/HIP/Kokkos_HIP.cpp b/lib/kokkos/core/src/HIP/Kokkos_HIP.cpp index aced2083ff..8de3a8758f 100644 --- a/lib/kokkos/core/src/HIP/Kokkos_HIP.cpp +++ b/lib/kokkos/core/src/HIP/Kokkos_HIP.cpp @@ -27,6 +27,8 @@ #include +#include + namespace Kokkos { #ifdef KOKKOS_ENABLE_DEPRECATED_CODE_4 @@ -49,34 +51,44 @@ void HIP::impl_initialize(InitializationSettings const& settings) { Impl::HIPInternal::m_hipDev = hip_device_id; KOKKOS_IMPL_HIP_SAFE_CALL( hipGetDeviceProperties(&Impl::HIPInternal::m_deviceProp, hip_device_id)); - const auto& hipProp = Impl::HIPInternal::m_deviceProp; KOKKOS_IMPL_HIP_SAFE_CALL(hipSetDevice(hip_device_id)); - // number of multiprocessors - Impl::HIPInternal::m_multiProcCount = hipProp.multiProcessorCount; - - //---------------------------------- - // Maximum number of warps, - // at most one warp per thread in a warp for reduction. - Impl::HIPInternal::m_maxWarpCount = - hipProp.maxThreadsPerBlock / Impl::HIPTraits::WarpSize; - if (Impl::HIPTraits::WarpSize < Impl::HIPInternal::m_maxWarpCount) { - Impl::HIPInternal::m_maxWarpCount = Impl::HIPTraits::WarpSize; + // Check that we are running on the expected architecture. We print a warning + // instead of erroring out because AMD does not guarantee that gcnArchName + // will always contain the gfx flag. + if (Kokkos::show_warnings()) { + if (std::string_view arch_name = + Impl::HIPInternal::m_deviceProp.gcnArchName; + arch_name.find(KOKKOS_ARCH_AMD_GPU) != 0) { + std::cerr + << "Kokkos::HIP::initialize WARNING: running kernels compiled for " + << KOKKOS_ARCH_AMD_GPU << " on " << arch_name << " device.\n"; + } } - //---------------------------------- - // Maximum number of blocks - Impl::HIPInternal::m_maxBlock[0] = hipProp.maxGridSize[0]; - Impl::HIPInternal::m_maxBlock[1] = hipProp.maxGridSize[1]; - Impl::HIPInternal::m_maxBlock[2] = hipProp.maxGridSize[2]; + // Print a warning if the user did not select the right GFX942 architecture +#ifdef KOKKOS_ARCH_AMD_GFX942 + if ((Kokkos::show_warnings()) && + (Impl::HIPInternal::m_deviceProp.integrated == 1)) { + std::cerr << "Kokkos::HIP::initialize WARNING: running kernels for MI300X " + "(discrete GPU) on a MI300A (APU).\n"; + } +#endif +#ifdef KOKKOS_ARCH_AMD_GFX942_APU + if ((Kokkos::show_warnings()) && + (Impl::HIPInternal::m_deviceProp.integrated == 0)) { + std::cerr << "Kokkos::HIP::initialize WARNING: running kernels for MI300A " + "(APU) on a MI300X (discrete GPU).\n"; + } +#endif - // theoretically, we can get 40 WF's / CU, but only can sustain 32 see - // https://github.com/ROCm-Developer-Tools/HIP/blob/a0b5dfd625d99af7e288629747b40dd057183173/vdi/hip_platform.cpp#L742 - Impl::HIPInternal::m_maxWavesPerCU = 32; - Impl::HIPInternal::m_shmemPerSM = hipProp.maxSharedMemoryPerMultiProcessor; - Impl::HIPInternal::m_maxShmemPerBlock = hipProp.sharedMemPerBlock; + // theoretically on GFX 9XX GPUs, we can get 40 WF's / CU, but only can + // sustain 32 see + // https://github.com/ROCm/clr/blob/4d0b815d06751735e6a50fa46e913fdf85f751f0/hipamd/src/hip_platform.cpp#L362-L366 + const int maxWavesPerCU = + Impl::HIPInternal::m_deviceProp.major <= 9 ? 32 : 64; Impl::HIPInternal::m_maxThreadsPerSM = - Impl::HIPInternal::m_maxWavesPerCU * Impl::HIPTraits::WarpSize; + maxWavesPerCU * Impl::HIPTraits::WarpSize; // Init the array for used for arbitrarily sized atomics desul::Impl::init_lock_arrays(); // FIXME @@ -146,10 +158,6 @@ void HIP::print_configuration(std::ostream& os, bool /*verbose*/) const { #else os << "no\n"; #endif -#ifdef KOKKOS_ENABLE_IMPL_HIP_UNIFIED_MEMORY - os << " KOKKOS_ENABLE_IMPL_HIP_UNIFIED_MEMORY: "; - os << "yes\n"; -#endif os << "\nRuntime Configuration:\n"; diff --git a/lib/kokkos/core/src/HIP/Kokkos_HIP_BlockSize_Deduction.hpp b/lib/kokkos/core/src/HIP/Kokkos_HIP_BlockSize_Deduction.hpp index 1f084c41e5..90e5cf7355 100644 --- a/lib/kokkos/core/src/HIP/Kokkos_HIP_BlockSize_Deduction.hpp +++ b/lib/kokkos/core/src/HIP/Kokkos_HIP_BlockSize_Deduction.hpp @@ -113,8 +113,9 @@ unsigned hip_internal_get_block_size(const HIPInternal *hip_instance, const unsigned min_waves_per_eu = LaunchBounds::minBperSM ? LaunchBounds::minBperSM : 1; const unsigned min_threads_per_sm = min_waves_per_eu * HIPTraits::WarpSize; - const unsigned shmem_per_sm = hip_instance->m_shmemPerSM; - unsigned block_size = tperb_reg; + const unsigned shmem_per_sm = + hip_instance->m_deviceProp.maxSharedMemoryPerMultiProcessor; + unsigned block_size = tperb_reg; do { unsigned total_shmem = f(block_size); // find how many threads we can fit with this blocksize based on LDS usage diff --git a/lib/kokkos/core/src/HIP/Kokkos_HIP_GraphNodeKernel.hpp b/lib/kokkos/core/src/HIP/Kokkos_HIP_GraphNodeKernel.hpp index 5f0df72df1..584cc63d95 100644 --- a/lib/kokkos/core/src/HIP/Kokkos_HIP_GraphNodeKernel.hpp +++ b/lib/kokkos/core/src/HIP/Kokkos_HIP_GraphNodeKernel.hpp @@ -44,22 +44,17 @@ class GraphNodeKernelImpl // TODO use the name and executionspace template - GraphNodeKernelImpl(std::string, Kokkos::HIP const&, Functor arg_functor, + GraphNodeKernelImpl(std::string label_, HIP const&, Functor arg_functor, PolicyDeduced&& arg_policy, ArgsDeduced&&... args) - : base_t(std::move(arg_functor), (PolicyDeduced &&) arg_policy, - (ArgsDeduced &&) args...) {} + : base_t(std::move(arg_functor), (PolicyDeduced&&)arg_policy, + (ArgsDeduced&&)args...), + label(std::move(label_)) {} template GraphNodeKernelImpl(Kokkos::HIP const& exec_space, Functor arg_functor, PolicyDeduced&& arg_policy) - : GraphNodeKernelImpl("", exec_space, std::move(arg_functor), - (PolicyDeduced &&) arg_policy) {} - - ~GraphNodeKernelImpl() { - if (m_driver_storage) { - Kokkos::HIPSpace().deallocate(m_driver_storage, sizeof(base_t)); - } - } + : GraphNodeKernelImpl("[unlabeled]", exec_space, std::move(arg_functor), + (PolicyDeduced&&)arg_policy) {} void set_hip_graph_ptr(hipGraph_t* arg_graph_ptr) { m_graph_ptr = arg_graph_ptr; @@ -73,18 +68,29 @@ class GraphNodeKernelImpl hipGraph_t const* get_hip_graph_ptr() const { return m_graph_ptr; } - Kokkos::ObservingRawPtr allocate_driver_memory_buffer() const { + Kokkos::ObservingRawPtr allocate_driver_memory_buffer( + const HIP& exec) const { KOKKOS_EXPECTS(m_driver_storage == nullptr); - m_driver_storage = static_cast(Kokkos::HIPSpace().allocate( - "GraphNodeKernel global memory functor storage", sizeof(base_t))); + std::string alloc_label = + label + " - GraphNodeKernel global memory functor storage"; + m_driver_storage = std::shared_ptr( + static_cast( + HIPSpace().allocate(exec, alloc_label.c_str(), sizeof(base_t))), + // FIXME_HIP Custom deletor should use same 'exec' as for allocation. + [alloc_label](base_t* ptr) { + HIPSpace().deallocate(alloc_label.c_str(), ptr, sizeof(base_t)); + }); KOKKOS_ENSURES(m_driver_storage != nullptr); - return m_driver_storage; + return m_driver_storage.get(); } + auto get_driver_storage() const { return m_driver_storage; } + private: Kokkos::ObservingRawPtr m_graph_ptr = nullptr; Kokkos::ObservingRawPtr m_graph_node_ptr = nullptr; - Kokkos::OwningRawPtr m_driver_storage = nullptr; + mutable std::shared_ptr m_driver_storage = nullptr; + std::string label; }; struct HIPGraphNodeAggregateKernel { @@ -114,13 +120,14 @@ struct get_graph_node_kernel_type Kokkos::ParallelReduceTag>> {}; template -auto* allocate_driver_storage_for_kernel(KernelType const& kernel) { +auto* allocate_driver_storage_for_kernel(const HIP& exec, + KernelType const& kernel) { using graph_node_kernel_t = typename get_graph_node_kernel_type::type; auto const& kernel_as_graph_kernel = static_cast(kernel); - return kernel_as_graph_kernel.allocate_driver_memory_buffer(); + return kernel_as_graph_kernel.allocate_driver_memory_buffer(exec); } template diff --git a/lib/kokkos/core/src/HIP/Kokkos_HIP_Graph_Impl.hpp b/lib/kokkos/core/src/HIP/Kokkos_HIP_Graph_Impl.hpp index a0989fe671..4f97214ca6 100644 --- a/lib/kokkos/core/src/HIP/Kokkos_HIP_Graph_Impl.hpp +++ b/lib/kokkos/core/src/HIP/Kokkos_HIP_Graph_Impl.hpp @@ -42,11 +42,11 @@ class GraphImpl { // Not movable or copyable; it spends its whole life as a shared_ptr in the // Graph object. - GraphImpl() = delete; - GraphImpl(GraphImpl const&) = delete; - GraphImpl(GraphImpl&&) = delete; + GraphImpl() = delete; + GraphImpl(GraphImpl const&) = delete; + GraphImpl(GraphImpl&&) = delete; GraphImpl& operator=(GraphImpl const&) = delete; - GraphImpl& operator=(GraphImpl&&) = delete; + GraphImpl& operator=(GraphImpl&&) = delete; ~GraphImpl(); @@ -60,7 +60,7 @@ class GraphImpl { template void add_predecessor(NodeImplPtr arg_node_ptr, PredecessorRef arg_pred_ref); - void submit(); + void submit(const Kokkos::HIP& exec); Kokkos::HIP const& get_execution_space() const noexcept; @@ -69,18 +69,28 @@ class GraphImpl { template auto create_aggregate_ptr(PredecessorRefs&&...); - private: - void instantiate_graph() { + void instantiate() { + KOKKOS_EXPECTS(!m_graph_exec); constexpr size_t error_log_size = 256; hipGraphNode_t error_node = nullptr; char error_log[error_log_size]; KOKKOS_IMPL_HIP_SAFE_CALL(hipGraphInstantiate( &m_graph_exec, m_graph, &error_node, error_log, error_log_size)); + KOKKOS_ENSURES(m_graph_exec); } + hipGraph_t hip_graph() { return m_graph; } + hipGraphExec_t hip_graph_exec() { return m_graph_exec; } + + private: Kokkos::HIP m_execution_space; hipGraph_t m_graph = nullptr; hipGraphExec_t m_graph_exec = nullptr; + + // Store drivers for the kernel nodes that launch in global memory. + // This is required as lifetime of drivers must be bounded to this instance's + // lifetime. + std::vector> m_driver_storage; }; inline GraphImpl::~GraphImpl() { @@ -123,6 +133,8 @@ inline void GraphImpl::add_node( kernel.set_hip_graph_node_ptr(&node); kernel.execute(); KOKKOS_ENSURES(node); + if (std::shared_ptr tmp = kernel.get_driver_storage()) + m_driver_storage.push_back(std::move(tmp)); } // Requires PredecessorRef is a specialization of GraphNodeRef that has @@ -145,16 +157,15 @@ inline void GraphImpl::add_predecessor( hipGraphAddDependencies(m_graph, &pred_node, &node, 1)); } -inline void GraphImpl::submit() { +inline void GraphImpl::submit(const Kokkos::HIP& exec) { if (!m_graph_exec) { - instantiate_graph(); + instantiate(); } - KOKKOS_IMPL_HIP_SAFE_CALL( - hipGraphLaunch(m_graph_exec, m_execution_space.hip_stream())); + KOKKOS_IMPL_HIP_SAFE_CALL(hipGraphLaunch(m_graph_exec, exec.hip_stream())); } -inline Kokkos::HIP const& GraphImpl::get_execution_space() const - noexcept { +inline Kokkos::HIP const& GraphImpl::get_execution_space() + const noexcept { return m_execution_space; } diff --git a/lib/kokkos/core/src/HIP/Kokkos_HIP_Instance.cpp b/lib/kokkos/core/src/HIP/Kokkos_HIP_Instance.cpp index e0b25c6939..54e8c315e3 100644 --- a/lib/kokkos/core/src/HIP/Kokkos_HIP_Instance.cpp +++ b/lib/kokkos/core/src/HIP/Kokkos_HIP_Instance.cpp @@ -77,7 +77,8 @@ std::size_t scratch_count(const std::size_t size) { //---------------------------------------------------------------------------- int HIPInternal::concurrency() { - static int const concurrency = m_maxThreadsPerSM * m_multiProcCount; + static int const concurrency = + m_maxThreadsPerSM * m_deviceProp.multiProcessorCount; return concurrency; } @@ -97,6 +98,13 @@ void HIPInternal::print_configuration(std::ostream &s) const { << "undefined\n"; #endif + s << "macro KOKKOS_ENABLE_IMPL_HIP_MALLOC_ASYNC: "; +#ifdef KOKKOS_ENABLE_IMPL_HIP_MALLOC_ASYNC + s << "yes\n"; +#else + s << "no\n"; +#endif + for (int i : get_visible_devices()) { hipDeviceProp_t hipProp; KOKKOS_IMPL_HIP_SAFE_CALL(hipGetDeviceProperties(&hipProp, i)); @@ -177,8 +185,16 @@ void HIPInternal::initialize(hipStream_t stream) { // and scratch space for partial reduction values. // Allocate some initial space. This will grow as needed. { + // Maximum number of warps, + // at most one warp per thread in a warp for reduction. + unsigned int maxWarpCount = + m_deviceProp.maxThreadsPerBlock / Impl::HIPTraits::WarpSize; + if (Impl::HIPTraits::WarpSize < maxWarpCount) { + maxWarpCount = Impl::HIPTraits::WarpSize; + } + const unsigned reduce_block_count = - m_maxWarpCount * Impl::HIPTraits::WarpSize; + maxWarpCount * Impl::HIPTraits::WarpSize; (void)scratch_flags(reduce_block_count * 2 * sizeof(size_type)); (void)scratch_space(reduce_block_count * 16 * sizeof(size_type)); @@ -353,14 +369,8 @@ void HIPInternal::finalize() { m_num_scratch_locks = 0; } -int HIPInternal::m_hipDev = -1; -unsigned HIPInternal::m_multiProcCount = 0; -unsigned HIPInternal::m_maxWarpCount = 0; -std::array HIPInternal::m_maxBlock = {0, 0, 0}; -unsigned HIPInternal::m_maxWavesPerCU = 0; -int HIPInternal::m_shmemPerSM = 0; -int HIPInternal::m_maxShmemPerBlock = 0; -int HIPInternal::m_maxThreadsPerSM = 0; +int HIPInternal::m_hipDev = -1; +int HIPInternal::m_maxThreadsPerSM = 0; hipDeviceProp_t HIPInternal::m_deviceProp; @@ -372,15 +382,7 @@ std::mutex HIPInternal::constantMemMutex; //---------------------------------------------------------------------------- Kokkos::HIP::size_type hip_internal_multiprocessor_count() { - return HIPInternal::singleton().m_multiProcCount; -} - -Kokkos::HIP::size_type hip_internal_maximum_warp_count() { - return HIPInternal::singleton().m_maxWarpCount; -} - -std::array hip_internal_maximum_grid_count() { - return HIPInternal::singleton().m_maxBlock; + return HIPInternal::singleton().m_deviceProp.multiProcessorCount; } Kokkos::HIP::size_type *hip_internal_scratch_space(const HIP &instance, diff --git a/lib/kokkos/core/src/HIP/Kokkos_HIP_Instance.hpp b/lib/kokkos/core/src/HIP/Kokkos_HIP_Instance.hpp index 437a84253f..d8043dc23d 100644 --- a/lib/kokkos/core/src/HIP/Kokkos_HIP_Instance.hpp +++ b/lib/kokkos/core/src/HIP/Kokkos_HIP_Instance.hpp @@ -31,7 +31,7 @@ namespace Impl { struct HIPTraits { #if defined(KOKKOS_ARCH_AMD_GFX906) || defined(KOKKOS_ARCH_AMD_GFX908) || \ defined(KOKKOS_ARCH_AMD_GFX90A) || defined(KOKKOS_ARCH_AMD_GFX940) || \ - defined(KOKKOS_ARCH_AMD_GFX942) + defined(KOKKOS_ARCH_AMD_GFX942) || defined(KOKKOS_ARCH_AMD_GFX942_APU) static constexpr int WarpSize = 64; static constexpr int WarpIndexMask = 0x003f; /* hexadecimal for 63 */ static constexpr int WarpIndexShift = 6; /* WarpSize == 1 << WarpShift*/ @@ -52,8 +52,6 @@ struct HIPTraits { //---------------------------------------------------------------------------- -HIP::size_type hip_internal_maximum_warp_count(); -std::array hip_internal_maximum_grid_count(); HIP::size_type hip_internal_multiprocessor_count(); HIP::size_type *hip_internal_scratch_space(const HIP &instance, @@ -72,12 +70,6 @@ class HIPInternal { using size_type = ::Kokkos::HIP::size_type; static int m_hipDev; - static unsigned m_multiProcCount; - static unsigned m_maxWarpCount; - static std::array m_maxBlock; - static unsigned m_maxWavesPerCU; - static int m_shmemPerSM; - static int m_maxShmemPerBlock; static int m_maxThreadsPerSM; static hipDeviceProp_t m_deviceProp; diff --git a/lib/kokkos/core/src/HIP/Kokkos_HIP_KernelLaunch.hpp b/lib/kokkos/core/src/HIP/Kokkos_HIP_KernelLaunch.hpp index 7cd0afcf47..e243eb07e7 100644 --- a/lib/kokkos/core/src/HIP/Kokkos_HIP_KernelLaunch.hpp +++ b/lib/kokkos/core/src/HIP/Kokkos_HIP_KernelLaunch.hpp @@ -25,11 +25,7 @@ #include #include -#if !((HIP_VERSION_MAJOR == 5) && (HIP_VERSION_MINOR == 2)) -#define KOKKOS_IMPL_HIP_GRAPH_ENABLED -#endif - -#ifdef KOKKOS_IMPL_HIP_GRAPH_ENABLED +#ifdef KOKKOS_IMPL_HIP_NATIVE_GRAPH #include #include #endif @@ -173,15 +169,15 @@ struct DeduceHIPLaunchMechanism { static constexpr HIPLaunchMechanism launch_mechanism = ((property & force_global_launch) == force_global_launch) ? HIPLaunchMechanism::GlobalMemory - : ((property & light_weight) == light_weight) - ? (sizeof(DriverType) < HIPTraits::KernelArgumentLimit - ? HIPLaunchMechanism::LocalMemory - : HIPLaunchMechanism::GlobalMemory) - : (((property & heavy_weight) == heavy_weight) - ? (sizeof(DriverType) < HIPTraits::ConstantMemoryUsage - ? HIPLaunchMechanism::ConstantMemory - : HIPLaunchMechanism::GlobalMemory) - : (default_launch_mechanism)); + : ((property & light_weight) == light_weight) + ? (sizeof(DriverType) < HIPTraits::KernelArgumentLimit + ? HIPLaunchMechanism::LocalMemory + : HIPLaunchMechanism::GlobalMemory) + : (((property & heavy_weight) == heavy_weight) + ? (sizeof(DriverType) < HIPTraits::ConstantMemoryUsage + ? HIPLaunchMechanism::ConstantMemory + : HIPLaunchMechanism::GlobalMemory) + : (default_launch_mechanism)); }; template m_stream, ManageStream::no), driver); // Unlike in the non-graph case, we can get away with doing an async copy // here because the `DriverType` instance is held in the GraphNodeImpl // which is guaranteed to be alive until the graph instance itself is // destroyed, where there should be a fence ensuring that the allocation // associated with this kernel on the device side isn't deleted. - hipMemcpyAsync(driver_ptr, &driver, sizeof(DriverType), hipMemcpyDefault, - hip_instance->m_stream); + KOKKOS_IMPL_HIP_SAFE_CALL( + hipMemcpyAsync(driver_ptr, &driver, sizeof(DriverType), + hipMemcpyDefault, hip_instance->m_stream)); void const *args[] = {&driver_ptr}; @@ -551,11 +549,11 @@ struct HIPParallelLaunch< LaunchMechanism>; HIPParallelLaunch(const DriverType &driver, const dim3 &grid, - const dim3 &block, const int shmem, + const dim3 &block, const unsigned int shmem, const HIPInternal *hip_instance, const bool /*prefer_shmem*/) { if ((grid.x != 0) && ((block.x * block.y * block.z) != 0)) { - if (hip_instance->m_maxShmemPerBlock < shmem) { + if (hip_instance->m_deviceProp.sharedMemPerBlock < shmem) { Kokkos::Impl::throw_runtime_exception( "HIPParallelLaunch FAILED: shared memory request is too large"); } @@ -585,7 +583,7 @@ void hip_parallel_launch(const DriverType &driver, const dim3 &grid, const dim3 &block, const int shmem, const HIPInternal *hip_instance, const bool prefer_shmem) { -#ifdef KOKKOS_IMPL_HIP_GRAPH_ENABLED +#ifdef KOKKOS_IMPL_HIP_NATIVE_GRAPH if constexpr (DoGraph) { // Graph launch using base_t = HIPParallelLaunchKernelInvoker, HIP> { const Policy m_policy; public: - ParallelFor() = delete; - ParallelFor(ParallelFor const&) = default; + ParallelFor() = delete; + ParallelFor(ParallelFor const&) = default; ParallelFor& operator=(ParallelFor const&) = delete; inline __device__ void operator()() const { @@ -57,7 +57,7 @@ class ParallelFor, HIP> { inline void execute() const { using ClosureType = ParallelFor; if (m_policy.m_num_tiles == 0) return; - auto const maxblocks = hip_internal_maximum_grid_count(); + auto const maxblocks = m_policy.space().hip_device_prop().maxGridSize; if (Policy::rank == 2) { dim3 const block(m_policy.m_tile[0], m_policy.m_tile[1], 1); dim3 const grid( diff --git a/lib/kokkos/core/src/HIP/Kokkos_HIP_ParallelFor_Range.hpp b/lib/kokkos/core/src/HIP/Kokkos_HIP_ParallelFor_Range.hpp index 9355c1c75f..3985dc60f0 100644 --- a/lib/kokkos/core/src/HIP/Kokkos_HIP_ParallelFor_Range.hpp +++ b/lib/kokkos/core/src/HIP/Kokkos_HIP_ParallelFor_Range.hpp @@ -53,8 +53,8 @@ class ParallelFor, Kokkos::HIP> { public: using functor_type = FunctorType; - ParallelFor() = delete; - ParallelFor(ParallelFor const&) = default; + ParallelFor() = delete; + ParallelFor(ParallelFor const&) = default; ParallelFor& operator=(ParallelFor const&) = delete; inline __device__ void operator()() const { diff --git a/lib/kokkos/core/src/HIP/Kokkos_HIP_ParallelFor_Team.hpp b/lib/kokkos/core/src/HIP/Kokkos_HIP_ParallelFor_Team.hpp index bf0c219338..83e890bce9 100644 --- a/lib/kokkos/core/src/HIP/Kokkos_HIP_ParallelFor_Team.hpp +++ b/lib/kokkos/core/src/HIP/Kokkos_HIP_ParallelFor_Team.hpp @@ -71,8 +71,8 @@ class ParallelFor, HIP> { } public: - ParallelFor() = delete; - ParallelFor(ParallelFor const&) = default; + ParallelFor() = delete; + ParallelFor(ParallelFor const&) = default; ParallelFor& operator=(ParallelFor const&) = delete; __device__ inline void operator()() const { @@ -120,9 +120,14 @@ class ParallelFor, HIP> { m_vector_size(arg_policy.impl_vector_length()) { auto internal_space_instance = m_policy.space().impl_internal_space_instance(); - m_team_size = m_team_size >= 0 ? m_team_size - : arg_policy.team_size_recommended( - arg_functor, ParallelForTag()); + if (m_team_size < 0) { + m_team_size = + arg_policy.team_size_recommended(arg_functor, ParallelForTag()); + if (m_team_size <= 0) + Kokkos::Impl::throw_runtime_exception( + "Kokkos::Impl::ParallelFor could not find a " + "valid execution configuration."); + } m_shmem_begin = (sizeof(double) * (m_team_size + 2)); m_shmem_size = @@ -149,8 +154,9 @@ class ParallelFor, HIP> { static_cast(m_league_size)))); } - int const shmem_size_total = m_shmem_begin + m_shmem_size; - if (internal_space_instance->m_maxShmemPerBlock < shmem_size_total) { + unsigned int const shmem_size_total = m_shmem_begin + m_shmem_size; + if (internal_space_instance->m_deviceProp.sharedMemPerBlock < + shmem_size_total) { Kokkos::Impl::throw_runtime_exception(std::string( "Kokkos::Impl::ParallelFor< HIP > insufficient shared memory")); } diff --git a/lib/kokkos/core/src/HIP/Kokkos_HIP_ParallelReduce_Team.hpp b/lib/kokkos/core/src/HIP/Kokkos_HIP_ParallelReduce_Team.hpp index 0c24e5cc62..fb4ff937cd 100644 --- a/lib/kokkos/core/src/HIP/Kokkos_HIP_ParallelReduce_Team.hpp +++ b/lib/kokkos/core/src/HIP/Kokkos_HIP_ParallelReduce_Team.hpp @@ -46,6 +46,22 @@ class ParallelReduce 4 bytes in size, indexing into shared/global memory relies + // on the block and grid dimensions to ensure that we index at the correct + // offset rather than at every 4 byte word; such that, when the join is + // performed, we have the correct data that was copied over in chunks of 4 + // bytes. + using word_size_type = std::conditional_t< + sizeof(value_type) < sizeof(Kokkos::HIP::size_type), + std::conditional_t, + Kokkos::HIP::size_type>; using reducer_type = ReducerType; using size_type = HIP::size_type; @@ -72,7 +88,7 @@ class ParallelReduce const - word_count(reducer.value_size() / sizeof(size_type)); + integral_nonzero_constant const + word_count(reducer.value_size() / sizeof(word_size_type)); - reference_type value = - reducer.init(kokkos_impl_hip_shared_memory() + - threadIdx.y * word_count.value); + reference_type value = reducer.init(reinterpret_cast( + kokkos_impl_hip_shared_memory() + + threadIdx.y * word_count.value)); // Iterate this block through the league iterate_through_league(threadid, value); // Reduce with final value at blockDim.y - 1 location. bool do_final_reduce = (m_league_size == 0); if (!do_final_reduce) - do_final_reduce = - hip_single_inter_block_reduce_scan( - reducer, blockIdx.x, gridDim.x, - kokkos_impl_hip_shared_memory(), m_scratch_space, - m_scratch_flags); + do_final_reduce = hip_single_inter_block_reduce_scan( + reducer, blockIdx.x, gridDim.x, + kokkos_impl_hip_shared_memory(), m_scratch_space, + m_scratch_flags); if (do_final_reduce) { // This is the final block with the final result at the final threads' // location - size_type* const shared = kokkos_impl_hip_shared_memory() + - (blockDim.y - 1) * word_count.value; - size_type* const global = m_result_ptr_device_accessible - ? reinterpret_cast(m_result_ptr) - : m_scratch_space; + word_size_type* const shared = + kokkos_impl_hip_shared_memory() + + (blockDim.y - 1) * word_count.value; + size_type* const global = + m_result_ptr_device_accessible + ? reinterpret_cast(m_result_ptr) + : m_scratch_space; if (threadIdx.y == 0) { reducer.final(reinterpret_cast(shared)); @@ -227,7 +244,8 @@ class ParallelReduce(m_scratch_space), result, m_scratch_flags, blockDim.y)) { unsigned int const id = threadIdx.y * blockDim.x + threadIdx.x; if (id == 0) { @@ -249,8 +267,9 @@ class ParallelReduce(hip_internal_scratch_space( + m_policy.space(), reducer.value_size() * block_count)); m_scratch_flags = hip_internal_scratch_flags(m_policy.space(), sizeof(size_type)); @@ -306,11 +325,15 @@ class ParallelReduce= 0 ? m_team_size - : arg_policy.team_size_recommended( - arg_functor_reducer.get_functor(), - arg_functor_reducer.get_reducer(), - ParallelReduceTag()); + if (m_team_size < 0) { + m_team_size = arg_policy.team_size_recommended( + arg_functor_reducer.get_functor(), arg_functor_reducer.get_reducer(), + ParallelReduceTag()); + if (m_team_size <= 0) + Kokkos::Impl::throw_runtime_exception( + "Kokkos::Impl::ParallelReduce could not find a " + "valid execution configuration."); + } m_team_begin = UseShflReduction @@ -356,7 +379,8 @@ class ParallelReduce bad team size")); } - if (internal_space_instance->m_maxShmemPerBlock < shmem_size_total) { + if (internal_space_instance->m_deviceProp.sharedMemPerBlock < + shmem_size_total) { Kokkos::Impl::throw_runtime_exception( std::string("Kokkos::Impl::ParallelReduce< HIP > requested too much " "L0 scratch memory")); diff --git a/lib/kokkos/core/src/HIP/Kokkos_HIP_SharedAllocationRecord.cpp b/lib/kokkos/core/src/HIP/Kokkos_HIP_SharedAllocationRecord.cpp index 83f829fdda..0b67921809 100644 --- a/lib/kokkos/core/src/HIP/Kokkos_HIP_SharedAllocationRecord.cpp +++ b/lib/kokkos/core/src/HIP/Kokkos_HIP_SharedAllocationRecord.cpp @@ -23,7 +23,7 @@ #include #include -#ifndef KOKKOS_ENABLE_IMPL_HIP_UNIFIED_MEMORY +#ifndef KOKKOS_IMPL_HIP_UNIFIED_MEMORY KOKKOS_IMPL_HOST_INACCESSIBLE_SHARED_ALLOCATION_RECORD_EXPLICIT_INSTANTIATION( Kokkos::HIPSpace); #else diff --git a/lib/kokkos/core/src/HIP/Kokkos_HIP_SharedAllocationRecord.hpp b/lib/kokkos/core/src/HIP/Kokkos_HIP_SharedAllocationRecord.hpp index 1ca7bd5cd0..a464609108 100644 --- a/lib/kokkos/core/src/HIP/Kokkos_HIP_SharedAllocationRecord.hpp +++ b/lib/kokkos/core/src/HIP/Kokkos_HIP_SharedAllocationRecord.hpp @@ -20,7 +20,7 @@ #include #include -#if defined(KOKKOS_ENABLE_IMPL_HIP_UNIFIED_MEMORY) +#if defined(KOKKOS_IMPL_HIP_UNIFIED_MEMORY) KOKKOS_IMPL_SHARED_ALLOCATION_SPECIALIZATION(Kokkos::HIPSpace); #else KOKKOS_IMPL_HOST_INACCESSIBLE_SHARED_ALLOCATION_SPECIALIZATION( diff --git a/lib/kokkos/core/src/HIP/Kokkos_HIP_Shuffle_Reduce.hpp b/lib/kokkos/core/src/HIP/Kokkos_HIP_Shuffle_Reduce.hpp index 4035bb0121..feee44ccaf 100644 --- a/lib/kokkos/core/src/HIP/Kokkos_HIP_Shuffle_Reduce.hpp +++ b/lib/kokkos/core/src/HIP/Kokkos_HIP_Shuffle_Reduce.hpp @@ -100,7 +100,7 @@ template __device__ inline bool hip_inter_block_shuffle_reduction( typename FunctorType::reference_type value, typename FunctorType::reference_type neutral, FunctorType const& reducer, - HIP::size_type* const m_scratch_space, + typename FunctorType::pointer_type const m_scratch_space, typename FunctorType::pointer_type const /*result*/, HIP::size_type* const m_scratch_flags, int const max_active_thread = blockDim.y) { @@ -115,9 +115,8 @@ __device__ inline bool hip_inter_block_shuffle_reduction( // One thread in the block writes block result to global scratch_memory if (id == 0) { - pointer_type global = - reinterpret_cast(m_scratch_space) + blockIdx.x; - *global = value; + pointer_type global = m_scratch_space + blockIdx.x; + *global = value; __threadfence(); } @@ -140,8 +139,7 @@ __device__ inline bool hip_inter_block_shuffle_reduction( last_block = true; value = neutral; - pointer_type const global = - reinterpret_cast(m_scratch_space); + pointer_type const global = m_scratch_space; // Reduce all global values with splitting work over threads in one warp const int step_size = blockDim.x * blockDim.y < warp_size diff --git a/lib/kokkos/core/src/HIP/Kokkos_HIP_Space.cpp b/lib/kokkos/core/src/HIP/Kokkos_HIP_Space.cpp index 67635fc1c4..47f07b31ab 100644 --- a/lib/kokkos/core/src/HIP/Kokkos_HIP_Space.cpp +++ b/lib/kokkos/core/src/HIP/Kokkos_HIP_Space.cpp @@ -51,28 +51,54 @@ static std::atomic is_first_hip_managed_allocation(true); namespace Kokkos { -HIPSpace::HIPSpace() : m_device(HIP().hip_device()) {} +HIPSpace::HIPSpace() + : m_device(HIP().hip_device()), m_stream(HIP().hip_stream()) {} HIPHostPinnedSpace::HIPHostPinnedSpace() {} HIPManagedSpace::HIPManagedSpace() : m_device(HIP().hip_device()) {} +#ifndef KOKKOS_IMPL_HIP_UNIFIED_MEMORY +void* HIPSpace::allocate(const HIP& exec_space, + const size_t arg_alloc_size) const { + return allocate(exec_space, "[unlabeled]", arg_alloc_size); +} + +void* HIPSpace::allocate(const HIP& exec_space, const char* arg_label, + const size_t arg_alloc_size, + const size_t arg_logical_size) const { + return impl_allocate(exec_space.hip_stream(), arg_label, arg_alloc_size, + arg_logical_size, true); +} +#endif + void* HIPSpace::allocate(const size_t arg_alloc_size) const { return allocate("[unlabeled]", arg_alloc_size); } -void* HIPSpace::allocate( - const char* arg_label, const size_t arg_alloc_size, - const size_t arg_logical_size) const { - return impl_allocate(arg_label, arg_alloc_size, arg_logical_size); +void* HIPSpace::allocate(const char* arg_label, const size_t arg_alloc_size, + const size_t arg_logical_size) const { + return impl_allocate(m_stream, arg_label, arg_alloc_size, arg_logical_size, + false); } + void* HIPSpace::impl_allocate( - const char* arg_label, const size_t arg_alloc_size, - const size_t arg_logical_size, - const Kokkos::Tools::SpaceHandle arg_handle) const { + [[maybe_unused]] const hipStream_t stream, const char* arg_label, + const size_t arg_alloc_size, const size_t arg_logical_size, + [[maybe_unused]] const bool stream_sync_only) const { void* ptr = nullptr; +#ifdef KOKKOS_ENABLE_IMPL_HIP_MALLOC_ASYNC + auto const error_code = hipMallocAsync(&ptr, arg_alloc_size, stream); + if (stream_sync_only) { + KOKKOS_IMPL_HIP_SAFE_CALL(hipStreamSynchronize(stream)); + } else { + KOKKOS_IMPL_HIP_SAFE_CALL(hipDeviceSynchronize()); + } +#else auto const error_code = hipMalloc(&ptr, arg_alloc_size); +#endif + if (error_code != hipSuccess) { // This is the only way to clear the last error, which we should do here // since we're turning it into an exception here @@ -80,6 +106,8 @@ void* HIPSpace::impl_allocate( Kokkos::Impl::throw_bad_alloc(name(), arg_alloc_size, arg_label); } if (Kokkos::Profiling::profileLibraryLoaded()) { + const Kokkos::Tools::SpaceHandle arg_handle = + Kokkos::Tools::make_space_handle(name()); const size_t reported_size = (arg_logical_size > 0) ? arg_logical_size : arg_alloc_size; Kokkos::Profiling::allocateData(arg_handle, arg_label, ptr, reported_size); @@ -219,7 +247,12 @@ void HIPSpace::impl_deallocate( Kokkos::Profiling::deallocateData(arg_handle, arg_label, arg_alloc_ptr, reported_size); } +#ifdef KOKKOS_ENABLE_IMPL_HIP_MALLOC_ASYNC + KOKKOS_IMPL_HIP_SAFE_CALL(hipFreeAsync(arg_alloc_ptr, m_stream)); + KOKKOS_IMPL_HIP_SAFE_CALL(hipDeviceSynchronize()); +#else KOKKOS_IMPL_HIP_SAFE_CALL(hipFree(arg_alloc_ptr)); +#endif } void HIPHostPinnedSpace::deallocate(void* const arg_alloc_ptr, diff --git a/lib/kokkos/core/src/HIP/Kokkos_HIP_Space.hpp b/lib/kokkos/core/src/HIP/Kokkos_HIP_Space.hpp index e1b4768b87..2380772cac 100644 --- a/lib/kokkos/core/src/HIP/Kokkos_HIP_Space.hpp +++ b/lib/kokkos/core/src/HIP/Kokkos_HIP_Space.hpp @@ -58,14 +58,14 @@ class HIPSpace { /*--------------------------------*/ HIPSpace(); - HIPSpace(HIPSpace&& rhs) = default; - HIPSpace(const HIPSpace& rhs) = default; - HIPSpace& operator=(HIPSpace&& rhs) = default; + HIPSpace(HIPSpace&& rhs) = default; + HIPSpace(const HIPSpace& rhs) = default; + HIPSpace& operator=(HIPSpace&& rhs) = default; HIPSpace& operator=(const HIPSpace& rhs) = default; ~HIPSpace() = default; /**\brief Allocate untracked memory in the hip space */ -#ifdef KOKKOS_ENABLE_IMPL_HIP_UNIFIED_MEMORY +#ifdef KOKKOS_IMPL_HIP_UNIFIED_MEMORY template void* allocate(const ExecutionSpace&, const size_t arg_alloc_size) const { return allocate(arg_alloc_size); @@ -77,15 +77,10 @@ class HIPSpace { return allocate(arg_label, arg_alloc_size, arg_logical_size); } #else - // FIXME_HIP Use execution space instance - void* allocate(const HIP&, const size_t arg_alloc_size) const { - return allocate(arg_alloc_size); - } - // FIXME_HIP Use execution space instance - void* allocate(const HIP&, const char* arg_label, const size_t arg_alloc_size, - const size_t arg_logical_size = 0) const { - return allocate(arg_label, arg_alloc_size, arg_logical_size); - } + void* allocate(const HIP& exec_space, const size_t arg_alloc_size) const; + void* allocate(const HIP& exec_space, const char* arg_label, + const size_t arg_alloc_size, + const size_t arg_logical_size = 0) const; #endif void* allocate(const size_t arg_alloc_size) const; void* allocate(const char* arg_label, const size_t arg_alloc_size, @@ -98,10 +93,10 @@ class HIPSpace { const size_t arg_logical_size = 0) const; private: - void* impl_allocate(const char* arg_label, const size_t arg_alloc_size, - const size_t arg_logical_size = 0, - const Kokkos::Tools::SpaceHandle = - Kokkos::Tools::make_space_handle(name())) const; + void* impl_allocate(const hipStream_t stream, const char* arg_label, + const size_t arg_alloc_size, + const size_t arg_logical_size, + bool stream_sync_only) const; void impl_deallocate(const char* arg_label, void* const arg_alloc_ptr, const size_t arg_alloc_size, const size_t arg_logical_size = 0, @@ -114,6 +109,7 @@ class HIPSpace { private: int m_device; ///< Which HIP device + hipStream_t m_stream; }; template <> @@ -140,9 +136,9 @@ class HIPHostPinnedSpace { /*--------------------------------*/ HIPHostPinnedSpace(); - HIPHostPinnedSpace(HIPHostPinnedSpace&& rhs) = default; - HIPHostPinnedSpace(const HIPHostPinnedSpace& rhs) = default; - HIPHostPinnedSpace& operator=(HIPHostPinnedSpace&& rhs) = default; + HIPHostPinnedSpace(HIPHostPinnedSpace&& rhs) = default; + HIPHostPinnedSpace(const HIPHostPinnedSpace& rhs) = default; + HIPHostPinnedSpace& operator=(HIPHostPinnedSpace&& rhs) = default; HIPHostPinnedSpace& operator=(const HIPHostPinnedSpace& rhs) = default; ~HIPHostPinnedSpace() = default; @@ -213,9 +209,9 @@ class HIPManagedSpace { /*--------------------------------*/ HIPManagedSpace(); - HIPManagedSpace(HIPManagedSpace&& rhs) = default; - HIPManagedSpace(const HIPManagedSpace& rhs) = default; - HIPManagedSpace& operator=(HIPManagedSpace&& rhs) = default; + HIPManagedSpace(HIPManagedSpace&& rhs) = default; + HIPManagedSpace(const HIPManagedSpace& rhs) = default; + HIPManagedSpace& operator=(HIPManagedSpace&& rhs) = default; HIPManagedSpace& operator=(const HIPManagedSpace& rhs) = default; ~HIPManagedSpace() = default; @@ -280,7 +276,7 @@ static_assert(Kokkos::Impl::MemorySpaceAccess::assignable); template <> struct MemorySpaceAccess { enum : bool { assignable = false }; -#if !defined(KOKKOS_ENABLE_IMPL_HIP_UNIFIED_MEMORY) +#if !defined(KOKKOS_IMPL_HIP_UNIFIED_MEMORY) enum : bool{accessible = false}; #else enum : bool { accessible = true }; diff --git a/lib/kokkos/core/src/HIP/Kokkos_HIP_Team.hpp b/lib/kokkos/core/src/HIP/Kokkos_HIP_Team.hpp index fb466d8a72..1724b4361d 100644 --- a/lib/kokkos/core/src/HIP/Kokkos_HIP_Team.hpp +++ b/lib/kokkos/core/src/HIP/Kokkos_HIP_Team.hpp @@ -183,7 +183,7 @@ class HIPTeamMember { typename Kokkos::Impl::FunctorAnalysis< FunctorPatternInterface::REDUCE, TeamPolicy, ReducerType, typename ReducerType::value_type>::Reducer wrapped_reducer(reducer); - hip_intra_block_shuffle_reduction(value, wrapped_reducer, blockDim.y); + impl_team_reduce(wrapped_reducer, value); reducer.reference() = value; #else (void)reducer; @@ -191,6 +191,19 @@ class HIPTeamMember { #endif } + template + KOKKOS_INLINE_FUNCTION std::enable_if_t::value> + impl_team_reduce( + WrappedReducerType const& wrapped_reducer, + typename WrappedReducerType::value_type& value) const noexcept { +#ifdef __HIP_DEVICE_COMPILE__ + hip_intra_block_shuffle_reduction(value, wrapped_reducer, blockDim.y); +#else + (void)wrapped_reducer; + (void)value; +#endif + } + //-------------------------------------------------------------------------- /** \brief Intra-team exclusive prefix sum with team_rank() ordering * with intra-team non-deterministic ordering accumulation. @@ -261,17 +274,37 @@ class HIPTeamMember { KOKKOS_INLINE_FUNCTION static std::enable_if_t::value> vector_reduce(ReducerType const& reducer, typename ReducerType::value_type& value) { +#ifdef __HIP_DEVICE_COMPILE__ + using value_type = typename ReducerType::value_type; + using wrapped_reducer_type = + typename Impl::FunctorAnalysis, ReducerType, + value_type>::Reducer; + + impl_vector_reduce(wrapped_reducer_type(reducer), value); + reducer.reference() = value; +#else + (void)reducer; + (void)value; +#endif + } + + template + KOKKOS_INLINE_FUNCTION static std::enable_if_t< + is_reducer::value> + impl_vector_reduce(WrappedReducerType const& wrapped_reducer, + typename WrappedReducerType::value_type& value) { #ifdef __HIP_DEVICE_COMPILE__ if (blockDim.x == 1) return; // Intra vector lane shuffle reduction: - typename ReducerType::value_type tmp(value); - typename ReducerType::value_type tmp2 = tmp; + typename WrappedReducerType::value_type tmp(value); + typename WrappedReducerType::value_type tmp2 = tmp; for (int i = blockDim.x; (i >>= 1);) { in_place_shfl_down(tmp2, tmp, i, blockDim.x); if (static_cast(threadIdx.x) < i) { - reducer.join(tmp, tmp2); + wrapped_reducer.join(&tmp, &tmp2); } } @@ -281,10 +314,9 @@ class HIPTeamMember { // and thus different threads could have different results. in_place_shfl(tmp2, tmp, 0, blockDim.x); - value = tmp2; - reducer.reference() = tmp2; + value = tmp2; #else - (void)reducer; + (void)wrapped_reducer; (void)value; #endif } @@ -479,15 +511,26 @@ parallel_reduce(const Impl::TeamThreadRangeBoundariesStruct< iType, Impl::HIPTeamMember>& loop_boundaries, const Closure& closure, const ReducerType& reducer) { #ifdef __HIP_DEVICE_COMPILE__ - typename ReducerType::value_type value; - reducer.init(value); + using value_type = typename ReducerType::value_type; + using functor_analysis_type = typename Impl::FunctorAnalysis< + Impl::FunctorPatternInterface::REDUCE, + TeamPolicy, ReducerType, + value_type>; + using wrapped_reducer_type = typename functor_analysis_type::Reducer; + + wrapped_reducer_type wrapped_reducer(reducer); + value_type value; + wrapped_reducer.init(&value); for (iType i = loop_boundaries.start + threadIdx.y; i < loop_boundaries.end; i += blockDim.y) { closure(i, value); } - loop_boundaries.member.team_reduce(reducer, value); + loop_boundaries.member.impl_team_reduce(wrapped_reducer, value); + + wrapped_reducer.final(&value); + reducer.reference() = value; #else (void)loop_boundaries; (void)closure; @@ -508,24 +551,24 @@ KOKKOS_INLINE_FUNCTION std::enable_if_t::value> parallel_reduce(const Impl::TeamThreadRangeBoundariesStruct< iType, Impl::HIPTeamMember>& loop_boundaries, const Closure& closure, ValueType& result) { -#ifdef __HIP_DEVICE_COMPILE__ - ValueType val; - Kokkos::Sum reducer(val); + KOKKOS_IF_ON_DEVICE( + (using functor_analysis_type = typename Impl::FunctorAnalysis< + Impl::FunctorPatternInterface::REDUCE, + TeamPolicy, Closure, + ValueType>; + using wrapped_reducer_type = typename functor_analysis_type::Reducer; + using value_type = typename wrapped_reducer_type::value_type; - reducer.init(reducer.reference()); + wrapped_reducer_type wrapped_reducer(closure); value_type value; + wrapped_reducer.init(&value); - for (iType i = loop_boundaries.start + threadIdx.y; i < loop_boundaries.end; - i += blockDim.y) { - closure(i, val); - } + for (iType i = loop_boundaries.start + threadIdx.y; + i < loop_boundaries.end; i += blockDim.y) { closure(i, value); } - loop_boundaries.member.team_reduce(reducer, val); - result = reducer.reference(); -#else - (void)loop_boundaries; - (void)closure; - (void)result; -#endif + loop_boundaries.member.impl_team_reduce(wrapped_reducer, value); + + wrapped_reducer.final(&value); result = value;)) + KOKKOS_IF_ON_HOST(((void)loop_boundaries; (void)closure; (void)result;)) } /** \brief Inter-thread parallel exclusive prefix sum. @@ -620,16 +663,26 @@ parallel_reduce(const Impl::TeamVectorRangeBoundariesStruct< iType, Impl::HIPTeamMember>& loop_boundaries, const Closure& closure, const ReducerType& reducer) { #ifdef __HIP_DEVICE_COMPILE__ - typename ReducerType::value_type value; - reducer.init(value); + using value_type = typename ReducerType::value_type; + using functor_analysis_type = typename Impl::FunctorAnalysis< + Impl::FunctorPatternInterface::REDUCE, + TeamPolicy, ReducerType, + value_type>; + using wrapped_reducer_type = typename functor_analysis_type::Reducer; + + wrapped_reducer_type wrapped_reducer(reducer); + value_type value; + wrapped_reducer.init(&value); for (iType i = loop_boundaries.start + threadIdx.y * blockDim.x + threadIdx.x; i < loop_boundaries.end; i += blockDim.y * blockDim.x) { closure(i, value); } - loop_boundaries.member.vector_reduce(reducer, value); - loop_boundaries.member.team_reduce(reducer, value); + loop_boundaries.member.impl_vector_reduce(wrapped_reducer, value); + loop_boundaries.member.impl_team_reduce(wrapped_reducer, value); + wrapped_reducer.final(&value); + reducer.reference() = value; #else (void)loop_boundaries; (void)closure; @@ -642,25 +695,27 @@ KOKKOS_INLINE_FUNCTION std::enable_if_t::value> parallel_reduce(const Impl::TeamVectorRangeBoundariesStruct< iType, Impl::HIPTeamMember>& loop_boundaries, const Closure& closure, ValueType& result) { -#ifdef __HIP_DEVICE_COMPILE__ - ValueType val; - Kokkos::Sum reducer(val); + KOKKOS_IF_ON_DEVICE( + (using functor_analysis_type = typename Impl::FunctorAnalysis< + Impl::FunctorPatternInterface::REDUCE, + TeamPolicy, Closure, + ValueType>; + using wrapped_reducer_type = typename functor_analysis_type::Reducer; + using value_type = typename wrapped_reducer_type::value_type; - reducer.init(reducer.reference()); + wrapped_reducer_type wrapped_reducer(closure); value_type value; + wrapped_reducer.init(&value); - for (iType i = loop_boundaries.start + threadIdx.y * blockDim.x + threadIdx.x; - i < loop_boundaries.end; i += blockDim.y * blockDim.x) { - closure(i, val); - } + for (iType i = + loop_boundaries.start + threadIdx.y * blockDim.x + threadIdx.x; + i < loop_boundaries.end; + i += blockDim.y * blockDim.x) { closure(i, value); } - loop_boundaries.member.vector_reduce(reducer); - loop_boundaries.member.team_reduce(reducer); - result = reducer.reference(); -#else - (void)loop_boundaries; - (void)closure; - (void)result; -#endif + loop_boundaries.member.impl_vector_reduce(wrapped_reducer, value); + loop_boundaries.member.impl_team_reduce(wrapped_reducer, value); + wrapped_reducer.final(&value); result = value;)) + + KOKKOS_IF_ON_HOST(((void)loop_boundaries; (void)closure; (void)result;)) } //---------------------------------------------------------------------------- @@ -706,14 +761,26 @@ parallel_reduce(Impl::ThreadVectorRangeBoundariesStruct< iType, Impl::HIPTeamMember> const& loop_boundaries, Closure const& closure, ReducerType const& reducer) { #ifdef __HIP_DEVICE_COMPILE__ - reducer.init(reducer.reference()); + using value_type = typename ReducerType::value_type; + using functor_analysis_type = typename Impl::FunctorAnalysis< + Impl::FunctorPatternInterface::REDUCE, + TeamPolicy, ReducerType, + value_type>; + using wrapped_reducer_type = typename functor_analysis_type::Reducer; + + wrapped_reducer_type wrapped_reducer(reducer); + value_type value; + wrapped_reducer.init(&value); for (iType i = loop_boundaries.start + threadIdx.x; i < loop_boundaries.end; i += blockDim.x) { - closure(i, reducer.reference()); + closure(i, value); } - Impl::HIPTeamMember::vector_reduce(reducer); + Impl::HIPTeamMember::impl_vector_reduce(wrapped_reducer, value); + + wrapped_reducer.final(&value); + reducer.reference() = value; #else (void)loop_boundaries; (void)closure; @@ -737,20 +804,24 @@ KOKKOS_INLINE_FUNCTION std::enable_if_t::value> parallel_reduce(Impl::ThreadVectorRangeBoundariesStruct< iType, Impl::HIPTeamMember> const& loop_boundaries, Closure const& closure, ValueType& result) { -#ifdef __HIP_DEVICE_COMPILE__ - result = ValueType(); + KOKKOS_IF_ON_DEVICE( + (using functor_analysis_type = typename Impl::FunctorAnalysis< + Impl::FunctorPatternInterface::REDUCE, + TeamPolicy, Closure, + ValueType>; + using wrapped_reducer_type = typename functor_analysis_type::Reducer; + using value_type = typename wrapped_reducer_type::value_type; - for (iType i = loop_boundaries.start + threadIdx.x; i < loop_boundaries.end; - i += blockDim.x) { - closure(i, result); - } + wrapped_reducer_type wrapped_reducer(closure); value_type value; + wrapped_reducer.init(&value); - Impl::HIPTeamMember::vector_reduce(Kokkos::Sum(result)); -#else - (void)loop_boundaries; - (void)closure; - (void)result; -#endif + for (iType i = loop_boundaries.start + threadIdx.x; + i < loop_boundaries.end; i += blockDim.x) { closure(i, value); } + + Impl::HIPTeamMember::impl_vector_reduce(wrapped_reducer, value); + wrapped_reducer.final(&value); result = value;)) + + KOKKOS_IF_ON_HOST(((void)loop_boundaries; (void)closure; (void)result;)) } //---------------------------------------------------------------------------- diff --git a/lib/kokkos/core/src/HIP/Kokkos_HIP_TeamPolicyInternal.hpp b/lib/kokkos/core/src/HIP/Kokkos_HIP_TeamPolicyInternal.hpp index 67e1181125..f21c65f16d 100644 --- a/lib/kokkos/core/src/HIP/Kokkos_HIP_TeamPolicyInternal.hpp +++ b/lib/kokkos/core/src/HIP/Kokkos_HIP_TeamPolicyInternal.hpp @@ -222,7 +222,8 @@ class TeamPolicyInternal m_tune_team_size(bool(team_size_request <= 0)), m_tune_vector_length(bool(vector_length_request <= 0)) { // Make sure league size is permissible - if (league_size_ >= static_cast(hip_internal_maximum_grid_count()[0])) + const int max_grid_size_x = m_space.hip_device_prop().maxGridSize[0]; + if (league_size_ >= max_grid_size_x) Impl::throw_runtime_exception( "Requested too large league_size for TeamPolicy on HIP execution " "space."); diff --git a/lib/kokkos/core/src/HIP/Kokkos_HIP_Vectorization.hpp b/lib/kokkos/core/src/HIP/Kokkos_HIP_Vectorization.hpp index 30774c898b..f5b1d321e8 100644 --- a/lib/kokkos/core/src/HIP/Kokkos_HIP_Vectorization.hpp +++ b/lib/kokkos/core/src/HIP/Kokkos_HIP_Vectorization.hpp @@ -40,8 +40,8 @@ struct in_place_shfl_op { template // requires _assignable_from_bits __device__ inline std::enable_if_t operator()( - Scalar& out, Scalar const& in, int lane_or_delta, int width) const - noexcept { + Scalar& out, Scalar const& in, int lane_or_delta, + int width) const noexcept { using shfl_type = int; union conv_type { Scalar orig; @@ -65,16 +65,16 @@ struct in_place_shfl_op { template // requires _assignable_from_bits __device__ inline std::enable_if_t operator()( - Scalar& out, Scalar const& in, int lane_or_delta, int width) const - noexcept { + Scalar& out, Scalar const& in, int lane_or_delta, + int width) const noexcept { reinterpret_cast(out) = self().do_shfl_op( reinterpret_cast(in), lane_or_delta, width); } template __device__ inline std::enable_if_t - operator()(Scalar& out, Scalar const& in, int lane_or_delta, int width) const - noexcept { + operator()(Scalar& out, Scalar const& in, int lane_or_delta, + int width) const noexcept { reinterpret_cast(out) = self().do_shfl_op( *reinterpret_cast(&in), lane_or_delta, width); } @@ -82,8 +82,8 @@ struct in_place_shfl_op { // sizeof(Scalar) > sizeof(double) case template __device__ inline std::enable_if_t<(sizeof(Scalar) > sizeof(double))> - operator()(Scalar& out, const Scalar& val, int lane_or_delta, int width) const - noexcept { + operator()(Scalar& out, const Scalar& val, int lane_or_delta, + int width) const noexcept { using shuffle_as_t = int; constexpr int N = sizeof(Scalar) / sizeof(shuffle_as_t); @@ -108,7 +108,7 @@ struct in_place_shfl_fn : in_place_shfl_op { template __device__ KOKKOS_IMPL_FORCEINLINE void in_place_shfl(Args&&... args) noexcept { - in_place_shfl_fn{}((Args &&) args...); + in_place_shfl_fn{}((Args&&)args...); } struct in_place_shfl_up_fn : in_place_shfl_op { @@ -123,7 +123,7 @@ struct in_place_shfl_up_fn : in_place_shfl_op { template __device__ KOKKOS_IMPL_FORCEINLINE void in_place_shfl_up( Args&&... args) noexcept { - in_place_shfl_up_fn{}((Args &&) args...); + in_place_shfl_up_fn{}((Args&&)args...); } struct in_place_shfl_down_fn : in_place_shfl_op { @@ -138,7 +138,7 @@ struct in_place_shfl_down_fn : in_place_shfl_op { template __device__ KOKKOS_IMPL_FORCEINLINE void in_place_shfl_down( Args&&... args) noexcept { - in_place_shfl_down_fn{}((Args &&) args...); + in_place_shfl_down_fn{}((Args&&)args...); } } // namespace Impl diff --git a/lib/kokkos/core/src/HIP/Kokkos_HIP_ZeroMemset.cpp b/lib/kokkos/core/src/HIP/Kokkos_HIP_ZeroMemset.cpp new file mode 100644 index 0000000000..34d5ecf1a6 --- /dev/null +++ b/lib/kokkos/core/src/HIP/Kokkos_HIP_ZeroMemset.cpp @@ -0,0 +1,36 @@ +//@HEADER +// ************************************************************************ +// +// Kokkos v. 4.0 +// Copyright (2022) National Technology & Engineering +// Solutions of Sandia, LLC (NTESS). +// +// Under the terms of Contract DE-NA0003525 with NTESS, +// the U.S. Government retains certain rights in this software. +// +// Part of Kokkos, under the Apache License v2.0 with LLVM Exceptions. +// See https://kokkos.org/LICENSE for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//@HEADER + +#ifndef KOKKOS_IMPL_PUBLIC_INCLUDE +#define KOKKOS_IMPL_PUBLIC_INCLUDE +#endif + +#include +#include + +namespace Kokkos { +namespace Impl { + +// alternative to hipMemsetAsync, which sets the first `cnt` bytes of `dst` to 0 +void zero_with_hip_kernel(const HIP& exec_space, void* dst, size_t cnt) { + Kokkos::parallel_for( + "Kokkos::ZeroMemset via parallel_for", + Kokkos::RangePolicy(exec_space, 0, cnt), + KOKKOS_LAMBDA(size_t i) { static_cast(dst)[i] = 0; }); +} + +} // namespace Impl +} // namespace Kokkos diff --git a/lib/kokkos/core/src/HIP/Kokkos_HIP_ZeroMemset.hpp b/lib/kokkos/core/src/HIP/Kokkos_HIP_ZeroMemset.hpp index 4bca29868f..18708cf8c5 100644 --- a/lib/kokkos/core/src/HIP/Kokkos_HIP_ZeroMemset.hpp +++ b/lib/kokkos/core/src/HIP/Kokkos_HIP_ZeroMemset.hpp @@ -23,12 +23,21 @@ namespace Kokkos { namespace Impl { -template -struct ZeroMemset> { - ZeroMemset(const HIP& exec_space, const View& dst) { - KOKKOS_IMPL_HIP_SAFE_CALL(hipMemsetAsync( - dst.data(), 0, dst.size() * sizeof(typename View::value_type), - exec_space.hip_stream())); +// hipMemsetAsync sets the first `cnt` bytes of `dst` to the provided value +void zero_with_hip_kernel(const HIP& exec_space, void* dst, size_t cnt); + +template <> +struct ZeroMemset { + ZeroMemset(const HIP& exec_space, void* dst, size_t cnt) { + // in ROCm <= 6.2.0, hipMemsetAsync on a host-allocated pointer + // returns an invalid value error, but accessing the data via a + // GPU kernel works. +#if defined(KOKKOS_IMPL_HIP_UNIFIED_MEMORY) + zero_with_hip_kernel(exec_space, dst, cnt); +#else + KOKKOS_IMPL_HIP_SAFE_CALL( + hipMemsetAsync(dst, 0, cnt, exec_space.hip_stream())); +#endif } }; diff --git a/lib/kokkos/core/src/HPX/Kokkos_HPX.hpp b/lib/kokkos/core/src/HPX/Kokkos_HPX.hpp index 245dc128ca..7d49933790 100644 --- a/lib/kokkos/core/src/HPX/Kokkos_HPX.hpp +++ b/lib/kokkos/core/src/HPX/Kokkos_HPX.hpp @@ -32,12 +32,10 @@ static_assert(false, #include #include #include -#include #include #include #include #include -#include #include #include @@ -75,12 +73,12 @@ class hpx_thread_buffer { } public: - hpx_thread_buffer() = default; - ~hpx_thread_buffer() = default; - hpx_thread_buffer(const hpx_thread_buffer &) = delete; - hpx_thread_buffer(hpx_thread_buffer &&) = delete; + hpx_thread_buffer() = default; + ~hpx_thread_buffer() = default; + hpx_thread_buffer(const hpx_thread_buffer &) = delete; + hpx_thread_buffer(hpx_thread_buffer &&) = delete; hpx_thread_buffer &operator=(const hpx_thread_buffer &) = delete; - hpx_thread_buffer &operator=(hpx_thread_buffer) = delete; + hpx_thread_buffer &operator=(hpx_thread_buffer) = delete; void resize(const std::size_t num_threads, const std::size_t size_per_thread, const std::size_t extra_space = 0) noexcept; @@ -140,10 +138,10 @@ class HPX { hpx::execution::experimental::unique_any_sender<> &&sender) : m_instance_id(instance_id), m_sender{std::move(sender)} {} - instance_data(const instance_data &) = delete; - instance_data(instance_data &&) = delete; + instance_data(const instance_data &) = delete; + instance_data(instance_data &&) = delete; instance_data &operator=(const instance_data &) = delete; - instance_data &operator=(instance_data) = delete; + instance_data &operator=(instance_data) = delete; uint32_t m_instance_id{HPX::impl_default_instance_id()}; hpx::execution::experimental::unique_any_sender<> m_sender{ @@ -196,7 +194,7 @@ class HPX { HPX(HPX &&other) = default; HPX(const HPX &other) = default; - HPX &operator=(HPX &&) = default; + HPX &operator=(HPX &&) = default; HPX &operator=(const HPX &) = default; void print_configuration(std::ostream &os, bool /*verbose*/ = false) const; @@ -214,9 +212,9 @@ class HPX { struct impl_in_parallel_scope { impl_in_parallel_scope() noexcept; ~impl_in_parallel_scope() noexcept; - impl_in_parallel_scope(impl_in_parallel_scope &&) = delete; - impl_in_parallel_scope(impl_in_parallel_scope const &) = delete; - impl_in_parallel_scope &operator=(impl_in_parallel_scope &&) = delete; + impl_in_parallel_scope(impl_in_parallel_scope &&) = delete; + impl_in_parallel_scope(impl_in_parallel_scope const &) = delete; + impl_in_parallel_scope &operator=(impl_in_parallel_scope &&) = delete; impl_in_parallel_scope &operator=(impl_in_parallel_scope const &) = delete; }; @@ -249,13 +247,15 @@ class HPX { impl_instance_fence(name); } - static bool is_asynchronous(HPX const & = HPX()) noexcept { +#ifdef KOKKOS_ENABLE_DEPRECATED_CODE_4 + KOKKOS_DEPRECATED static bool is_asynchronous(HPX const & = HPX()) noexcept { #if defined(KOKKOS_ENABLE_IMPL_HPX_ASYNC_DISPATCH) return true; #else return false; #endif } +#endif #ifdef KOKKOS_ENABLE_DEPRECATED_CODE_4 static int concurrency(); @@ -281,8 +281,8 @@ class HPX { return impl_get_instance_data().m_buffer; } - hpx::execution::experimental::unique_any_sender<> &impl_get_sender() const - noexcept { + hpx::execution::experimental::unique_any_sender<> &impl_get_sender() + const noexcept { return impl_get_instance_data().m_sender; } @@ -447,6 +447,20 @@ class HPX { } }; +template +std::vector partition_space(HPX const &, Args... args) { + std::vector instances(sizeof...(args)); + for (auto &in : instances) in = HPX(HPX::instance_mode::independent); + return instances; +} + +template +std::vector partition_space(HPX const &, std::vector const &weights) { + std::vector instances(weights.size()); + for (auto &in : instances) in = HPX(HPX::instance_mode::independent); + return instances; +} + extern template void HPX::impl_bulk_plain_erased( bool, bool, std::function &&, int const, hpx::threads::thread_stacksize stacksize) const; @@ -1772,11 +1786,24 @@ KOKKOS_INLINE_FUNCTION void parallel_reduce( const Impl::TeamThreadRangeBoundariesStruct &loop_boundaries, const Lambda &lambda, ValueType &result) { - result = ValueType(); + using functor_analysis_type = typename Impl::FunctorAnalysis< + Impl::FunctorPatternInterface::REDUCE, + TeamPolicy, Lambda, + ValueType>; + using wrapped_reducer_type = typename functor_analysis_type::Reducer; + using value_type = typename wrapped_reducer_type::value_type; + + wrapped_reducer_type wrapped_reducer(lambda); + value_type value; + wrapped_reducer.init(&value); + for (iType i = loop_boundaries.start; i < loop_boundaries.end; i += loop_boundaries.increment) { - lambda(i, result); + lambda(i, value); } + + wrapped_reducer.final(&value); + result = value; } /** \brief Intra-thread vector parallel_for. Executes lambda(iType i) for each @@ -1810,14 +1837,26 @@ KOKKOS_INLINE_FUNCTION void parallel_reduce( const Impl::ThreadVectorRangeBoundariesStruct &loop_boundaries, const Lambda &lambda, ValueType &result) { - result = ValueType(); + using functor_analysis_type = typename Impl::FunctorAnalysis< + Impl::FunctorPatternInterface::REDUCE, + TeamPolicy, Lambda, + ValueType>; + using wrapped_reducer_type = typename functor_analysis_type::Reducer; + using value_type = typename wrapped_reducer_type::value_type; + + wrapped_reducer_type wrapped_reducer(lambda); + value_type value; + wrapped_reducer.init(&value); + #ifdef KOKKOS_ENABLE_PRAGMA_IVDEP #pragma ivdep #endif for (iType i = loop_boundaries.start; i < loop_boundaries.end; i += loop_boundaries.increment) { - lambda(i, result); + lambda(i, value); } + wrapped_reducer.final(&value); + result = value; } template &loop_boundaries, const Lambda &lambda, const ReducerType &reducer) { - reducer.init(reducer.reference()); + using value_type = typename ReducerType::value_type; + using functor_analysis_type = typename Impl::FunctorAnalysis< + Impl::FunctorPatternInterface::REDUCE, + TeamPolicy, ReducerType, + value_type>; + using wrapped_reducer_type = typename functor_analysis_type::Reducer; + + wrapped_reducer_type wrapped_reducer(reducer); + value_type value; + wrapped_reducer.init(&value); + for (iType i = loop_boundaries.start; i < loop_boundaries.end; i += loop_boundaries.increment) { - lambda(i, reducer.reference()); + lambda(i, value); } + + wrapped_reducer.final(&value); + reducer.reference() = value; } template &loop_boundaries, const Lambda &lambda, const ReducerType &reducer) { - reducer.init(reducer.reference()); + using value_type = typename ReducerType::value_type; + using functor_analysis_type = typename Impl::FunctorAnalysis< + Impl::FunctorPatternInterface::REDUCE, + TeamPolicy, ReducerType, + value_type>; + using wrapped_reducer_type = typename functor_analysis_type::Reducer; + + wrapped_reducer_type wrapped_reducer(reducer); + value_type value; + wrapped_reducer.init(&value); + #ifdef KOKKOS_ENABLE_PRAGMA_IVDEP #pragma ivdep #endif for (iType i = loop_boundaries.start; i < loop_boundaries.end; i += loop_boundaries.increment) { - lambda(i, reducer.reference()); + lambda(i, value); } + + wrapped_reducer.final(&value); + reducer.reference() = value; } template @@ -1995,7 +2060,9 @@ KOKKOS_INLINE_FUNCTION void single( } // namespace Kokkos +#ifdef KOKKOS_ENABLE_DEPRECATED_CODE_4 #include +#endif #endif /* #if defined( KOKKOS_ENABLE_HPX ) */ #endif /* #ifndef KOKKOS_HPX_HPP */ diff --git a/lib/kokkos/core/src/HPX/Kokkos_HPX_Task.hpp b/lib/kokkos/core/src/HPX/Kokkos_HPX_Task.hpp index 28c75b2515..d775b7fac3 100644 --- a/lib/kokkos/core/src/HPX/Kokkos_HPX_Task.hpp +++ b/lib/kokkos/core/src/HPX/Kokkos_HPX_Task.hpp @@ -25,6 +25,8 @@ #include +#include + #include #include @@ -33,6 +35,11 @@ //---------------------------------------------------------------------------- //---------------------------------------------------------------------------- +#ifdef KOKKOS_ENABLE_DEPRECATION_WARNINGS +// We allow using deprecated classes in this file +KOKKOS_IMPL_DISABLE_DEPRECATED_WARNINGS_PUSH() +#endif + namespace Kokkos { namespace Impl { @@ -256,6 +263,10 @@ extern template class TaskQueue< } // namespace Impl } // namespace Kokkos +#ifdef KOKKOS_ENABLE_DEPRECATION_WARNINGS +KOKKOS_IMPL_DISABLE_DEPRECATED_WARNINGS_POP() +#endif + //---------------------------------------------------------------------------- //---------------------------------------------------------------------------- diff --git a/lib/kokkos/core/src/KokkosExp_MDRangePolicy.hpp b/lib/kokkos/core/src/KokkosExp_MDRangePolicy.hpp index 297b1fadee..92dc506c5e 100644 --- a/lib/kokkos/core/src/KokkosExp_MDRangePolicy.hpp +++ b/lib/kokkos/core/src/KokkosExp_MDRangePolicy.hpp @@ -30,6 +30,7 @@ static_assert(false, #include #include #include +#include namespace Kokkos { @@ -60,13 +61,13 @@ namespace Impl { // NOTE the comparison below is encapsulated to silent warnings about pointless // comparison of unsigned integer with zero template -constexpr std::enable_if_t::value, bool> +constexpr std::enable_if_t, bool> is_less_than_value_initialized_variable(T) { return false; } template -constexpr std::enable_if_t::value, bool> +constexpr std::enable_if_t, bool> is_less_than_value_initialized_variable(T arg) { return arg < T{}; } @@ -75,7 +76,7 @@ is_less_than_value_initialized_variable(T arg) { template constexpr To checked_narrow_cast(From arg, std::size_t idx) { constexpr const bool is_different_signedness = - (std::is_signed::value != std::is_signed::value); + (std::is_signed_v != std::is_signed_v); auto const ret = static_cast(arg); if (static_cast(ret) != arg || (is_different_signedness && @@ -183,7 +184,7 @@ struct MDRangePolicy template friend struct MDRangePolicy; - static_assert(!std::is_void::value, + static_assert(!std::is_void_v, "Kokkos Error: MD iteration pattern not defined"); using iteration_pattern = typename traits::iteration_pattern; @@ -238,9 +239,9 @@ struct MDRangePolicy template ::value && - std::is_integral::value && - std::is_integral::value>> + typename = std::enable_if_t && + std::is_integral_v && + std::is_integral_v>> MDRangePolicy(const LT (&lower)[LN], const UT (&upper)[UN], const TT (&tile)[TN] = {}) : MDRangePolicy( @@ -257,9 +258,9 @@ struct MDRangePolicy template ::value && - std::is_integral::value && - std::is_integral::value>> + typename = std::enable_if_t && + std::is_integral_v && + std::is_integral_v>> MDRangePolicy(const typename traits::execution_space& work_space, const LT (&lower)[LN], const UT (&upper)[UN], const TT (&tile)[TN] = {}) @@ -291,14 +292,14 @@ struct MDRangePolicy } template ::value>> + typename = std::enable_if_t>> MDRangePolicy(Kokkos::Array const& lower, Kokkos::Array const& upper, Kokkos::Array const& tile = Kokkos::Array{}) : MDRangePolicy(typename traits::execution_space(), lower, upper, tile) {} template ::value>> + typename = std::enable_if_t>> MDRangePolicy(const typename traits::execution_space& work_space, Kokkos::Array const& lower, Kokkos::Array const& upper, @@ -330,7 +331,44 @@ struct MDRangePolicy } bool impl_tune_tile_size() const { return m_tune_tile_size; } + tile_type tile_size_recommended() const { + tile_type rec_tile_sizes = {}; + + for (std::size_t i = 0; i < rec_tile_sizes.size(); ++i) { + rec_tile_sizes[i] = tile_size_recommended(i); + } + return rec_tile_sizes; + } + + int max_total_tile_size() const { + return Impl::get_tile_size_properties(m_space).max_total_tile_size; + } + private: + int tile_size_recommended(const int tile_rank) const { + auto properties = Impl::get_tile_size_properties(m_space); + int last_rank = (inner_direction == Iterate::Right) ? rank - 1 : 0; + int rank_acc = + (inner_direction == Iterate::Right) ? tile_rank + 1 : tile_rank - 1; + int rec_tile_size = (std::pow(properties.default_tile_size, rank_acc) < + properties.max_total_tile_size) + ? properties.default_tile_size + : 1; + + if (tile_rank == last_rank) { + rec_tile_size = tile_size_last_rank( + properties, m_upper[last_rank] - m_lower[last_rank]); + } + return rec_tile_size; + } + + int tile_size_last_rank(const Impl::TileSizeProperties properties, + const index_type length) const { + return properties.default_largest_tile_size == 0 + ? std::max(length, 1) + : properties.default_largest_tile_size; + } + void init_helper(Impl::TileSizeProperties properties) { m_prod_tile_dims = 1; int increment = 1; @@ -341,6 +379,7 @@ struct MDRangePolicy rank_start = rank - 1; rank_end = -1; } + for (int i = rank_start; i != rank_end; i += increment) { const index_type length = m_upper[i] - m_lower[i]; @@ -368,9 +407,7 @@ struct MDRangePolicy m_tile[i] = 1; } } else { - m_tile[i] = properties.default_largest_tile_size == 0 - ? std::max(length, 1) - : properties.default_largest_tile_size; + m_tile[i] = tile_size_last_rank(properties, length); } } m_tile_end[i] = @@ -389,58 +426,55 @@ struct MDRangePolicy }; template -MDRangePolicy(const LT (&)[N], const UT (&)[N])->MDRangePolicy>; +MDRangePolicy(const LT (&)[N], const UT (&)[N]) -> MDRangePolicy>; template MDRangePolicy(const LT (&)[N], const UT (&)[N], const TT (&)[TN]) - ->MDRangePolicy>; + -> MDRangePolicy>; template MDRangePolicy(DefaultExecutionSpace const&, const LT (&)[N], const UT (&)[N]) - ->MDRangePolicy>; + -> MDRangePolicy>; template MDRangePolicy(DefaultExecutionSpace const&, const LT (&)[N], const UT (&)[N], - const TT (&)[TN]) - ->MDRangePolicy>; + const TT (&)[TN]) -> MDRangePolicy>; template >> MDRangePolicy(ES const&, const LT (&)[N], const UT (&)[N]) - ->MDRangePolicy>; + -> MDRangePolicy>; template >> MDRangePolicy(ES const&, const LT (&)[N], const UT (&)[N], const TT (&)[TN]) - ->MDRangePolicy>; + -> MDRangePolicy>; template -MDRangePolicy(Array const&, Array const&)->MDRangePolicy>; +MDRangePolicy(Array const&, Array const&) -> MDRangePolicy>; template MDRangePolicy(Array const&, Array const&, Array const&) - ->MDRangePolicy>; + -> MDRangePolicy>; template MDRangePolicy(DefaultExecutionSpace const&, Array const&, - Array const&) - ->MDRangePolicy>; + Array const&) -> MDRangePolicy>; template MDRangePolicy(DefaultExecutionSpace const&, Array const&, Array const&, Array const&) - ->MDRangePolicy>; + -> MDRangePolicy>; template >> MDRangePolicy(ES const&, Array const&, Array const&) - ->MDRangePolicy>; + -> MDRangePolicy>; template >> MDRangePolicy(ES const&, Array const&, Array const&, - Array const&) - ->MDRangePolicy>; + Array const&) -> MDRangePolicy>; } // namespace Kokkos diff --git a/lib/kokkos/core/src/Kokkos_AnonymousSpace.hpp b/lib/kokkos/core/src/Kokkos_AnonymousSpace.hpp index 9f5deed5d6..62f527aa02 100644 --- a/lib/kokkos/core/src/Kokkos_AnonymousSpace.hpp +++ b/lib/kokkos/core/src/Kokkos_AnonymousSpace.hpp @@ -41,10 +41,10 @@ class AnonymousSpace { using device_type = Kokkos::Device; /**\brief Default memory space instance */ - AnonymousSpace() = default; - AnonymousSpace(AnonymousSpace &&rhs) = default; - AnonymousSpace(const AnonymousSpace &rhs) = default; - AnonymousSpace &operator=(AnonymousSpace &&) = default; + AnonymousSpace() = default; + AnonymousSpace(AnonymousSpace &&rhs) = default; + AnonymousSpace(const AnonymousSpace &rhs) = default; + AnonymousSpace &operator=(AnonymousSpace &&) = default; AnonymousSpace &operator=(const AnonymousSpace &) = default; ~AnonymousSpace() = default; diff --git a/lib/kokkos/core/src/Kokkos_Array.hpp b/lib/kokkos/core/src/Kokkos_Array.hpp index 4d905fbc55..493536b53b 100644 --- a/lib/kokkos/core/src/Kokkos_Array.hpp +++ b/lib/kokkos/core/src/Kokkos_Array.hpp @@ -35,7 +35,7 @@ namespace Kokkos { #ifdef KOKKOS_ENABLE_DEBUG_BOUNDS_CHECK namespace Impl { -template ::value> +template > struct ArrayBoundsCheck; template @@ -195,8 +195,10 @@ struct Array { return *reinterpret_cast(-1); } - KOKKOS_INLINE_FUNCTION pointer data() { return nullptr; } - KOKKOS_INLINE_FUNCTION const_pointer data() const { return nullptr; } + KOKKOS_INLINE_FUNCTION constexpr pointer data() { return nullptr; } + KOKKOS_INLINE_FUNCTION constexpr const_pointer data() const { + return nullptr; + } friend KOKKOS_FUNCTION constexpr bool operator==(Array const&, Array const&) noexcept { @@ -365,7 +367,7 @@ struct KOKKOS_DEPRECATED #endif template -Array(T, Us...)->Array; +Array(T, Us...) -> Array; namespace Impl { @@ -377,7 +379,7 @@ KOKKOS_FUNCTION constexpr Array, N> to_array_impl( template KOKKOS_FUNCTION constexpr Array, N> to_array_impl( - T(&&a)[N], std::index_sequence) { + T (&&a)[N], std::index_sequence) { return {{std::move(a[I])...}}; } @@ -389,7 +391,7 @@ KOKKOS_FUNCTION constexpr auto to_array(T (&a)[N]) { } template -KOKKOS_FUNCTION constexpr auto to_array(T(&&a)[N]) { +KOKKOS_FUNCTION constexpr auto to_array(T (&&a)[N]) { return Impl::to_array_impl(std::move(a), std::make_index_sequence{}); } @@ -435,6 +437,32 @@ KOKKOS_FUNCTION constexpr T const&& get(Array const&& a) noexcept { } // namespace Kokkos // +// +namespace Kokkos { + +template +KOKKOS_FUNCTION constexpr T const* begin(Array const& a) noexcept { + return a.data(); +} + +template +KOKKOS_FUNCTION constexpr T* begin(Array& a) noexcept { + return a.data(); +} + +template +KOKKOS_FUNCTION constexpr T const* end(Array const& a) noexcept { + return a.data() + a.size(); +} + +template +KOKKOS_FUNCTION constexpr T* end(Array& a) noexcept { + return a.data() + a.size(); +} + +} // namespace Kokkos +// + #ifdef KOKKOS_IMPL_PUBLIC_INCLUDE_NOTDEFINED_ARRAY #undef KOKKOS_IMPL_PUBLIC_INCLUDE #undef KOKKOS_IMPL_PUBLIC_INCLUDE_NOTDEFINED_ARRAY diff --git a/lib/kokkos/core/src/Kokkos_Atomic.hpp b/lib/kokkos/core/src/Kokkos_Atomic.hpp index 6fc903f274..ba61136092 100644 --- a/lib/kokkos/core/src/Kokkos_Atomic.hpp +++ b/lib/kokkos/core/src/Kokkos_Atomic.hpp @@ -47,7 +47,6 @@ #include #include -#include #ifdef KOKKOS_IMPL_PUBLIC_INCLUDE_NOTDEFINED_ATOMIC #undef KOKKOS_IMPL_PUBLIC_INCLUDE diff --git a/lib/kokkos/core/src/Kokkos_Atomics_Desul_Volatile_Wrapper.hpp b/lib/kokkos/core/src/Kokkos_Atomics_Desul_Volatile_Wrapper.hpp deleted file mode 100644 index bf57dcae65..0000000000 --- a/lib/kokkos/core/src/Kokkos_Atomics_Desul_Volatile_Wrapper.hpp +++ /dev/null @@ -1,196 +0,0 @@ -//@HEADER -// ************************************************************************ -// -// Kokkos v. 4.0 -// Copyright (2022) National Technology & Engineering -// Solutions of Sandia, LLC (NTESS). -// -// Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. -// -// Part of Kokkos, under the Apache License v2.0 with LLVM Exceptions. -// See https://kokkos.org/LICENSE for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// -//@HEADER - -#ifndef KOKKOS_IMPL_PUBLIC_INCLUDE -#include -static_assert(false, - "Including non-public Kokkos header files is not allowed."); -#endif -#ifndef KOKKOS_DESUL_ATOMICS_VOLATILE_WRAPPER_HPP_ -#define KOKKOS_DESUL_ATOMICS_VOLATILE_WRAPPER_HPP_ -#include -#include - -#ifdef KOKKOS_ENABLE_ATOMICS_BYPASS -#define KOKKOS_DESUL_MEM_SCOPE desul::MemoryScopeCaller() -#else -#define KOKKOS_DESUL_MEM_SCOPE desul::MemoryScopeDevice() -#endif - -// clang-format off -namespace Kokkos { - -template KOKKOS_INLINE_FUNCTION -T atomic_load(volatile T* const dest) { return desul::atomic_load(const_cast(dest), desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); } - -template KOKKOS_INLINE_FUNCTION -void atomic_store(volatile T* const dest, desul::Impl::dont_deduce_this_parameter_t val) { return desul::atomic_store(const_cast(dest), val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); } - -// atomic_fetch_op -template KOKKOS_INLINE_FUNCTION -T atomic_fetch_add (volatile T* const dest, desul::Impl::dont_deduce_this_parameter_t val) { return desul::atomic_fetch_add (const_cast(dest), val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); } - -template KOKKOS_INLINE_FUNCTION -T atomic_fetch_sub (volatile T* const dest, desul::Impl::dont_deduce_this_parameter_t val) { return desul::atomic_fetch_sub (const_cast(dest), val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); } - -template KOKKOS_INLINE_FUNCTION -T atomic_fetch_max (volatile T* const dest, desul::Impl::dont_deduce_this_parameter_t val) { return desul::atomic_fetch_max (const_cast(dest), val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); } - -template KOKKOS_INLINE_FUNCTION -T atomic_fetch_min (volatile T* const dest, desul::Impl::dont_deduce_this_parameter_t val) { return desul::atomic_fetch_min (const_cast(dest), val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); } - -template KOKKOS_INLINE_FUNCTION -T atomic_fetch_mul (volatile T* const dest, desul::Impl::dont_deduce_this_parameter_t val) { return desul::atomic_fetch_mul (const_cast(dest), val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); } - -template KOKKOS_INLINE_FUNCTION -T atomic_fetch_div (volatile T* const dest, desul::Impl::dont_deduce_this_parameter_t val) { return desul::atomic_fetch_div (const_cast(dest), val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); } - -template KOKKOS_INLINE_FUNCTION -T atomic_fetch_mod (volatile T* const dest, desul::Impl::dont_deduce_this_parameter_t val) { return desul::atomic_fetch_mod (const_cast(dest), val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); } - -template KOKKOS_INLINE_FUNCTION -T atomic_fetch_and (volatile T* const dest, desul::Impl::dont_deduce_this_parameter_t val) { return desul::atomic_fetch_and (const_cast(dest), val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); } - -template KOKKOS_INLINE_FUNCTION -T atomic_fetch_or (volatile T* const dest, desul::Impl::dont_deduce_this_parameter_t val) { return desul::atomic_fetch_or (const_cast(dest), val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); } - -template KOKKOS_INLINE_FUNCTION -T atomic_fetch_xor (volatile T* const dest, desul::Impl::dont_deduce_this_parameter_t val) { return desul::atomic_fetch_xor (const_cast(dest), val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); } - -template KOKKOS_INLINE_FUNCTION -T atomic_fetch_nand(volatile T* const dest, desul::Impl::dont_deduce_this_parameter_t val) { return desul::atomic_fetch_nand(const_cast(dest), val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); } - -template KOKKOS_INLINE_FUNCTION -T atomic_fetch_lshift(volatile T* const dest, desul::Impl::dont_deduce_this_parameter_t val) { return desul::atomic_fetch_lshift(const_cast(dest), val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); } - -template KOKKOS_INLINE_FUNCTION -T atomic_fetch_rshift(volatile T* const dest, desul::Impl::dont_deduce_this_parameter_t val) { return desul::atomic_fetch_rshift(const_cast(dest), val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); } - -template KOKKOS_INLINE_FUNCTION -T atomic_fetch_inc(volatile T* const dest) { return desul::atomic_fetch_inc(const_cast(dest),desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); } - -template KOKKOS_INLINE_FUNCTION -T atomic_fetch_dec(volatile T* const dest) { return desul::atomic_fetch_dec(const_cast(dest),desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); } - - -// atomic_op_fetch -template KOKKOS_INLINE_FUNCTION -T atomic_add_fetch (volatile T* const dest, desul::Impl::dont_deduce_this_parameter_t val) { return desul::atomic_add_fetch (const_cast(dest), val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); } - -template KOKKOS_INLINE_FUNCTION -T atomic_sub_fetch (volatile T* const dest, desul::Impl::dont_deduce_this_parameter_t val) { return desul::atomic_sub_fetch (const_cast(dest), val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); } - -template KOKKOS_INLINE_FUNCTION -T atomic_max_fetch (volatile T* const dest, desul::Impl::dont_deduce_this_parameter_t val) { return desul::atomic_max_fetch (const_cast(dest), val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); } - -template KOKKOS_INLINE_FUNCTION -T atomic_min_fetch (volatile T* const dest, desul::Impl::dont_deduce_this_parameter_t val) { return desul::atomic_min_fetch (const_cast(dest), val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); } - -template KOKKOS_INLINE_FUNCTION -T atomic_mul_fetch (volatile T* const dest, desul::Impl::dont_deduce_this_parameter_t val) { return desul::atomic_mul_fetch (const_cast(dest), val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); } - -template KOKKOS_INLINE_FUNCTION -T atomic_div_fetch (volatile T* const dest, desul::Impl::dont_deduce_this_parameter_t val) { return desul::atomic_div_fetch (const_cast(dest), val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); } - -template KOKKOS_INLINE_FUNCTION -T atomic_mod_fetch (volatile T* const dest, desul::Impl::dont_deduce_this_parameter_t val) { return desul::atomic_mod_fetch (const_cast(dest), val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); } - -template KOKKOS_INLINE_FUNCTION -T atomic_and_fetch (volatile T* const dest, desul::Impl::dont_deduce_this_parameter_t val) { return desul::atomic_and_fetch (const_cast(dest), val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); } - -template KOKKOS_INLINE_FUNCTION -T atomic_or_fetch (volatile T* const dest, desul::Impl::dont_deduce_this_parameter_t val) { return desul::atomic_or_fetch (const_cast(dest), val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); } - -template KOKKOS_INLINE_FUNCTION -T atomic_xor_fetch (volatile T* const dest, desul::Impl::dont_deduce_this_parameter_t val) { return desul::atomic_xor_fetch (const_cast(dest), val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); } - -template KOKKOS_INLINE_FUNCTION -T atomic_nand_fetch(volatile T* const dest, desul::Impl::dont_deduce_this_parameter_t val) { return desul::atomic_nand_fetch(const_cast(dest), val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); } - -template KOKKOS_INLINE_FUNCTION -T atomic_lshift_fetch(volatile T* const dest, desul::Impl::dont_deduce_this_parameter_t val) { return desul::atomic_lshift_fetch(const_cast(dest), val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); } - -template KOKKOS_INLINE_FUNCTION -T atomic_rshift_fetch(volatile T* const dest, desul::Impl::dont_deduce_this_parameter_t val) { return desul::atomic_rshift_fetch(const_cast(dest), val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); } - -template KOKKOS_INLINE_FUNCTION -T atomic_inc_fetch(volatile T* const dest) { return desul::atomic_inc_fetch(const_cast(dest),desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); } - -template KOKKOS_INLINE_FUNCTION -T atomic_dec_fetch(volatile T* const dest) { return desul::atomic_dec_fetch(const_cast(dest),desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); } - - -// atomic_op -template KOKKOS_INLINE_FUNCTION -void atomic_add(volatile T* const dest, desul::Impl::dont_deduce_this_parameter_t val) { return desul::atomic_add (const_cast(dest), val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); } - -template KOKKOS_INLINE_FUNCTION -void atomic_sub(volatile T* const dest, desul::Impl::dont_deduce_this_parameter_t val) { return desul::atomic_sub (const_cast(dest), val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); } - -template KOKKOS_INLINE_FUNCTION -void atomic_mul(volatile T* const dest, desul::Impl::dont_deduce_this_parameter_t val) { return desul::atomic_mul (const_cast(dest), val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); } - -template KOKKOS_INLINE_FUNCTION -void atomic_div(volatile T* const dest, desul::Impl::dont_deduce_this_parameter_t val) { return desul::atomic_div (const_cast(dest), val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); } - -template KOKKOS_INLINE_FUNCTION -void atomic_min(volatile T* const dest, desul::Impl::dont_deduce_this_parameter_t val) { return desul::atomic_min (const_cast(dest), val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); } - -template KOKKOS_INLINE_FUNCTION -void atomic_max(volatile T* const dest, desul::Impl::dont_deduce_this_parameter_t val) { return desul::atomic_max (const_cast(dest), val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); } - -// FIXME: Desul doesn't have atomic_and yet so call fetch_and -template KOKKOS_INLINE_FUNCTION -void atomic_and(volatile T* const dest, desul::Impl::dont_deduce_this_parameter_t val) { (void) desul::atomic_fetch_and (const_cast(dest), val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); } - -// FIXME: Desul doesn't have atomic_or yet so call fetch_or -template KOKKOS_INLINE_FUNCTION -void atomic_or (volatile T* const dest, desul::Impl::dont_deduce_this_parameter_t val) { (void) desul::atomic_fetch_or (const_cast(dest), val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); } - -template KOKKOS_INLINE_FUNCTION -void atomic_inc(volatile T* const dest) { return desul::atomic_inc(const_cast(dest),desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); } - -template KOKKOS_INLINE_FUNCTION -void atomic_dec(volatile T* const dest) { return desul::atomic_dec(const_cast(dest),desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); } - -template KOKKOS_INLINE_FUNCTION -void atomic_increment(volatile T* const dest) { return desul::atomic_inc(const_cast(dest),desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); } - -template KOKKOS_INLINE_FUNCTION -void atomic_decrement(volatile T* const dest) { return desul::atomic_dec(const_cast(dest),desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); } - -// Exchange - -template KOKKOS_INLINE_FUNCTION -T atomic_exchange(volatile T* const dest, desul::Impl::dont_deduce_this_parameter_t val) { return desul::atomic_exchange(const_cast(dest), val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); } - -template KOKKOS_INLINE_FUNCTION -bool atomic_compare_exchange_strong(volatile T* const dest, T& expected, const T desired) { - return desul::atomic_compare_exchange_strong(const_cast(dest),expected, desired, - desul::MemoryOrderRelaxed(), desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); -} - -template KOKKOS_INLINE_FUNCTION -T atomic_compare_exchange(volatile T* const dest, const T compare, const T desired) { - return desul::atomic_compare_exchange(const_cast(dest),compare, desired, - desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); -} - -} -#undef KOKKOS_DESUL_MEM_SCOPE - -// clang-format on -#endif diff --git a/lib/kokkos/core/src/Kokkos_Atomics_Desul_Wrapper.hpp b/lib/kokkos/core/src/Kokkos_Atomics_Desul_Wrapper.hpp index 26db69ac1f..40f51c5a33 100644 --- a/lib/kokkos/core/src/Kokkos_Atomics_Desul_Wrapper.hpp +++ b/lib/kokkos/core/src/Kokkos_Atomics_Desul_Wrapper.hpp @@ -24,14 +24,16 @@ static_assert(false, #include #include +#include // identity_type #include -// clang-format off namespace Kokkos { -// FIXME: These functions don't have any use/test in unit tests ... -// ========================================================== -inline const char* atomic_query_version() { return "KOKKOS_DESUL_ATOMICS"; } +#ifdef KOKKOS_ENABLE_DEPRECATED_CODE_4 +KOKKOS_DEPRECATED inline const char* atomic_query_version() { + return "KOKKOS_DESUL_ATOMICS"; +} +#endif #if defined(KOKKOS_COMPILER_GNU) && !defined(__PGIC__) && \ !defined(__CUDA_ARCH__) @@ -53,197 +55,120 @@ inline const char* atomic_query_version() { return "KOKKOS_DESUL_ATOMICS"; } #define KOKKOS_DESUL_MEM_SCOPE desul::MemoryScopeDevice() #endif -template KOKKOS_INLINE_FUNCTION -T atomic_load(T* const dest) { return desul::atomic_load(dest, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); } - -template KOKKOS_INLINE_FUNCTION -void atomic_store(T* const dest, desul::Impl::dont_deduce_this_parameter_t val) { return desul::atomic_store(dest, val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); } - -template KOKKOS_INLINE_FUNCTION -void atomic_assign(T* const dest, desul::Impl::dont_deduce_this_parameter_t val) { atomic_store(dest,val); } - -KOKKOS_INLINE_FUNCTION -void memory_fence() { - desul::atomic_thread_fence(desul::MemoryOrderSeqCst(), KOKKOS_DESUL_MEM_SCOPE); -} - -KOKKOS_INLINE_FUNCTION -void load_fence() { return desul::atomic_thread_fence(desul::MemoryOrderAcquire(), KOKKOS_DESUL_MEM_SCOPE); } - -KOKKOS_INLINE_FUNCTION -void store_fence() { return desul::atomic_thread_fence(desul::MemoryOrderRelease(), KOKKOS_DESUL_MEM_SCOPE); } - -// atomic_fetch_op -template KOKKOS_INLINE_FUNCTION -T atomic_fetch_add (T* const dest, desul::Impl::dont_deduce_this_parameter_t val) { return desul::atomic_fetch_add (dest, val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); } - -template KOKKOS_INLINE_FUNCTION -T atomic_fetch_sub (T* const dest, desul::Impl::dont_deduce_this_parameter_t val) { return desul::atomic_fetch_sub (dest, val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); } - -template KOKKOS_INLINE_FUNCTION -T atomic_fetch_max (T* const dest, desul::Impl::dont_deduce_this_parameter_t val) { return desul::atomic_fetch_max (dest, val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); } - -template KOKKOS_INLINE_FUNCTION -T atomic_fetch_min (T* const dest, desul::Impl::dont_deduce_this_parameter_t val) { return desul::atomic_fetch_min (dest, val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); } - -template KOKKOS_INLINE_FUNCTION -T atomic_fetch_mul (T* const dest, desul::Impl::dont_deduce_this_parameter_t val) { return desul::atomic_fetch_mul (dest, val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); } - -template KOKKOS_INLINE_FUNCTION -T atomic_fetch_div (T* const dest, desul::Impl::dont_deduce_this_parameter_t val) { return desul::atomic_fetch_div (dest, val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); } - -template KOKKOS_INLINE_FUNCTION -T atomic_fetch_mod (T* const dest, desul::Impl::dont_deduce_this_parameter_t val) { return desul::atomic_fetch_mod (dest, val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); } - -template KOKKOS_INLINE_FUNCTION -T atomic_fetch_and (T* const dest, desul::Impl::dont_deduce_this_parameter_t val) { return desul::atomic_fetch_and (dest, val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); } - -template KOKKOS_INLINE_FUNCTION -T atomic_fetch_or (T* const dest, desul::Impl::dont_deduce_this_parameter_t val) { return desul::atomic_fetch_or (dest, val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); } - -template KOKKOS_INLINE_FUNCTION -T atomic_fetch_xor (T* const dest, desul::Impl::dont_deduce_this_parameter_t val) { return desul::atomic_fetch_xor (dest, val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); } - -template KOKKOS_INLINE_FUNCTION -T atomic_fetch_nand(T* const dest, desul::Impl::dont_deduce_this_parameter_t val) { return desul::atomic_fetch_nand(dest, val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); } - -template KOKKOS_INLINE_FUNCTION -T atomic_fetch_lshift(T* const dest, desul::Impl::dont_deduce_this_parameter_t val) { return desul::atomic_fetch_lshift(dest, val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); } - -template KOKKOS_INLINE_FUNCTION -T atomic_fetch_rshift(T* const dest, desul::Impl::dont_deduce_this_parameter_t val) { return desul::atomic_fetch_rshift(dest, val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); } - -template KOKKOS_INLINE_FUNCTION -T atomic_fetch_inc(T* const dest) { return desul::atomic_fetch_inc(dest, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); } - -template KOKKOS_INLINE_FUNCTION -T atomic_fetch_dec(T* const dest) { return desul::atomic_fetch_dec(dest, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); } - - -// atomic_op_fetch -template KOKKOS_INLINE_FUNCTION -T atomic_add_fetch (T* const dest, desul::Impl::dont_deduce_this_parameter_t val) { return desul::atomic_add_fetch (dest, val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); } - -template KOKKOS_INLINE_FUNCTION -T atomic_sub_fetch (T* const dest, desul::Impl::dont_deduce_this_parameter_t val) { return desul::atomic_sub_fetch (dest, val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); } - -template KOKKOS_INLINE_FUNCTION -T atomic_max_fetch (T* const dest, desul::Impl::dont_deduce_this_parameter_t val) { return desul::atomic_max_fetch (dest, val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); } - -template KOKKOS_INLINE_FUNCTION -T atomic_min_fetch (T* const dest, desul::Impl::dont_deduce_this_parameter_t val) { return desul::atomic_min_fetch (dest, val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); } - -template KOKKOS_INLINE_FUNCTION -T atomic_mul_fetch (T* const dest, desul::Impl::dont_deduce_this_parameter_t val) { return desul::atomic_mul_fetch (dest, val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); } - -template KOKKOS_INLINE_FUNCTION -T atomic_div_fetch (T* const dest, desul::Impl::dont_deduce_this_parameter_t val) { return desul::atomic_div_fetch (dest, val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); } - -template KOKKOS_INLINE_FUNCTION -T atomic_mod_fetch (T* const dest, desul::Impl::dont_deduce_this_parameter_t val) { return desul::atomic_mod_fetch (dest, val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); } - -template KOKKOS_INLINE_FUNCTION -T atomic_and_fetch (T* const dest, desul::Impl::dont_deduce_this_parameter_t val) { return desul::atomic_and_fetch (dest, val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); } - -template KOKKOS_INLINE_FUNCTION -T atomic_or_fetch (T* const dest, desul::Impl::dont_deduce_this_parameter_t val) { return desul::atomic_or_fetch (dest, val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); } - -template KOKKOS_INLINE_FUNCTION -T atomic_xor_fetch (T* const dest, desul::Impl::dont_deduce_this_parameter_t val) { return desul::atomic_xor_fetch (dest, val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); } - -template KOKKOS_INLINE_FUNCTION -T atomic_nand_fetch(T* const dest, desul::Impl::dont_deduce_this_parameter_t val) { return desul::atomic_nand_fetch(dest, val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); } - -template KOKKOS_INLINE_FUNCTION -T atomic_lshift_fetch(T* const dest, desul::Impl::dont_deduce_this_parameter_t val) { return desul::atomic_lshift_fetch(dest, val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); } - -template KOKKOS_INLINE_FUNCTION -T atomic_rshift_fetch(T* const dest, desul::Impl::dont_deduce_this_parameter_t val) { return desul::atomic_rshift_fetch(dest, val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); } - -template KOKKOS_INLINE_FUNCTION -T atomic_inc_fetch(T* const dest) { return desul::atomic_inc_fetch(dest, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); } - -template KOKKOS_INLINE_FUNCTION -T atomic_dec_fetch(T* const dest) { return desul::atomic_dec_fetch(dest, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); } - - -// atomic_op -template KOKKOS_INLINE_FUNCTION -void atomic_add(T* const dest, desul::Impl::dont_deduce_this_parameter_t val) { return desul::atomic_add (dest, val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); } - -template KOKKOS_INLINE_FUNCTION -void atomic_sub(T* const dest, desul::Impl::dont_deduce_this_parameter_t val) { return desul::atomic_sub (dest, val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); } - -template KOKKOS_INLINE_FUNCTION -void atomic_mul(T* const dest, desul::Impl::dont_deduce_this_parameter_t val) { return desul::atomic_mul (dest, val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); } - -template KOKKOS_INLINE_FUNCTION -void atomic_div(T* const dest, desul::Impl::dont_deduce_this_parameter_t val) { return desul::atomic_div (dest, val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); } - -template KOKKOS_INLINE_FUNCTION -void atomic_min(T* const dest, desul::Impl::dont_deduce_this_parameter_t val) { return desul::atomic_min (dest, val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); } - -template KOKKOS_INLINE_FUNCTION -void atomic_max(T* const dest, desul::Impl::dont_deduce_this_parameter_t val) { return desul::atomic_max (dest, val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); } - -// FIXME: Desul doesn't have atomic_and yet so call fetch_and -template KOKKOS_INLINE_FUNCTION -void atomic_and(T* const dest, desul::Impl::dont_deduce_this_parameter_t val) { (void) desul::atomic_fetch_and (dest, val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); } - -// FIXME: Desul doesn't have atomic_or yet so call fetch_or -template KOKKOS_INLINE_FUNCTION -void atomic_or(T* const dest, desul::Impl::dont_deduce_this_parameter_t val) { (void) desul::atomic_fetch_or (dest, val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); } - -template KOKKOS_INLINE_FUNCTION -void atomic_inc(T* const dest) { return desul::atomic_inc(dest, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); } - -template KOKKOS_INLINE_FUNCTION -void atomic_dec(T* const dest) { return desul::atomic_dec(dest, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); } - -template KOKKOS_INLINE_FUNCTION -void atomic_increment(T* const dest) { return desul::atomic_inc(dest, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); } - -template KOKKOS_INLINE_FUNCTION -void atomic_decrement(T* const dest) { return desul::atomic_dec(dest, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); } - -// Exchange - -template KOKKOS_INLINE_FUNCTION -T atomic_exchange(T* const dest, desul::Impl::dont_deduce_this_parameter_t val) { return desul::atomic_exchange(dest, val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); } - -template KOKKOS_INLINE_FUNCTION -bool atomic_compare_exchange_strong(T* const dest, desul::Impl::dont_deduce_this_parameter_t expected, desul::Impl::dont_deduce_this_parameter_t desired) { - T expected_ref = expected; - return desul::atomic_compare_exchange_strong(dest, expected_ref, desired, - desul::MemoryOrderRelaxed(), desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); -} - -template KOKKOS_INLINE_FUNCTION -T atomic_compare_exchange(T* const dest, desul::Impl::dont_deduce_this_parameter_t compare, desul::Impl::dont_deduce_this_parameter_t desired) { - return desul::atomic_compare_exchange(dest, compare, desired, - desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); -} - namespace Impl { - template KOKKOS_INLINE_FUNCTION - bool atomic_compare_exchange_strong(T* const dest, T& expected, const T desired, MemOrderSuccess succ, MemOrderFailure fail) { - return desul::atomic_compare_exchange_strong(dest, expected, desired, succ, fail, KOKKOS_DESUL_MEM_SCOPE); - } - template - KOKKOS_INLINE_FUNCTION - T atomic_load(const T* const src, MemoryOrder order) { - return desul::atomic_load(src, order, KOKKOS_DESUL_MEM_SCOPE); - } - template - KOKKOS_INLINE_FUNCTION - void atomic_store(T* const src, const T val, MemoryOrder order) { - return desul::atomic_store(src, val, order, KOKKOS_DESUL_MEM_SCOPE); - } +template +using not_deduced_atomic_t = + std::add_const_t>>; + +template +using enable_if_atomic_t = + std::enable_if_t && !std::is_const_v, + std::remove_volatile_t>; } // namespace Impl +// clang-format off + +// fences +KOKKOS_INLINE_FUNCTION void memory_fence() { desul::atomic_thread_fence(desul::MemoryOrderSeqCst(), KOKKOS_DESUL_MEM_SCOPE); } +KOKKOS_INLINE_FUNCTION void load_fence() { desul::atomic_thread_fence(desul::MemoryOrderAcquire(), KOKKOS_DESUL_MEM_SCOPE); } +KOKKOS_INLINE_FUNCTION void store_fence() { desul::atomic_thread_fence(desul::MemoryOrderRelease(), KOKKOS_DESUL_MEM_SCOPE); } + +// load/store +template KOKKOS_FUNCTION Impl::enable_if_atomic_t atomic_load (T const* ptr) { return desul::atomic_load (const_cast*>(ptr), desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); } +template KOKKOS_FUNCTION Impl::enable_if_atomic_t atomic_store(T* ptr, Impl::not_deduced_atomic_t val) { return desul::atomic_store(const_cast*>(ptr), val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); } +#ifdef KOKKOS_ENABLE_DEPRECATED_CODE_4 +template KOKKOS_DEPRECATED_WITH_COMMENT("Use atomic_store() instead!") KOKKOS_FUNCTION Impl::enable_if_atomic_t atomic_assign(T* ptr, Impl::not_deduced_atomic_t val) { atomic_store(ptr, val); } +#endif + +// atomic_fetch_op +template KOKKOS_FUNCTION Impl::enable_if_atomic_t atomic_fetch_add(T* ptr, Impl::not_deduced_atomic_t val) { return desul::atomic_fetch_add(const_cast*>(ptr), val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); } +template KOKKOS_FUNCTION Impl::enable_if_atomic_t atomic_fetch_sub(T* ptr, Impl::not_deduced_atomic_t val) { return desul::atomic_fetch_sub(const_cast*>(ptr), val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); } +template KOKKOS_FUNCTION Impl::enable_if_atomic_t atomic_fetch_max(T* ptr, Impl::not_deduced_atomic_t val) { return desul::atomic_fetch_max(const_cast*>(ptr), val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); } +template KOKKOS_FUNCTION Impl::enable_if_atomic_t atomic_fetch_min(T* ptr, Impl::not_deduced_atomic_t val) { return desul::atomic_fetch_min(const_cast*>(ptr), val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); } +template KOKKOS_FUNCTION Impl::enable_if_atomic_t atomic_fetch_mul(T* ptr, Impl::not_deduced_atomic_t val) { return desul::atomic_fetch_mul(const_cast*>(ptr), val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); } +template KOKKOS_FUNCTION Impl::enable_if_atomic_t atomic_fetch_div(T* ptr, Impl::not_deduced_atomic_t val) { return desul::atomic_fetch_div(const_cast*>(ptr), val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); } +template KOKKOS_FUNCTION Impl::enable_if_atomic_t atomic_fetch_mod(T* ptr, Impl::not_deduced_atomic_t val) { return desul::atomic_fetch_mod(const_cast*>(ptr), val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); } +template KOKKOS_FUNCTION Impl::enable_if_atomic_t atomic_fetch_and(T* ptr, Impl::not_deduced_atomic_t val) { return desul::atomic_fetch_and(const_cast*>(ptr), val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); } +template KOKKOS_FUNCTION Impl::enable_if_atomic_t atomic_fetch_or (T* ptr, Impl::not_deduced_atomic_t val) { return desul::atomic_fetch_or (const_cast*>(ptr), val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); } +template KOKKOS_FUNCTION Impl::enable_if_atomic_t atomic_fetch_xor(T* ptr, Impl::not_deduced_atomic_t val) { return desul::atomic_fetch_xor(const_cast*>(ptr), val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); } +template KOKKOS_FUNCTION Impl::enable_if_atomic_t atomic_fetch_nand(T* ptr, Impl::not_deduced_atomic_t val) { return desul::atomic_fetch_nand(const_cast*>(ptr), val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); } +template KOKKOS_FUNCTION Impl::enable_if_atomic_t atomic_fetch_lshift(T* ptr, Impl::not_deduced_atomic_t val) { return desul::atomic_fetch_lshift(const_cast*>(ptr), val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); } +template KOKKOS_FUNCTION Impl::enable_if_atomic_t atomic_fetch_rshift(T* ptr, Impl::not_deduced_atomic_t val) { return desul::atomic_fetch_rshift(const_cast*>(ptr), val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); } +template KOKKOS_FUNCTION Impl::enable_if_atomic_t atomic_fetch_inc(T* ptr) { return desul::atomic_fetch_inc(const_cast*>(ptr), desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); } +template KOKKOS_FUNCTION Impl::enable_if_atomic_t atomic_fetch_dec(T* ptr) { return desul::atomic_fetch_dec(const_cast*>(ptr), desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); } + +// atomic_op_fetch +template KOKKOS_FUNCTION Impl::enable_if_atomic_t atomic_add_fetch(T* ptr, Impl::not_deduced_atomic_t val) { return desul::atomic_add_fetch(const_cast*>(ptr), val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); } +template KOKKOS_FUNCTION Impl::enable_if_atomic_t atomic_sub_fetch(T* ptr, Impl::not_deduced_atomic_t val) { return desul::atomic_sub_fetch(const_cast*>(ptr), val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); } +template KOKKOS_FUNCTION Impl::enable_if_atomic_t atomic_max_fetch(T* ptr, Impl::not_deduced_atomic_t val) { return desul::atomic_max_fetch(const_cast*>(ptr), val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); } +template KOKKOS_FUNCTION Impl::enable_if_atomic_t atomic_min_fetch(T* ptr, Impl::not_deduced_atomic_t val) { return desul::atomic_min_fetch(const_cast*>(ptr), val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); } +template KOKKOS_FUNCTION Impl::enable_if_atomic_t atomic_mul_fetch(T* ptr, Impl::not_deduced_atomic_t val) { return desul::atomic_mul_fetch(const_cast*>(ptr), val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); } +template KOKKOS_FUNCTION Impl::enable_if_atomic_t atomic_div_fetch(T* ptr, Impl::not_deduced_atomic_t val) { return desul::atomic_div_fetch(const_cast*>(ptr), val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); } +template KOKKOS_FUNCTION Impl::enable_if_atomic_t atomic_mod_fetch(T* ptr, Impl::not_deduced_atomic_t val) { return desul::atomic_mod_fetch(const_cast*>(ptr), val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); } +template KOKKOS_FUNCTION Impl::enable_if_atomic_t atomic_and_fetch(T* ptr, Impl::not_deduced_atomic_t val) { return desul::atomic_and_fetch(const_cast*>(ptr), val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); } +template KOKKOS_FUNCTION Impl::enable_if_atomic_t atomic_or_fetch (T* ptr, Impl::not_deduced_atomic_t val) { return desul::atomic_or_fetch (const_cast*>(ptr), val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); } +template KOKKOS_FUNCTION Impl::enable_if_atomic_t atomic_xor_fetch(T* ptr, Impl::not_deduced_atomic_t val) { return desul::atomic_xor_fetch(const_cast*>(ptr), val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); } +template KOKKOS_FUNCTION Impl::enable_if_atomic_t atomic_nand_fetch(T* ptr, Impl::not_deduced_atomic_t val) { return desul::atomic_nand_fetch(const_cast*>(ptr), val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); } +template KOKKOS_FUNCTION Impl::enable_if_atomic_t atomic_lshift_fetch(T* ptr, Impl::not_deduced_atomic_t val) { return desul::atomic_lshift_fetch(const_cast*>(ptr), val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); } +template KOKKOS_FUNCTION Impl::enable_if_atomic_t atomic_rshift_fetch(T* ptr, Impl::not_deduced_atomic_t val) { return desul::atomic_rshift_fetch(const_cast*>(ptr), val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); } +template KOKKOS_FUNCTION Impl::enable_if_atomic_t atomic_inc_fetch(T* ptr) { return desul::atomic_inc_fetch(const_cast*>(ptr), desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); } +template KOKKOS_FUNCTION Impl::enable_if_atomic_t atomic_dec_fetch(T* ptr) { return desul::atomic_dec_fetch(const_cast*>(ptr), desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); } + +// atomic_op +template KOKKOS_FUNCTION Impl::enable_if_atomic_t atomic_add(T* ptr, Impl::not_deduced_atomic_t val) { desul::atomic_add(const_cast*>(ptr), val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); } +template KOKKOS_FUNCTION Impl::enable_if_atomic_t atomic_sub(T* ptr, Impl::not_deduced_atomic_t val) { desul::atomic_sub(const_cast*>(ptr), val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); } +template KOKKOS_FUNCTION Impl::enable_if_atomic_t atomic_max(T* ptr, Impl::not_deduced_atomic_t val) { desul::atomic_max(const_cast*>(ptr), val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); } +template KOKKOS_FUNCTION Impl::enable_if_atomic_t atomic_min(T* ptr, Impl::not_deduced_atomic_t val) { desul::atomic_min(const_cast*>(ptr), val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); } +template KOKKOS_FUNCTION Impl::enable_if_atomic_t atomic_mul(T* ptr, Impl::not_deduced_atomic_t val) { desul::atomic_mul(const_cast*>(ptr), val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); } +template KOKKOS_FUNCTION Impl::enable_if_atomic_t atomic_div(T* ptr, Impl::not_deduced_atomic_t val) { desul::atomic_div(const_cast*>(ptr), val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); } +template KOKKOS_FUNCTION Impl::enable_if_atomic_t atomic_mod(T* ptr, Impl::not_deduced_atomic_t val) { (void)desul::atomic_fetch_mod(const_cast*>(ptr), val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); } +template KOKKOS_FUNCTION Impl::enable_if_atomic_t atomic_and(T* ptr, Impl::not_deduced_atomic_t val) { (void)desul::atomic_fetch_and(const_cast*>(ptr), val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); } +template KOKKOS_FUNCTION Impl::enable_if_atomic_t atomic_or (T* ptr, Impl::not_deduced_atomic_t val) { (void)desul::atomic_fetch_or (const_cast*>(ptr), val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); } +template KOKKOS_FUNCTION Impl::enable_if_atomic_t atomic_xor(T* ptr, Impl::not_deduced_atomic_t val) { (void)desul::atomic_fetch_xor(const_cast*>(ptr), val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); } +template KOKKOS_FUNCTION Impl::enable_if_atomic_t atomic_nand(T* ptr, Impl::not_deduced_atomic_t val) { (void)desul::atomic_nand_fetch(const_cast*>(ptr), val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); } +template KOKKOS_FUNCTION Impl::enable_if_atomic_t atomic_lshift(T* ptr, Impl::not_deduced_atomic_t val) { (void)desul::atomic_fetch_lshift(const_cast*>(ptr), val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); } +template KOKKOS_FUNCTION Impl::enable_if_atomic_t atomic_rshift(T* ptr, Impl::not_deduced_atomic_t val) { (void)desul::atomic_fetch_rshift(const_cast*>(ptr), val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); } +template KOKKOS_FUNCTION Impl::enable_if_atomic_t atomic_inc(T* ptr) { desul::atomic_inc(const_cast*>(ptr), desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); } +template KOKKOS_FUNCTION Impl::enable_if_atomic_t atomic_dec(T* ptr) { desul::atomic_dec(const_cast*>(ptr), desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); } +#ifdef KOKKOS_ENABLE_DEPRECATED_CODE_4 +template KOKKOS_DEPRECATED_WITH_COMMENT("Use atomic_inc() instead!") KOKKOS_FUNCTION Impl::enable_if_atomic_t atomic_increment(T* ptr) { atomic_inc(ptr); } +template KOKKOS_DEPRECATED_WITH_COMMENT("Use atomic_dec() instead!") KOKKOS_FUNCTION Impl::enable_if_atomic_t atomic_decrement(T* ptr) { atomic_dec(ptr); } +#endif + +// exchange +template KOKKOS_FUNCTION Impl::enable_if_atomic_t atomic_exchange (T* ptr, Impl::not_deduced_atomic_t val) { return desul::atomic_exchange (const_cast*>(ptr), val, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); } +template KOKKOS_FUNCTION Impl::enable_if_atomic_t atomic_compare_exchange(T* ptr, Impl::not_deduced_atomic_t expected, Impl::not_deduced_atomic_t desired) { return desul::atomic_compare_exchange(const_cast*>(ptr), expected, desired, desul::MemoryOrderRelaxed(), KOKKOS_DESUL_MEM_SCOPE); } +#ifdef KOKKOS_ENABLE_DEPRECATED_CODE_4 +template KOKKOS_DEPRECATED_WITH_COMMENT("Use atomic_compare_exchange() instead!") KOKKOS_FUNCTION Impl::enable_if_atomic_t atomic_compare_exchange_strong(T* ptr, Impl::not_deduced_atomic_t expected, Impl::not_deduced_atomic_t desired) { return expected == atomic_compare_exchange(ptr, expected, desired); } +#endif + +// clang-format on } // namespace Kokkos +namespace Kokkos::Impl { + +template +KOKKOS_FUNCTION bool atomic_compare_exchange_strong(T* const dest, T& expected, + const T desired, + MemOrderSuccess succ, + MemOrderFailure fail) { + return desul::atomic_compare_exchange_strong(dest, expected, desired, succ, + fail, KOKKOS_DESUL_MEM_SCOPE); +} + +template +KOKKOS_FUNCTION T atomic_load(const T* const src, MemoryOrder order) { + return desul::atomic_load(src, order, KOKKOS_DESUL_MEM_SCOPE); +} + +template +KOKKOS_FUNCTION void atomic_store(T* const src, const T val, + MemoryOrder order) { + return desul::atomic_store(src, val, order, KOKKOS_DESUL_MEM_SCOPE); +} + +} // namespace Kokkos::Impl + #undef KOKKOS_DESUL_MEM_SCOPE -// clang-format on #endif diff --git a/lib/kokkos/core/src/Kokkos_Complex.hpp b/lib/kokkos/core/src/Kokkos_Complex.hpp index 7dd2a9ddbb..8233c30b24 100644 --- a/lib/kokkos/core/src/Kokkos_Complex.hpp +++ b/lib/kokkos/core/src/Kokkos_Complex.hpp @@ -70,9 +70,8 @@ class complex& operator=(const complex&) noexcept = default; /// \brief Conversion constructor from compatible RType - template < - class RType, - std::enable_if_t::value, int> = 0> + template , int> = 0> KOKKOS_INLINE_FUNCTION complex(const complex& other) noexcept // Intentionally do the conversions implicitly here so that users don't // get any warnings about narrowing, etc., that they would expect to get @@ -265,9 +264,8 @@ class #ifdef KOKKOS_ENABLE_DEPRECATED_CODE_4 //! Copy constructor from volatile. - template < - class RType, - std::enable_if_t::value, int> = 0> + template , int> = 0> KOKKOS_DEPRECATED KOKKOS_INLINE_FUNCTION complex(const volatile complex& src) noexcept // Intentionally do the conversions implicitly here so that users don't @@ -296,7 +294,7 @@ class // vl = r; // vl = cr; template ::value, int> = 0> + std::enable_if_t, int> = 0> KOKKOS_DEPRECATED KOKKOS_INLINE_FUNCTION void operator=( const Complex& src) volatile noexcept { re_ = src.re_; @@ -319,7 +317,7 @@ class // vl = vr; // vl = cvr; template ::value, int> = 0> + std::enable_if_t, int> = 0> KOKKOS_DEPRECATED KOKKOS_INLINE_FUNCTION volatile complex& operator=( const volatile Complex& src) volatile noexcept { re_ = src.re_; @@ -341,7 +339,7 @@ class // l = cvr; // template ::value, int> = 0> + std::enable_if_t, int> = 0> KOKKOS_DEPRECATED KOKKOS_INLINE_FUNCTION complex& operator=( const volatile Complex& src) noexcept { re_ = src.re_; @@ -539,7 +537,7 @@ inline bool operator==(complex const& x, template < class RealType1, class RealType2, // Constraints to avoid participation in oparator==() for every possible RHS - std::enable_if_t::value, int> = 0> + std::enable_if_t, int> = 0> KOKKOS_INLINE_FUNCTION bool operator==(complex const& x, RealType2 const& y) noexcept { using common_type = std::common_type_t; @@ -551,7 +549,7 @@ KOKKOS_INLINE_FUNCTION bool operator==(complex const& x, template < class RealType1, class RealType2, // Constraints to avoid participation in oparator==() for every possible RHS - std::enable_if_t::value, int> = 0> + std::enable_if_t, int> = 0> KOKKOS_INLINE_FUNCTION bool operator==(RealType1 const& x, complex const& y) noexcept { using common_type = std::common_type_t; @@ -590,7 +588,7 @@ inline bool operator!=(complex const& x, template < class RealType1, class RealType2, // Constraints to avoid participation in oparator==() for every possible RHS - std::enable_if_t::value, int> = 0> + std::enable_if_t, int> = 0> KOKKOS_INLINE_FUNCTION bool operator!=(complex const& x, RealType2 const& y) noexcept { using common_type = std::common_type_t; @@ -602,7 +600,7 @@ KOKKOS_INLINE_FUNCTION bool operator!=(complex const& x, template < class RealType1, class RealType2, // Constraints to avoid participation in oparator==() for every possible RHS - std::enable_if_t::value, int> = 0> + std::enable_if_t, int> = 0> KOKKOS_INLINE_FUNCTION bool operator!=(RealType1 const& x, complex const& y) noexcept { using common_type = std::common_type_t; @@ -778,16 +776,14 @@ KOKKOS_INLINE_FUNCTION complex pow(const complex& x, return x == T() ? T() : exp(y * log(x)); } -template ::value>> +template >> KOKKOS_INLINE_FUNCTION complex> pow( const T& x, const complex& y) { using type = Impl::promote_2_t; return pow(type(x), complex(y)); } -template ::value>> +template >> KOKKOS_INLINE_FUNCTION complex> pow(const complex& x, const U& y) { using type = Impl::promote_2_t; diff --git a/lib/kokkos/core/src/Kokkos_Concepts.hpp b/lib/kokkos/core/src/Kokkos_Concepts.hpp index df78a644a0..0bfb9eb5fa 100644 --- a/lib/kokkos/core/src/Kokkos_Concepts.hpp +++ b/lib/kokkos/core/src/Kokkos_Concepts.hpp @@ -41,8 +41,7 @@ struct Dynamic {}; // Schedule Wrapper Type template struct Schedule { - static_assert(std::is_same::value || - std::is_same::value, + static_assert(std::is_same_v || std::is_same_v, "Kokkos: Invalid Schedule<> type."); using schedule_type = Schedule; using type = T; @@ -51,7 +50,7 @@ struct Schedule { // Specify Iteration Index Type template struct IndexType { - static_assert(std::is_integral::value, "Kokkos: Invalid IndexType<>."); + static_assert(std::is_integral_v, "Kokkos: Invalid IndexType<>."); using index_type = IndexType; using type = T; }; @@ -139,8 +138,8 @@ namespace Kokkos { \ public: \ static constexpr bool value = \ - std::is_base_of, T>::value || \ - std::is_base_of, T>::value; \ + std::is_base_of_v, T> || \ + std::is_base_of_v, T>; \ constexpr operator bool() const noexcept { return value; } \ }; \ template \ @@ -292,44 +291,6 @@ struct is_space { using execution_space = typename is_exe::space; using memory_space = typename is_mem::space; - - // For backward compatibility, deprecated in favor of - // Kokkos::Impl::HostMirror::host_mirror_space - - private: - // The actual definitions for host_memory_space and host_execution_spaces are - // in do_not_use_host_memory_space and do_not_use_host_execution_space to be - // able to use them within this class without deprecation warnings. - using do_not_use_host_memory_space = std::conditional_t< - std::is_same::value -#if defined(KOKKOS_ENABLE_CUDA) - || std::is_same::value || - std::is_same::value -#elif defined(KOKKOS_ENABLE_HIP) - || std::is_same::value || - std::is_same::value -#elif defined(KOKKOS_ENABLE_SYCL) - || std::is_same::value || - std::is_same::value -#endif - , - memory_space, Kokkos::HostSpace>; - - using do_not_use_host_execution_space = std::conditional_t< -#if defined(KOKKOS_ENABLE_CUDA) - std::is_same::value || -#elif defined(KOKKOS_ENABLE_HIP) - std::is_same::value || -#elif defined(KOKKOS_ENABLE_SYCL) - std::is_same::value || -#elif defined(KOKKOS_ENABLE_OPENMPTARGET) - std::is_same::value || -#endif - false, - Kokkos::DefaultHostExecutionSpace, execution_space>; }; } // namespace Kokkos @@ -357,7 +318,7 @@ struct MemorySpaceAccess { * 2. All execution spaces that can access DstMemorySpace can also access * SrcMemorySpace. */ - enum { assignable = std::is_same::value }; + enum { assignable = std::is_same_v }; /**\brief For all DstExecSpace::memory_space == DstMemorySpace * DstExecSpace can access SrcMemorySpace. @@ -442,7 +403,7 @@ struct SpaceAccessibility { // If same memory space or not accessible use the AccessSpace // else construct a device with execution space and memory space. using space = std::conditional_t< - std::is_same::value || + std::is_same_v || !exe_access::accessible, AccessSpace, Kokkos::Device>; diff --git a/lib/kokkos/core/src/Kokkos_CopyViews.hpp b/lib/kokkos/core/src/Kokkos_CopyViews.hpp index e856b19247..7da59aa4e4 100644 --- a/lib/kokkos/core/src/Kokkos_CopyViews.hpp +++ b/lib/kokkos/core/src/Kokkos_CopyViews.hpp @@ -561,21 +561,20 @@ void view_copy(const ExecutionSpace& space, const DstType& dst, int64_t strides[DstType::rank + 1]; dst.stride(strides); Kokkos::Iterate iterate; - if (std::is_same::value) { + if (std::is_same_v) { iterate = Kokkos::Iterate::Right; - } else if (std::is_same::value) { + } else if (std::is_same_v) { iterate = Kokkos::Iterate::Left; - } else if (std::is_same::value) { + } else if (std::is_same_v) { if (strides[0] > strides[DstType::rank - 1]) iterate = Kokkos::Iterate::Right; else iterate = Kokkos::Iterate::Left; } else { - if (std::is_same::value) + if (std::is_same_v) iterate = Kokkos::Iterate::Right; else iterate = Kokkos::Iterate::Left; @@ -649,21 +648,20 @@ void view_copy(const DstType& dst, const SrcType& src) { int64_t strides[DstType::rank + 1]; dst.stride(strides); Kokkos::Iterate iterate; - if (std::is_same::value) { + if (std::is_same_v) { iterate = Kokkos::Iterate::Right; - } else if (std::is_same::value) { + } else if (std::is_same_v) { iterate = Kokkos::Iterate::Left; - } else if (std::is_same::value) { + } else if (std::is_same_v) { if (strides[0] > strides[DstType::rank - 1]) iterate = Kokkos::Iterate::Right; else iterate = Kokkos::Iterate::Left; } else { - if (std::is_same::value) + if (std::is_same_v) iterate = Kokkos::Iterate::Right; else iterate = Kokkos::Iterate::Left; @@ -1350,22 +1348,20 @@ inline void contiguous_fill( } // Default implementation for execution spaces that don't provide a definition -template +template struct ZeroMemset { - ZeroMemset(const ExecutionSpace& exec_space, const ViewType& dst) { - using ValueType = typename ViewType::value_type; - alignas(alignof(ValueType)) unsigned char - zero_initialized_storage[sizeof(ValueType)] = {}; - contiguous_fill(exec_space, dst, - *reinterpret_cast(zero_initialized_storage)); + ZeroMemset(const ExecutionSpace& exec_space, void* dst, size_t cnt) { + contiguous_fill( + exec_space, + Kokkos::View( + static_cast(dst), cnt), + std::byte{}); } }; template inline std::enable_if_t< - std::is_trivial::value_type>::value && - std::is_trivially_copy_assignable< - typename ViewTraits::value_type>::value> + std::is_trivial_v::value_type>> contiguous_fill_or_memset( const ExecutionSpace& exec_space, const View& dst, typename ViewTraits::const_value_type& value) { @@ -1375,20 +1371,20 @@ contiguous_fill_or_memset( && !std::is_same_v #endif ) - // FIXME intel/19 icpc fails to deduce template parameters here, + // FIXME intel/19 icpc fails to deduce template parameter here, // resulting in compilation errors; explicitly passing the template - // parameters to ZeroMemset helps workaround the issue - // See https://github.com/kokkos/kokkos/issues/6775 - ZeroMemset>(exec_space, dst); + // parameter to ZeroMemset helps workaround the issue. + // See https://github.com/kokkos/kokkos/issues/7273. + ZeroMemset( + exec_space, dst.data(), + dst.size() * sizeof(typename ViewTraits::value_type)); else contiguous_fill(exec_space, dst, value); } template inline std::enable_if_t< - !(std::is_trivial::value_type>::value && - std::is_trivially_copy_assignable< - typename ViewTraits::value_type>::value)> + !std::is_trivial_v::value_type>> contiguous_fill_or_memset( const ExecutionSpace& exec_space, const View& dst, typename ViewTraits::const_value_type& value) { @@ -1397,9 +1393,7 @@ contiguous_fill_or_memset( template inline std::enable_if_t< - std::is_trivial::value_type>::value && - std::is_trivially_copy_assignable< - typename ViewTraits::value_type>::value> + std::is_trivial_v::value_type>> contiguous_fill_or_memset( const View& dst, typename ViewTraits::const_value_type& value) { @@ -1411,11 +1405,12 @@ contiguous_fill_or_memset( // leading to the significant performance issues #ifndef KOKKOS_ARCH_A64FX if (Impl::is_zero_byte(value)) - // FIXME intel/19 icpc fails to deduce template parameters here, + // FIXME intel/19 icpc fails to deduce template parameter here, // resulting in compilation errors; explicitly passing the template - // parameters to ZeroMemset helps workaround the issue - // See https://github.com/kokkos/kokkos/issues/6775 - ZeroMemset(exec, dst); + // parameter to ZeroMemset helps workaround the issue. + // See https://github.com/kokkos/kokkos/issues/7273. + ZeroMemset( + exec, dst.data(), dst.size() * sizeof(typename ViewType::value_type)); else #endif contiguous_fill(exec, dst, value); @@ -1423,9 +1418,7 @@ contiguous_fill_or_memset( template inline std::enable_if_t< - !(std::is_trivial::value_type>::value && - std::is_trivially_copy_assignable< - typename ViewTraits::value_type>::value)> + !std::is_trivial_v::value_type>> contiguous_fill_or_memset( const View& dst, typename ViewTraits::const_value_type& value) { @@ -1441,8 +1434,8 @@ template inline void deep_copy( const View& dst, typename ViewTraits::const_value_type& value, - std::enable_if_t::specialize, - void>::value>* = nullptr) { + std::enable_if_t::specialize, + void>>* = nullptr) { using ViewType = View; using exec_space_type = typename ViewType::execution_space; @@ -1464,8 +1457,8 @@ inline void deep_copy( } Kokkos::fence("Kokkos::deep_copy: scalar copy, pre copy fence"); - static_assert(std::is_same::value, + static_assert(std::is_same_v, "deep_copy requires non-const type"); // If contiguous we can simply do a 1D flat loop or use memset @@ -1482,21 +1475,20 @@ inline void deep_copy( int64_t strides[ViewType::rank + 1]; dst.stride(strides); Kokkos::Iterate iterate; - if (std::is_same::value) { + if (std::is_same_v) { iterate = Kokkos::Iterate::Right; - } else if (std::is_same::value) { + } else if (std::is_same_v) { iterate = Kokkos::Iterate::Left; - } else if (std::is_same::value) { + } else if (std::is_same_v) { if (strides[0] > strides[ViewType::rank > 0 ? ViewType::rank - 1 : 0]) iterate = Kokkos::Iterate::Right; else iterate = Kokkos::Iterate::Left; } else { - if (std::is_same::value) + if (std::is_same_v) iterate = Kokkos::Iterate::Right; else iterate = Kokkos::Iterate::Left; @@ -1539,8 +1531,8 @@ template inline void deep_copy( typename ViewTraits::non_const_value_type& dst, const View& src, - std::enable_if_t::specialize, - void>::value>* = nullptr) { + std::enable_if_t::specialize, + void>>* = nullptr) { using src_traits = ViewTraits; using src_memory_space = typename src_traits::memory_space; @@ -1576,8 +1568,8 @@ template inline void deep_copy( const View& dst, const View& src, std::enable_if_t< - (std::is_void::specialize>::value && - std::is_void::specialize>::value && + (std::is_void_v::specialize> && + std::is_void_v::specialize> && (unsigned(ViewTraits::rank) == unsigned(0) && unsigned(ViewTraits::rank) == unsigned(0)))>* = nullptr) { using dst_type = View; @@ -1587,8 +1579,8 @@ inline void deep_copy( using dst_memory_space = typename dst_type::memory_space; using src_memory_space = typename src_type::memory_space; - static_assert(std::is_same::value, + static_assert(std::is_same_v, "deep_copy requires matching non-const destination type"); if (Kokkos::Tools::Experimental::get_callbacks().begin_deep_copy != nullptr) { @@ -1628,8 +1620,8 @@ template inline void deep_copy( const View& dst, const View& src, std::enable_if_t< - (std::is_void::specialize>::value && - std::is_void::specialize>::value && + (std::is_void_v::specialize> && + std::is_void_v::specialize> && (unsigned(ViewTraits::rank) != 0 || unsigned(ViewTraits::rank) != 0))>* = nullptr) { using dst_type = View; @@ -1641,8 +1633,8 @@ inline void deep_copy( using dst_value_type = typename dst_type::value_type; using src_value_type = typename src_type::value_type; - static_assert(std::is_same::value, + static_assert(std::is_same_v, "deep_copy requires non-const destination type"); static_assert((unsigned(dst_type::rank) == unsigned(src_type::rank)), @@ -1772,10 +1764,10 @@ inline void deep_copy( // If same type, equal layout, equal dimensions, equal span, and contiguous // memory then can byte-wise copy - if (std::is_same::value && - (std::is_same::value || + if (std::is_same_v && + (std::is_same_v || (dst_type::rank == 1 && src_type::rank == 1)) && dst.span_is_contiguous() && src.span_is_contiguous() && ((dst_type::rank < 1) || (dst.stride_0() == src.stride_0())) && @@ -2191,8 +2183,8 @@ template void KOKKOS_INLINE_FUNCTION local_deep_copy_contiguous( const TeamType& team, const View& dst, typename ViewTraits::const_value_type& value, - std::enable_if_t::specialize, - void>::value>* = nullptr) { + std::enable_if_t::specialize, + void>>* = nullptr) { Kokkos::parallel_for(Kokkos::TeamVectorRange(team, dst.span()), [&](const int& i) { dst.data()[i] = value; }); } @@ -2201,8 +2193,8 @@ template void KOKKOS_INLINE_FUNCTION local_deep_copy_contiguous( const View& dst, typename ViewTraits::const_value_type& value, - std::enable_if_t::specialize, - void>::value>* = nullptr) { + std::enable_if_t::specialize, + void>>* = nullptr) { for (size_t i = 0; i < dst.span(); ++i) { dst.data()[i] = value; } @@ -2568,13 +2560,13 @@ inline void deep_copy( typename ViewTraits::const_value_type& value, std::enable_if_t< Kokkos::is_execution_space::value && - std::is_void::specialize>::value && + std::is_void_v::specialize> && Kokkos::SpaceAccessibility:: memory_space>::accessible>* = nullptr) { using dst_traits = ViewTraits; - static_assert(std::is_same::value, + static_assert(std::is_same_v, "deep_copy requires non-const type"); using dst_memory_space = typename dst_traits::memory_space; if (Kokkos::Tools::Experimental::get_callbacks().begin_deep_copy != nullptr) { @@ -2594,21 +2586,20 @@ inline void deep_copy( int64_t strides[ViewType::rank + 1]; dst.stride(strides); Kokkos::Iterate iterate; - if (std::is_same::value) { + if (std::is_same_v) { iterate = Kokkos::Iterate::Right; - } else if (std::is_same::value) { + } else if (std::is_same_v) { iterate = Kokkos::Iterate::Left; - } else if (std::is_same::value) { + } else if (std::is_same_v) { if (strides[0] > strides[ViewType::rank > 0 ? ViewType::rank - 1 : 0]) iterate = Kokkos::Iterate::Right; else iterate = Kokkos::Iterate::Left; } else { - if (std::is_same::value) + if (std::is_same_v) iterate = Kokkos::Iterate::Right; else iterate = Kokkos::Iterate::Left; @@ -2649,13 +2640,13 @@ inline void deep_copy( typename ViewTraits::const_value_type& value, std::enable_if_t< Kokkos::is_execution_space::value && - std::is_void::specialize>::value && + std::is_void_v::specialize> && !Kokkos::SpaceAccessibility:: memory_space>::accessible>* = nullptr) { using dst_traits = ViewTraits; - static_assert(std::is_same::value, + static_assert(std::is_same_v, "deep_copy requires non-const type"); using dst_memory_space = typename dst_traits::memory_space; if (Kokkos::Tools::Experimental::get_callbacks().begin_deep_copy != nullptr) { @@ -2696,8 +2687,8 @@ inline void deep_copy( typename ViewTraits::non_const_value_type& dst, const View& src, std::enable_if_t::value && - std::is_same::specialize, - void>::value>* = nullptr) { + std::is_same_v::specialize, + void>>* = nullptr) { using src_traits = ViewTraits; using src_memory_space = typename src_traits::memory_space; static_assert(src_traits::rank == 0, @@ -2734,8 +2725,8 @@ inline void deep_copy( const View& src, std::enable_if_t< (Kokkos::is_execution_space::value && - std::is_void::specialize>::value && - std::is_void::specialize>::value && + std::is_void_v::specialize> && + std::is_void_v::specialize> && (unsigned(ViewTraits::rank) == unsigned(0) && unsigned(ViewTraits::rank) == unsigned(0)))>* = nullptr) { using src_traits = ViewTraits; @@ -2743,8 +2734,8 @@ inline void deep_copy( using src_memory_space = typename src_traits::memory_space; using dst_memory_space = typename dst_traits::memory_space; - static_assert(std::is_same::value, + static_assert(std::is_same_v, "deep_copy requires matching non-const destination type"); if (Kokkos::Tools::Experimental::get_callbacks().begin_deep_copy != nullptr) { @@ -2784,15 +2775,15 @@ inline void deep_copy( const View& src, std::enable_if_t< (Kokkos::is_execution_space::value && - std::is_void::specialize>::value && - std::is_void::specialize>::value && + std::is_void_v::specialize> && + std::is_void_v::specialize> && (unsigned(ViewTraits::rank) != 0 || unsigned(ViewTraits::rank) != 0))>* = nullptr) { using dst_type = View; using src_type = View; - static_assert(std::is_same::value, + static_assert(std::is_same_v, "deep_copy requires non-const destination type"); static_assert((unsigned(dst_type::rank) == unsigned(src_type::rank)), @@ -2922,10 +2913,10 @@ inline void deep_copy( // If same type, equal layout, equal dimensions, equal span, and contiguous // memory then can byte-wise copy - if (std::is_same::value && - (std::is_same::value || + if (std::is_same_v && + (std::is_same_v || (dst_type::rank == 1 && src_type::rank == 1)) && dst.span_is_contiguous() && src.span_is_contiguous() && ((dst_type::rank < 1) || (dst.stride_0() == src.stride_0())) && @@ -2994,11 +2985,11 @@ bool size_mismatch(const ViewType& view, unsigned int max_extent, /** \brief Resize a view with copying old data to new data at the corresponding * indices. */ template -inline typename std::enable_if< - std::is_same::array_layout, - Kokkos::LayoutLeft>::value || - std::is_same::array_layout, - Kokkos::LayoutRight>::value>::type +inline std::enable_if_t< + std::is_same_v::array_layout, + Kokkos::LayoutLeft> || + std::is_same_v::array_layout, + Kokkos::LayoutRight>> impl_resize(const Impl::ViewCtorProp& arg_prop, Kokkos::View& v, const size_t n0, const size_t n1, const size_t n2, const size_t n3, const size_t n4, const size_t n5, @@ -3048,10 +3039,10 @@ impl_resize(const Impl::ViewCtorProp& arg_prop, template inline std::enable_if_t< - std::is_same::array_layout, - Kokkos::LayoutLeft>::value || - std::is_same::array_layout, - Kokkos::LayoutRight>::value> + std::is_same_v::array_layout, + Kokkos::LayoutLeft> || + std::is_same_v::array_layout, + Kokkos::LayoutRight>> resize(const Impl::ViewCtorProp& arg_prop, Kokkos::View& v, const size_t n0 = KOKKOS_IMPL_CTOR_DEFAULT_ARG, const size_t n1 = KOKKOS_IMPL_CTOR_DEFAULT_ARG, @@ -3066,10 +3057,10 @@ resize(const Impl::ViewCtorProp& arg_prop, template inline std::enable_if_t< - std::is_same::array_layout, - Kokkos::LayoutLeft>::value || - std::is_same::array_layout, - Kokkos::LayoutRight>::value> + std::is_same_v::array_layout, + Kokkos::LayoutLeft> || + std::is_same_v::array_layout, + Kokkos::LayoutRight>> resize(Kokkos::View& v, const size_t n0 = KOKKOS_IMPL_CTOR_DEFAULT_ARG, const size_t n1 = KOKKOS_IMPL_CTOR_DEFAULT_ARG, const size_t n2 = KOKKOS_IMPL_CTOR_DEFAULT_ARG, @@ -3085,10 +3076,10 @@ template inline std::enable_if_t< (Impl::is_view_ctor_property::value || Kokkos::is_execution_space::value) && - (std::is_same::array_layout, - Kokkos::LayoutLeft>::value || - std::is_same::array_layout, - Kokkos::LayoutRight>::value)> + (std::is_same_v::array_layout, + Kokkos::LayoutLeft> || + std::is_same_v::array_layout, + Kokkos::LayoutRight>)> resize(const I& arg_prop, Kokkos::View& v, const size_t n0 = KOKKOS_IMPL_CTOR_DEFAULT_ARG, const size_t n1 = KOKKOS_IMPL_CTOR_DEFAULT_ARG, @@ -3103,12 +3094,12 @@ resize(const I& arg_prop, Kokkos::View& v, template inline std::enable_if_t< - std::is_same::array_layout, - Kokkos::LayoutLeft>::value || - std::is_same::array_layout, - Kokkos::LayoutRight>::value || - std::is_same::array_layout, - Kokkos::LayoutStride>::value> + std::is_same_v::array_layout, + Kokkos::LayoutLeft> || + std::is_same_v::array_layout, + Kokkos::LayoutRight> || + std::is_same_v::array_layout, + Kokkos::LayoutStride>> impl_resize(const Impl::ViewCtorProp& arg_prop, Kokkos::View& v, const typename Kokkos::View::array_layout& layout) { @@ -3149,12 +3140,12 @@ impl_resize(const Impl::ViewCtorProp& arg_prop, // the same as the existing one. template inline std::enable_if_t< - !(std::is_same::array_layout, - Kokkos::LayoutLeft>::value || - std::is_same::array_layout, - Kokkos::LayoutRight>::value || - std::is_same::array_layout, - Kokkos::LayoutStride>::value)> + !(std::is_same_v::array_layout, + Kokkos::LayoutLeft> || + std::is_same_v::array_layout, + Kokkos::LayoutRight> || + std::is_same_v::array_layout, + Kokkos::LayoutStride>)> impl_resize(const Impl::ViewCtorProp& arg_prop, Kokkos::View& v, const typename Kokkos::View::array_layout& layout) { @@ -3218,10 +3209,10 @@ inline void resize(Kokkos::View& v, /** \brief Resize a view with discarding old data. */ template inline std::enable_if_t< - std::is_same::array_layout, - Kokkos::LayoutLeft>::value || - std::is_same::array_layout, - Kokkos::LayoutRight>::value> + std::is_same_v::array_layout, + Kokkos::LayoutLeft> || + std::is_same_v::array_layout, + Kokkos::LayoutRight>> impl_realloc(Kokkos::View& v, const size_t n0, const size_t n1, const size_t n2, const size_t n3, const size_t n4, const size_t n5, const size_t n6, const size_t n7, @@ -3264,10 +3255,10 @@ impl_realloc(Kokkos::View& v, const size_t n0, const size_t n1, template inline std::enable_if_t< - std::is_same::array_layout, - Kokkos::LayoutLeft>::value || - std::is_same::array_layout, - Kokkos::LayoutRight>::value> + std::is_same_v::array_layout, + Kokkos::LayoutLeft> || + std::is_same_v::array_layout, + Kokkos::LayoutRight>> realloc(const Impl::ViewCtorProp& arg_prop, Kokkos::View& v, const size_t n0 = KOKKOS_IMPL_CTOR_DEFAULT_ARG, @@ -3283,10 +3274,10 @@ realloc(const Impl::ViewCtorProp& arg_prop, template inline std::enable_if_t< - std::is_same::array_layout, - Kokkos::LayoutLeft>::value || - std::is_same::array_layout, - Kokkos::LayoutRight>::value> + std::is_same_v::array_layout, + Kokkos::LayoutLeft> || + std::is_same_v::array_layout, + Kokkos::LayoutRight>> realloc(Kokkos::View& v, const size_t n0 = KOKKOS_IMPL_CTOR_DEFAULT_ARG, const size_t n1 = KOKKOS_IMPL_CTOR_DEFAULT_ARG, @@ -3302,10 +3293,10 @@ realloc(Kokkos::View& v, template inline std::enable_if_t< Impl::is_view_ctor_property::value && - (std::is_same::array_layout, - Kokkos::LayoutLeft>::value || - std::is_same::array_layout, - Kokkos::LayoutRight>::value)> + (std::is_same_v::array_layout, + Kokkos::LayoutLeft> || + std::is_same_v::array_layout, + Kokkos::LayoutRight>)> realloc(const I& arg_prop, Kokkos::View& v, const size_t n0 = KOKKOS_IMPL_CTOR_DEFAULT_ARG, const size_t n1 = KOKKOS_IMPL_CTOR_DEFAULT_ARG, @@ -3320,12 +3311,12 @@ realloc(const I& arg_prop, Kokkos::View& v, template inline std::enable_if_t< - std::is_same::array_layout, - Kokkos::LayoutLeft>::value || - std::is_same::array_layout, - Kokkos::LayoutRight>::value || - std::is_same::array_layout, - Kokkos::LayoutStride>::value> + std::is_same_v::array_layout, + Kokkos::LayoutLeft> || + std::is_same_v::array_layout, + Kokkos::LayoutRight> || + std::is_same_v::array_layout, + Kokkos::LayoutStride>> impl_realloc(Kokkos::View& v, const typename Kokkos::View::array_layout& layout, const Impl::ViewCtorProp& arg_prop) { @@ -3365,12 +3356,12 @@ impl_realloc(Kokkos::View& v, // the same as the existing one. template inline std::enable_if_t< - !(std::is_same::array_layout, - Kokkos::LayoutLeft>::value || - std::is_same::array_layout, - Kokkos::LayoutRight>::value || - std::is_same::array_layout, - Kokkos::LayoutStride>::value)> + !(std::is_same_v::array_layout, + Kokkos::LayoutLeft> || + std::is_same_v::array_layout, + Kokkos::LayoutRight> || + std::is_same_v::array_layout, + Kokkos::LayoutStride>)> impl_realloc(Kokkos::View& v, const typename Kokkos::View::array_layout& layout, const Impl::ViewCtorProp& arg_prop) { @@ -3435,7 +3426,7 @@ struct MirrorViewType { // Check whether it is the same memory space enum { is_same_memspace = - std::is_same::value + std::is_same_v }; // The array_layout using array_layout = typename src_view_type::array_layout; @@ -3450,26 +3441,6 @@ struct MirrorViewType { std::conditional_t; }; -template -struct MirrorType { - // The incoming view_type - using src_view_type = typename Kokkos::View; - // The memory space for the mirror view - using memory_space = typename Space::memory_space; - // Check whether it is the same memory space - enum { - is_same_memspace = - std::is_same::value - }; - // The array_layout - using array_layout = typename src_view_type::array_layout; - // The data type (we probably want it non-const since otherwise we can't even - // deep_copy to it. - using data_type = typename src_view_type::non_const_data_type; - // The destination view type if it is not the same memory space - using view_type = Kokkos::View; -}; - // collection of static asserts for create_mirror and create_mirror_view template void check_view_ctor_args_create_mirror() { @@ -3503,7 +3474,7 @@ inline auto create_mirror(const Kokkos::View& src, if constexpr (Impl::ViewCtorProp::has_memory_space) { using memory_space = typename decltype(prop_copy)::memory_space; using dst_type = - typename Impl::MirrorType::view_type; + typename Impl::MirrorViewType::dest_view_type; return dst_type(prop_copy, src.layout()); } else { using dst_type = typename View::HostMirror; @@ -3636,12 +3607,12 @@ inline auto create_mirror_view( const Kokkos::View& src, [[maybe_unused]] const Impl::ViewCtorProp& arg_prop) { if constexpr (!Impl::ViewCtorProp::has_memory_space) { - if constexpr (std::is_same::memory_space, - typename Kokkos::View< - T, P...>::HostMirror::memory_space>::value && - std::is_same::data_type, - typename Kokkos::View< - T, P...>::HostMirror::data_type>::value) { + if constexpr (std::is_same_v::memory_space, + typename Kokkos::View< + T, P...>::HostMirror::memory_space> && + std::is_same_v< + typename Kokkos::View::data_type, + typename Kokkos::View::HostMirror::data_type>) { check_view_ctor_args_create_mirror(); return typename Kokkos::View::HostMirror(src); } else { @@ -3785,8 +3756,7 @@ create_mirror_view_and_copy( const Space&, const Kokkos::View& src, std::string const& name = "", std::enable_if_t< - std::is_void::specialize>::value>* = - nullptr) { + std::is_void_v::specialize>>* = nullptr) { return create_mirror_view_and_copy( Kokkos::view_alloc(typename Space::memory_space{}, name), src); } diff --git a/lib/kokkos/core/src/Kokkos_Core.hpp b/lib/kokkos/core/src/Kokkos_Core.hpp index 1f146563be..9588d289a9 100644 --- a/lib/kokkos/core/src/Kokkos_Core.hpp +++ b/lib/kokkos/core/src/Kokkos_Core.hpp @@ -63,7 +63,9 @@ #include #include #include +#ifdef KOKKOS_ENABLE_DEPRECATED_CODE_4 #include +#endif #include #include #include @@ -248,9 +250,9 @@ class KOKKOS_ATTRIBUTE_NODISCARD ScopeGuard { } ScopeGuard& operator=(const ScopeGuard&) = delete; - ScopeGuard& operator=(ScopeGuard&&) = delete; - ScopeGuard(const ScopeGuard&) = delete; - ScopeGuard(ScopeGuard&&) = delete; + ScopeGuard& operator=(ScopeGuard&&) = delete; + ScopeGuard(const ScopeGuard&) = delete; + ScopeGuard(ScopeGuard&&) = delete; }; } // namespace Kokkos @@ -281,7 +283,7 @@ std::vector partition_space(ExecSpace const& space, "Kokkos Error: partition_space expects an Execution Space as " "first argument"); static_assert( - std::is_arithmetic::value, + std::is_arithmetic_v, "Kokkos Error: partitioning arguments must be integers or floats"); std::vector instances(weights.size()); diff --git a/lib/kokkos/core/src/Kokkos_Core_fwd.hpp b/lib/kokkos/core/src/Kokkos_Core_fwd.hpp index 7edb35f00e..5dbe571429 100644 --- a/lib/kokkos/core/src/Kokkos_Core_fwd.hpp +++ b/lib/kokkos/core/src/Kokkos_Core_fwd.hpp @@ -106,8 +106,7 @@ using DefaultExecutionSpace KOKKOS_IMPL_DEFAULT_EXEC_SPACE_ANNOTATION = #elif defined(KOKKOS_ENABLE_DEFAULT_DEVICE_TYPE_HIP) using DefaultExecutionSpace KOKKOS_IMPL_DEFAULT_EXEC_SPACE_ANNOTATION = HIP; #elif defined(KOKKOS_ENABLE_DEFAULT_DEVICE_TYPE_SYCL) -using DefaultExecutionSpace KOKKOS_IMPL_DEFAULT_EXEC_SPACE_ANNOTATION = - Experimental::SYCL; +using DefaultExecutionSpace KOKKOS_IMPL_DEFAULT_EXEC_SPACE_ANNOTATION = SYCL; #elif defined(KOKKOS_ENABLE_DEFAULT_DEVICE_TYPE_OPENACC) using DefaultExecutionSpace KOKKOS_IMPL_DEFAULT_EXEC_SPACE_ANNOTATION = Experimental::OpenACC; @@ -122,7 +121,7 @@ using DefaultExecutionSpace KOKKOS_IMPL_DEFAULT_EXEC_SPACE_ANNOTATION = using DefaultExecutionSpace KOKKOS_IMPL_DEFAULT_EXEC_SPACE_ANNOTATION = Serial; #else #error \ - "At least one of the following execution spaces must be defined in order to use Kokkos: Kokkos::Cuda, Kokkos::HIP, Kokkos::Experimental::SYCL, Kokkos::Experimental::OpenMPTarget, Kokkos::Experimental::OpenACC, Kokkos::OpenMP, Kokkos::Threads, Kokkos::Experimental::HPX, or Kokkos::Serial." + "At least one of the following execution spaces must be defined in order to use Kokkos: Kokkos::Cuda, Kokkos::HIP, Kokkos::SYCL, Kokkos::Experimental::OpenMPTarget, Kokkos::Experimental::OpenACC, Kokkos::OpenMP, Kokkos::Threads, Kokkos::Experimental::HPX, or Kokkos::Serial." #endif #if defined(KOKKOS_ENABLE_DEFAULT_DEVICE_TYPE_OPENMP) @@ -162,7 +161,7 @@ using SharedSpace = CudaUVMSpace; using SharedSpace = HIPManagedSpace; #define KOKKOS_HAS_SHARED_SPACE #elif defined(KOKKOS_ENABLE_SYCL) -using SharedSpace = Experimental::SYCLSharedUSMSpace; +using SharedSpace = SYCLSharedUSMSpace; #define KOKKOS_HAS_SHARED_SPACE // if only host compile point to HostSpace #elif !defined(KOKKOS_ENABLE_OPENACC) && !defined(KOKKOS_ENABLE_OPENMPTARGET) @@ -184,7 +183,7 @@ using SharedHostPinnedSpace = CudaHostPinnedSpace; using SharedHostPinnedSpace = HIPHostPinnedSpace; #define KOKKOS_HAS_SHARED_HOST_PINNED_SPACE #elif defined(KOKKOS_ENABLE_SYCL) - using SharedHostPinnedSpace = Experimental::SYCLHostUSMSpace; + using SharedHostPinnedSpace = SYCLHostUSMSpace; #define KOKKOS_HAS_SHARED_HOST_PINNED_SPACE #elif !defined(KOKKOS_ENABLE_OPENACC) && !defined(KOKKOS_ENABLE_OPENMPTARGET) using SharedHostPinnedSpace = HostSpace; diff --git a/lib/kokkos/core/src/Kokkos_Crs.hpp b/lib/kokkos/core/src/Kokkos_Crs.hpp index 92931b5849..69223b6412 100644 --- a/lib/kokkos/core/src/Kokkos_Crs.hpp +++ b/lib/kokkos/core/src/Kokkos_Crs.hpp @@ -84,12 +84,12 @@ class Crs { /* * Default Constructors, operators and destructor */ - KOKKOS_DEFAULTED_FUNCTION Crs() = default; - KOKKOS_DEFAULTED_FUNCTION Crs(Crs const&) = default; - KOKKOS_DEFAULTED_FUNCTION Crs(Crs&&) = default; + KOKKOS_DEFAULTED_FUNCTION Crs() = default; + KOKKOS_DEFAULTED_FUNCTION Crs(Crs const&) = default; + KOKKOS_DEFAULTED_FUNCTION Crs(Crs&&) = default; KOKKOS_DEFAULTED_FUNCTION Crs& operator=(Crs const&) = default; - KOKKOS_DEFAULTED_FUNCTION Crs& operator=(Crs&&) = default; - KOKKOS_DEFAULTED_FUNCTION ~Crs() = default; + KOKKOS_DEFAULTED_FUNCTION Crs& operator=(Crs&&) = default; + KOKKOS_DEFAULTED_FUNCTION ~Crs() = default; /** \brief Assign to a view of the rhs array. * If the old view is the last view @@ -148,7 +148,7 @@ class GetCrsTransposeCounts { public: KOKKOS_INLINE_FUNCTION - void operator()(index_type i) const { atomic_increment(&out[in.entries(i)]); } + void operator()(index_type i) const { atomic_inc(&out[in.entries(i)]); } GetCrsTransposeCounts(InCrs const& arg_in, OutCounts const& arg_out) : in(arg_in), out(arg_out) { using policy_type = RangePolicy; @@ -345,7 +345,7 @@ struct CountAndFill : public CountAndFillBase { closure.execute(); } auto nentries = Kokkos::get_crs_row_map_from_counts(this->m_crs.row_map, - this->m_counts); + this->m_counts); this->m_counts = counts_type(); this->m_crs.entries = entries_type("entries", nentries); { diff --git a/lib/kokkos/core/src/Kokkos_DetectionIdiom.hpp b/lib/kokkos/core/src/Kokkos_DetectionIdiom.hpp index ae28805a42..8af10b2a40 100644 --- a/lib/kokkos/core/src/Kokkos_DetectionIdiom.hpp +++ b/lib/kokkos/core/src/Kokkos_DetectionIdiom.hpp @@ -54,8 +54,8 @@ struct detector>, Op, Args...> { } // namespace Impl struct nonesuch : private Impl::nonesuch_base { - ~nonesuch() = delete; - nonesuch(nonesuch const&) = delete; + ~nonesuch() = delete; + nonesuch(nonesuch const&) = delete; void operator=(nonesuch const&) = delete; }; @@ -81,7 +81,7 @@ inline constexpr bool is_detected_v = is_detected::value; template class Op, class... Args> inline constexpr bool is_detected_exact_v = - is_detected_exact::value; + is_detected_exact::value; // NOLINT template class Op, class... Args> inline constexpr bool is_detected_convertible_v = diff --git a/lib/kokkos/core/src/Kokkos_ExecPolicy.hpp b/lib/kokkos/core/src/Kokkos_ExecPolicy.hpp index b8d7f77deb..dd7ce5ce21 100644 --- a/lib/kokkos/core/src/Kokkos_ExecPolicy.hpp +++ b/lib/kokkos/core/src/Kokkos_ExecPolicy.hpp @@ -27,7 +27,10 @@ static_assert(false, #include #include #include +#include +#ifndef KOKKOS_ENABLE_IMPL_TYPEINFO #include +#endif #include //---------------------------------------------------------------------------- @@ -197,8 +200,7 @@ class RangePolicy : public Impl::PolicyTraits { /** \brief finalize chunk_size if it was set to AUTO*/ inline void set_auto_chunk_size() { #ifdef KOKKOS_ENABLE_SYCL - if (std::is_same_v) { + if (std::is_same_v) { // chunk_size <=1 lets the compiler choose the workgroup size when // launching kernels m_granularity = 1; @@ -248,46 +250,49 @@ class RangePolicy : public Impl::PolicyTraits { // To be replaced with std::in_range (c++20) template - static void check_conversion_safety(const IndexType bound) { + static void check_conversion_safety([[maybe_unused]] const IndexType bound) { + // Checking that the round-trip conversion preserves input index value + if constexpr (std::is_convertible_v) { #if !defined(KOKKOS_ENABLE_DEPRECATED_CODE_4) || \ defined(KOKKOS_ENABLE_DEPRECATION_WARNINGS) - std::string msg = - "Kokkos::RangePolicy bound type error: an unsafe implicit conversion " - "is performed on a bound (" + - std::to_string(bound) + - "), which may " - "not preserve its original value.\n"; - bool warn = false; + std::string msg = + "Kokkos::RangePolicy bound type error: an unsafe implicit conversion " + "is performed on a bound (" + + std::to_string(bound) + + "), which may " + "not preserve its original value.\n"; + bool warn = false; - if constexpr (std::is_signed_v != - std::is_signed_v) { - // check signed to unsigned - if constexpr (std::is_signed_v) - warn |= (bound < static_cast( - std::numeric_limits::min())); + if constexpr (std::is_arithmetic_v && + (std::is_signed_v != + std::is_signed_v)) { + // check signed to unsigned + if constexpr (std::is_signed_v) + warn |= (bound < static_cast( + std::numeric_limits::min())); - // check unsigned to signed - if constexpr (std::is_signed_v) - warn |= (bound > static_cast( - std::numeric_limits::max())); - } + // check unsigned to signed + if constexpr (std::is_signed_v) + warn |= (bound > static_cast( + std::numeric_limits::max())); + } - // check narrowing - warn |= (static_cast(static_cast(bound)) != bound); + // check narrowing + warn |= + (static_cast(static_cast(bound)) != bound); - if (warn) { + if (warn) { #ifndef KOKKOS_ENABLE_DEPRECATED_CODE_4 - Kokkos::abort(msg.c_str()); + Kokkos::abort(msg.c_str()); #endif #ifdef KOKKOS_ENABLE_DEPRECATION_WARNINGS - Kokkos::Impl::log_warning(msg); + Kokkos::Impl::log_warning(msg); +#endif + } #endif } -#else - (void)bound; -#endif } public: @@ -333,20 +338,20 @@ class RangePolicy : public Impl::PolicyTraits { }; }; -RangePolicy()->RangePolicy<>; +RangePolicy() -> RangePolicy<>; -RangePolicy(int64_t, int64_t)->RangePolicy<>; -RangePolicy(int64_t, int64_t, ChunkSize const&)->RangePolicy<>; +RangePolicy(int64_t, int64_t) -> RangePolicy<>; +RangePolicy(int64_t, int64_t, ChunkSize const&) -> RangePolicy<>; -RangePolicy(DefaultExecutionSpace const&, int64_t, int64_t)->RangePolicy<>; +RangePolicy(DefaultExecutionSpace const&, int64_t, int64_t) -> RangePolicy<>; RangePolicy(DefaultExecutionSpace const&, int64_t, int64_t, ChunkSize const&) - ->RangePolicy<>; + -> RangePolicy<>; template >> -RangePolicy(ES const&, int64_t, int64_t)->RangePolicy; +RangePolicy(ES const&, int64_t, int64_t) -> RangePolicy; template >> -RangePolicy(ES const&, int64_t, int64_t, ChunkSize const&)->RangePolicy; +RangePolicy(ES const&, int64_t, int64_t, ChunkSize const&) -> RangePolicy; } // namespace Kokkos @@ -515,24 +520,24 @@ struct PerThreadValue { template struct ExtractVectorLength { static inline iType value( - std::enable_if_t::value, iType> val, Args...) { + std::enable_if_t, iType> val, Args...) { return val; } - static inline std::enable_if_t::value, int> value( - std::enable_if_t::value, iType>, Args...) { + static inline std::enable_if_t, int> value( + std::enable_if_t, iType>, Args...) { return 1; } }; template -inline std::enable_if_t::value, iType> -extract_vector_length(iType val, Args...) { +inline std::enable_if_t, iType> extract_vector_length( + iType val, Args...) { return val; } template -inline std::enable_if_t::value, int> -extract_vector_length(iType, Args...) { +inline std::enable_if_t, int> extract_vector_length( + iType, Args...) { return 1; } @@ -577,7 +582,7 @@ struct ScratchRequest { } }; -// Throws a runtime exception if level is not `0` or `1` +// Causes abnormal program termination if level is not `0` or `1` void team_policy_check_valid_storage_level_argument(int level); /** \brief Execution policy for parallel work over a league of teams of @@ -721,55 +726,54 @@ class TeamPolicy // Execution space not provided deduces to TeamPolicy<> -TeamPolicy()->TeamPolicy<>; +TeamPolicy() -> TeamPolicy<>; -TeamPolicy(int, int)->TeamPolicy<>; -TeamPolicy(int, int, int)->TeamPolicy<>; -TeamPolicy(int, Kokkos::AUTO_t const&)->TeamPolicy<>; -TeamPolicy(int, Kokkos::AUTO_t const&, int)->TeamPolicy<>; -TeamPolicy(int, Kokkos::AUTO_t const&, Kokkos::AUTO_t const&)->TeamPolicy<>; -TeamPolicy(int, int, Kokkos::AUTO_t const&)->TeamPolicy<>; +TeamPolicy(int, int) -> TeamPolicy<>; +TeamPolicy(int, int, int) -> TeamPolicy<>; +TeamPolicy(int, Kokkos::AUTO_t const&) -> TeamPolicy<>; +TeamPolicy(int, Kokkos::AUTO_t const&, int) -> TeamPolicy<>; +TeamPolicy(int, Kokkos::AUTO_t const&, Kokkos::AUTO_t const&) -> TeamPolicy<>; +TeamPolicy(int, int, Kokkos::AUTO_t const&) -> TeamPolicy<>; // DefaultExecutionSpace deduces to TeamPolicy<> -TeamPolicy(DefaultExecutionSpace const&, int, int)->TeamPolicy<>; -TeamPolicy(DefaultExecutionSpace const&, int, int, int)->TeamPolicy<>; +TeamPolicy(DefaultExecutionSpace const&, int, int) -> TeamPolicy<>; +TeamPolicy(DefaultExecutionSpace const&, int, int, int) -> TeamPolicy<>; TeamPolicy(DefaultExecutionSpace const&, int, Kokkos::AUTO_t const&) - ->TeamPolicy<>; + -> TeamPolicy<>; TeamPolicy(DefaultExecutionSpace const&, int, Kokkos::AUTO_t const&, int) - ->TeamPolicy<>; + -> TeamPolicy<>; TeamPolicy(DefaultExecutionSpace const&, int, Kokkos::AUTO_t const&, - Kokkos::AUTO_t const&) - ->TeamPolicy<>; + Kokkos::AUTO_t const&) -> TeamPolicy<>; TeamPolicy(DefaultExecutionSpace const&, int, int, Kokkos::AUTO_t const&) - ->TeamPolicy<>; + -> TeamPolicy<>; // ES != DefaultExecutionSpace deduces to TeamPolicy template >> -TeamPolicy(ES const&, int, int)->TeamPolicy; +TeamPolicy(ES const&, int, int) -> TeamPolicy; template >> -TeamPolicy(ES const&, int, int, int)->TeamPolicy; +TeamPolicy(ES const&, int, int, int) -> TeamPolicy; template >> -TeamPolicy(ES const&, int, Kokkos::AUTO_t const&)->TeamPolicy; +TeamPolicy(ES const&, int, Kokkos::AUTO_t const&) -> TeamPolicy; template >> -TeamPolicy(ES const&, int, Kokkos::AUTO_t const&, int)->TeamPolicy; +TeamPolicy(ES const&, int, Kokkos::AUTO_t const&, int) -> TeamPolicy; template >> TeamPolicy(ES const&, int, Kokkos::AUTO_t const&, Kokkos::AUTO_t const&) - ->TeamPolicy; + -> TeamPolicy; template >> -TeamPolicy(ES const&, int, int, Kokkos::AUTO_t const&)->TeamPolicy; +TeamPolicy(ES const&, int, int, Kokkos::AUTO_t const&) -> TeamPolicy; namespace Impl { @@ -1041,7 +1045,7 @@ struct TeamThreadMDRange, TeamHandle> { template KOKKOS_DEDUCTION_GUIDE TeamThreadMDRange(TeamHandle const&, Args&&...) - ->TeamThreadMDRange, TeamHandle>; + -> TeamThreadMDRange, TeamHandle>; template struct ThreadVectorMDRange; @@ -1078,7 +1082,7 @@ struct ThreadVectorMDRange, TeamHandle> { template KOKKOS_DEDUCTION_GUIDE ThreadVectorMDRange(TeamHandle const&, Args&&...) - ->ThreadVectorMDRange, TeamHandle>; + -> ThreadVectorMDRange, TeamHandle>; template struct TeamVectorMDRange; @@ -1115,7 +1119,7 @@ struct TeamVectorMDRange, TeamHandle> { template KOKKOS_DEDUCTION_GUIDE TeamVectorMDRange(TeamHandle const&, Args&&...) - ->TeamVectorMDRange, TeamHandle>; + -> TeamVectorMDRange, TeamHandle>; template @@ -1162,7 +1166,7 @@ KOKKOS_INLINE_FUNCTION void parallel_reduce( Kokkos::HIP> #elif defined(KOKKOS_ENABLE_SYCL) || std::is_same_v + Kokkos::SYCL> #endif ) policy.team.vector_reduce( @@ -1198,7 +1202,7 @@ KOKKOS_INLINE_FUNCTION void parallel_reduce( Kokkos::HIP> #elif defined(KOKKOS_ENABLE_SYCL) || std::is_same_v + Kokkos::SYCL> #endif ) policy.team.vector_reduce( @@ -1217,15 +1221,21 @@ KOKKOS_INLINE_FUNCTION void parallel_for( namespace Impl { template ::value> + bool HasTag = !std::is_void_v> struct ParallelConstructName; template struct ParallelConstructName { ParallelConstructName(std::string const& label) : label_ref(label) { if (label.empty()) { +#ifdef KOKKOS_ENABLE_IMPL_TYPEINFO + default_name = + std::string(TypeInfo>::name()) + + "/" + std::string(TypeInfo::name()); +#else default_name = std::string(typeid(FunctorType).name()) + "/" + typeid(TagType).name(); +#endif } } std::string const& get() { @@ -1239,7 +1249,11 @@ template struct ParallelConstructName { ParallelConstructName(std::string const& label) : label_ref(label) { if (label.empty()) { - default_name = std::string(typeid(FunctorType).name()); +#ifdef KOKKOS_ENABLE_IMPL_TYPEINFO + default_name = TypeInfo>::name(); +#else + default_name = typeid(FunctorType).name(); +#endif } } std::string const& get() { diff --git a/lib/kokkos/core/src/Kokkos_Extents.hpp b/lib/kokkos/core/src/Kokkos_Extents.hpp index 9bc2eda604..7d1f8c755d 100644 --- a/lib/kokkos/core/src/Kokkos_Extents.hpp +++ b/lib/kokkos/core/src/Kokkos_Extents.hpp @@ -134,7 +134,7 @@ struct ApplyExtent { template struct ApplyExtent { - using type = ValueType * [Ext]; + using type = ValueType* [Ext]; }; template diff --git a/lib/kokkos/core/src/Kokkos_Future.hpp b/lib/kokkos/core/src/Kokkos_Future.hpp index 0b3a153de8..c26d08be1c 100644 --- a/lib/kokkos/core/src/Kokkos_Future.hpp +++ b/lib/kokkos/core/src/Kokkos_Future.hpp @@ -14,11 +14,17 @@ // //@HEADER -#ifndef KOKKOS_IMPL_PUBLIC_INCLUDE #include + +#ifndef KOKKOS_IMPL_PUBLIC_INCLUDE static_assert(false, "Including non-public Kokkos header files is not allowed."); #endif + +#ifndef KOKKOS_ENABLE_DEPRECATED_CODE_4 +#error "The tasking framework is deprecated" +#endif + #ifndef KOKKOS_FUTURE_HPP #define KOKKOS_FUTURE_HPP @@ -41,13 +47,19 @@ static_assert(false, //---------------------------------------------------------------------------- //---------------------------------------------------------------------------- +#ifdef KOKKOS_ENABLE_DEPRECATION_WARNINGS +// We allow using deprecated classes in this file +KOKKOS_IMPL_DISABLE_DEPRECATED_WARNINGS_PUSH() +#endif + namespace Kokkos { // For now, hack this in as a partial specialization // TODO @tasking @cleanup Make this the "normal" class template and make the old // code the specialization template -class BasicFuture> { +class KOKKOS_DEPRECATED + BasicFuture> { public: using value_type = ValueType; using execution_space = ExecutionSpace; @@ -244,7 +256,7 @@ class BasicFuture> { //////////////////////////////////////////////////////////////////////////////// template -class BasicFuture { +class KOKKOS_DEPRECATED BasicFuture { private: template friend class BasicTaskScheduler; @@ -413,13 +425,13 @@ class BasicFuture { // Is a Future with the given execution space template -struct is_future : public std::false_type {}; +struct KOKKOS_DEPRECATED is_future : public std::false_type {}; template -struct is_future, ExecSpace> +struct KOKKOS_DEPRECATED is_future, ExecSpace> : std::bool_constant< - std::is_same::value || - std::is_void::value> {}; + std::is_same_v || + std::is_void_v> {}; //////////////////////////////////////////////////////////////////////////////// // END OLD CODE @@ -432,8 +444,8 @@ class ResolveFutureArgOrder { private: enum { Arg1_is_space = Kokkos::is_space::value }; enum { Arg2_is_space = Kokkos::is_space::value }; - enum { Arg1_is_value = !Arg1_is_space && !std::is_void::value }; - enum { Arg2_is_value = !Arg2_is_space && !std::is_void::value }; + enum { Arg1_is_value = !Arg1_is_space && !std::is_void_v }; + enum { Arg2_is_value = !Arg2_is_space && !std::is_void_v }; static_assert(!(Arg1_is_space && Arg2_is_space), "Future cannot be given two spaces"); @@ -463,10 +475,15 @@ class ResolveFutureArgOrder { * */ template -using Future = typename Impl::ResolveFutureArgOrder::type; +using Future KOKKOS_DEPRECATED = + typename Impl::ResolveFutureArgOrder::type; } // namespace Kokkos +#ifdef KOKKOS_ENABLE_DEPRECATION_WARNINGS +KOKKOS_IMPL_DISABLE_DEPRECATED_WARNINGS_POP() +#endif + //---------------------------------------------------------------------------- //---------------------------------------------------------------------------- diff --git a/lib/kokkos/core/src/Kokkos_Graph.hpp b/lib/kokkos/core/src/Kokkos_Graph.hpp index 9cc6650e26..05d774ac61 100644 --- a/lib/kokkos/core/src/Kokkos_Graph.hpp +++ b/lib/kokkos/core/src/Kokkos_Graph.hpp @@ -86,10 +86,21 @@ struct [[nodiscard]] Graph { return m_impl_ptr->get_execution_space(); } - void submit() const { + void instantiate() { KOKKOS_EXPECTS(bool(m_impl_ptr)) - (*m_impl_ptr).submit(); + (*m_impl_ptr).instantiate(); } + + void submit(const execution_space& exec) const { + KOKKOS_EXPECTS(bool(m_impl_ptr)) + (*m_impl_ptr).submit(exec); + } + + void submit() const { submit(get_execution_space()); } + + decltype(auto) native_graph(); + + decltype(auto) native_graph_exec(); }; // end Graph }}}1 @@ -135,22 +146,68 @@ Graph create_graph(ExecutionSpace ex, Closure&& arg_closure) { // function template injection works. auto rv = Kokkos::Impl::GraphAccess::construct_graph(std::move(ex)); // Invoke the user's graph construction closure - ((Closure &&) arg_closure)(Kokkos::Impl::GraphAccess::create_root_ref(rv)); + ((Closure&&)arg_closure)(Kokkos::Impl::GraphAccess::create_root_ref(rv)); // and given them back the graph // KOKKOS_ENSURES(rv.m_impl_ptr.use_count() == 1) return rv; } +template +std::enable_if_t, + Graph> +create_graph(ExecutionSpace exec = ExecutionSpace{}) { + return Kokkos::Impl::GraphAccess::construct_graph(std::move(exec)); +} + template < class ExecutionSpace = DefaultExecutionSpace, class Closure = Kokkos::Impl::DoNotExplicitlySpecifyThisTemplateParameter> -Graph create_graph(Closure&& arg_closure) { - return create_graph(ExecutionSpace{}, (Closure &&) arg_closure); +std::enable_if_t< + !Kokkos::is_execution_space_v>, + Graph> +create_graph(Closure&& arg_closure) { + return create_graph(ExecutionSpace{}, (Closure&&)arg_closure); } // end create_graph }}}1 //============================================================================== +template +decltype(auto) Graph::native_graph() { + KOKKOS_EXPECTS(bool(m_impl_ptr)); +#if defined(KOKKOS_ENABLE_CUDA) + if constexpr (std::is_same_v) { + return m_impl_ptr->cuda_graph(); + } +#elif defined(KOKKOS_ENABLE_HIP) && defined(KOKKOS_IMPL_HIP_NATIVE_GRAPH) + if constexpr (std::is_same_v) { + return m_impl_ptr->hip_graph(); + } +#elif defined(KOKKOS_ENABLE_SYCL) && defined(SYCL_EXT_ONEAPI_GRAPH) + if constexpr (std::is_same_v) { + return m_impl_ptr->sycl_graph(); + } +#endif +} + +template +decltype(auto) Graph::native_graph_exec() { + KOKKOS_EXPECTS(bool(m_impl_ptr)); +#if defined(KOKKOS_ENABLE_CUDA) + if constexpr (std::is_same_v) { + return m_impl_ptr->cuda_graph_exec(); + } +#elif defined(KOKKOS_ENABLE_HIP) && defined(KOKKOS_IMPL_HIP_NATIVE_GRAPH) + if constexpr (std::is_same_v) { + return m_impl_ptr->hip_graph_exec(); + } +#elif defined(KOKKOS_ENABLE_SYCL) && defined(SYCL_EXT_ONEAPI_GRAPH) + if constexpr (std::is_same_v) { + return m_impl_ptr->sycl_graph_exec(); + } +#endif +} + } // end namespace Experimental } // namespace Kokkos @@ -163,7 +220,7 @@ Graph create_graph(Closure&& arg_closure) { #include #if defined(KOKKOS_ENABLE_HIP) // The implementation of hipGraph in ROCm 5.2 is bugged, so we cannot use it. -#if !((HIP_VERSION_MAJOR == 5) && (HIP_VERSION_MINOR == 2)) +#if defined(KOKKOS_IMPL_HIP_NATIVE_GRAPH) #include #endif #endif diff --git a/lib/kokkos/core/src/Kokkos_GraphNode.hpp b/lib/kokkos/core/src/Kokkos_GraphNode.hpp index 2a4e2cf641..a0a60c07d0 100644 --- a/lib/kokkos/core/src/Kokkos_GraphNode.hpp +++ b/lib/kokkos/core/src/Kokkos_GraphNode.hpp @@ -48,7 +48,7 @@ class GraphNodeRef { // intended to be SFINAE-safe, so do validation before you instantiate. static_assert( - std::is_same::value || + std::is_same_v || Kokkos::Impl::is_specialization_of::value, "Invalid predecessor template parameter given to GraphNodeRef"); @@ -56,7 +56,7 @@ class GraphNodeRef { Kokkos::is_execution_space::value, "Invalid execution space template parameter given to GraphNodeRef"); - static_assert(std::is_same::value || + static_assert(std::is_same_v || Kokkos::Impl::is_graph_kernel::value, "Invalid kernel template parameter given to GraphNodeRef"); @@ -151,7 +151,7 @@ class GraphNodeRef { typename return_t::node_impl_t>( m_node_impl->execution_space_instance(), Kokkos::Impl::_graph_node_kernel_ctor_tag{}, - (NextKernelDeduced &&) arg_kernel, + (NextKernelDeduced&&)arg_kernel, // *this is the predecessor Kokkos::Impl::_graph_node_predecessor_ctor_tag{}, *this)); @@ -184,10 +184,10 @@ class GraphNodeRef { // {{{3 // Copyable and movable (basically just shared_ptr semantics - GraphNodeRef() noexcept = default; - GraphNodeRef(GraphNodeRef const&) = default; - GraphNodeRef(GraphNodeRef&&) noexcept = default; - GraphNodeRef& operator=(GraphNodeRef const&) = default; + GraphNodeRef() noexcept = default; + GraphNodeRef(GraphNodeRef const&) = default; + GraphNodeRef(GraphNodeRef&&) noexcept = default; + GraphNodeRef& operator=(GraphNodeRef const&) = default; GraphNodeRef& operator=(GraphNodeRef&&) noexcept = default; ~GraphNodeRef() = default; @@ -197,19 +197,19 @@ class GraphNodeRef { //- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - // {{{3 - template < - class OtherKernel, class OtherPredecessor, - std::enable_if_t< - // Not a copy/move constructor - !std::is_same>::value && - // must be an allowed type erasure of the kernel - Kokkos::Impl::is_compatible_type_erasure::value && - // must be an allowed type erasure of the predecessor - Kokkos::Impl::is_compatible_type_erasure< - OtherPredecessor, graph_predecessor>::value, - int> = 0> + template > && + // must be an allowed type erasure of the kernel + Kokkos::Impl::is_compatible_type_erasure< + OtherKernel, graph_kernel>::value && + // must be an allowed type erasure of the predecessor + Kokkos::Impl::is_compatible_type_erasure< + OtherPredecessor, graph_predecessor>::value, + int> = 0> /* implicit */ GraphNodeRef( GraphNodeRef const& other) @@ -257,7 +257,7 @@ class GraphNodeRef { //|| policy_t::execution_space_is_defaulted, "Execution Space mismatch between execution policy and graph"); - auto policy = Experimental::require((Policy &&) arg_policy, + auto policy = Experimental::require((Policy&&)arg_policy, Kokkos::Impl::KernelInGraphProperty{}); using next_policy_t = decltype(policy); @@ -266,8 +266,8 @@ class GraphNodeRef { std::decay_t, Kokkos::ParallelForTag>; return this->_then_kernel(next_kernel_t{std::move(arg_name), policy.space(), - (Functor &&) functor, - (Policy &&) policy}); + (Functor&&)functor, + (Policy&&)policy}); } template < @@ -280,8 +280,7 @@ class GraphNodeRef { int> = 0> auto then_parallel_for(Policy&& policy, Functor&& functor) const { // needs to static assert constraint: DataParallelFunctor - return this->then_parallel_for("", (Policy &&) policy, - (Functor &&) functor); + return this->then_parallel_for("", (Policy&&)policy, (Functor&&)functor); } template @@ -290,13 +289,13 @@ class GraphNodeRef { // needs to static assert constraint: DataParallelFunctor return this->then_parallel_for(std::move(name), Kokkos::RangePolicy(0, n), - (Functor &&) functor); + (Functor&&)functor); } template auto then_parallel_for(std::size_t n, Functor&& functor) const { // needs to static assert constraint: DataParallelFunctor - return this->then_parallel_for("", n, (Functor &&) functor); + return this->then_parallel_for("", n, (Functor&&)functor); } // end then_parallel_for }}}2 @@ -359,6 +358,23 @@ class GraphNodeRef { Kokkos::is_reducer::value, "Output argument to parallel reduce in a graph must be a " "View or a Reducer"); + + if constexpr (Kokkos::is_reducer_v) { + static_assert( + Kokkos::SpaceAccessibility< + ExecutionSpace, typename return_type_remove_cvref:: + result_view_type::memory_space>::accessible, + "The reduction target must be accessible by the graph execution " + "space."); + } else { + static_assert( + Kokkos::SpaceAccessibility< + ExecutionSpace, + typename return_type_remove_cvref::memory_space>::accessible, + "The reduction target must be accessible by the graph execution " + "space."); + } + using return_type = // Yes, you do really have to do this... std::conditional_t::value, @@ -373,7 +389,7 @@ class GraphNodeRef { // End of Kokkos reducer disaster //---------------------------------------- - auto policy = Experimental::require((Policy &&) arg_policy, + auto policy = Experimental::require((Policy&&)arg_policy, Kokkos::Impl::KernelInGraphProperty{}); using passed_reducer_type = typename return_value_adapter::reducer_type; @@ -399,7 +415,7 @@ class GraphNodeRef { return this->_then_kernel(next_kernel_t{ std::move(arg_name), graph_impl_ptr->get_execution_space(), - functor_reducer, (Policy &&) policy, + functor_reducer, (Policy&&)policy, return_value_adapter::return_value(return_value, functor)}); } @@ -413,9 +429,9 @@ class GraphNodeRef { int> = 0> auto then_parallel_reduce(Policy&& arg_policy, Functor&& functor, ReturnType&& return_value) const { - return this->then_parallel_reduce("", (Policy &&) arg_policy, - (Functor &&) functor, - (ReturnType &&) return_value); + return this->then_parallel_reduce("", (Policy&&)arg_policy, + (Functor&&)functor, + (ReturnType&&)return_value); } template @@ -425,15 +441,15 @@ class GraphNodeRef { ReturnType&& return_value) const { return this->then_parallel_reduce( std::move(label), Kokkos::RangePolicy{0, idx_end}, - (Functor &&) functor, (ReturnType &&) return_value); + (Functor&&)functor, (ReturnType&&)return_value); } template auto then_parallel_reduce(typename execution_space::size_type idx_end, Functor&& functor, ReturnType&& return_value) const { - return this->then_parallel_reduce("", idx_end, (Functor &&) functor, - (ReturnType &&) return_value); + return this->then_parallel_reduce("", idx_end, (Functor&&)functor, + (ReturnType&&)return_value); } // end then_parallel_reduce }}}2 diff --git a/lib/kokkos/core/src/Kokkos_HostSpace.hpp b/lib/kokkos/core/src/Kokkos_HostSpace.hpp index 8b5f29f95b..706586826f 100644 --- a/lib/kokkos/core/src/Kokkos_HostSpace.hpp +++ b/lib/kokkos/core/src/Kokkos_HostSpace.hpp @@ -63,10 +63,10 @@ class HostSpace { //! This memory space preferred device_type using device_type = Kokkos::Device; - HostSpace() = default; - HostSpace(HostSpace&& rhs) = default; - HostSpace(const HostSpace& rhs) = default; - HostSpace& operator=(HostSpace&&) = default; + HostSpace() = default; + HostSpace(HostSpace&& rhs) = default; + HostSpace(const HostSpace& rhs) = default; + HostSpace& operator=(HostSpace&&) = default; HostSpace& operator=(const HostSpace&) = default; ~HostSpace() = default; @@ -183,18 +183,6 @@ namespace Kokkos { namespace Impl { -template <> -struct DeepCopy { - DeepCopy(void* dst, const void* src, size_t n) { - hostspace_parallel_deepcopy(dst, src, n); - } - - DeepCopy(const DefaultHostExecutionSpace& exec, void* dst, const void* src, - size_t n) { - hostspace_parallel_deepcopy_async(exec, dst, src, n); - } -}; - template struct DeepCopy { DeepCopy(void* dst, const void* src, size_t n) { @@ -202,10 +190,15 @@ struct DeepCopy { } DeepCopy(const ExecutionSpace& exec, void* dst, const void* src, size_t n) { - exec.fence( - "Kokkos::Impl::DeepCopy::DeepCopy: fence before copy"); - hostspace_parallel_deepcopy_async(dst, src, n); + if constexpr (!Kokkos::SpaceAccessibility::accessible) { + exec.fence( + "Kokkos::Impl::DeepCopy::DeepCopy: fence before copy"); + hostspace_parallel_deepcopy_async(dst, src, n); + } else { + hostspace_parallel_deepcopy_async(exec, dst, src, n); + } } }; diff --git a/lib/kokkos/core/src/Kokkos_Layout.hpp b/lib/kokkos/core/src/Kokkos_Layout.hpp index 37b80e54a8..a760e7054a 100644 --- a/lib/kokkos/core/src/Kokkos_Layout.hpp +++ b/lib/kokkos/core/src/Kokkos_Layout.hpp @@ -52,13 +52,17 @@ struct LayoutLeft { using array_layout = LayoutLeft; size_t dimension[ARRAY_LAYOUT_MAX_RANK]; + // we don't have a constructor to set the stride directly + // but we will deprecate the class anyway (or at least using an instance of + // this class) when switching the internal implementation to use mdspan + size_t stride; enum : bool { is_extent_constructible = true }; - LayoutLeft(LayoutLeft const&) = default; - LayoutLeft(LayoutLeft&&) = default; + LayoutLeft(LayoutLeft const&) = default; + LayoutLeft(LayoutLeft&&) = default; LayoutLeft& operator=(LayoutLeft const&) = default; - LayoutLeft& operator=(LayoutLeft&&) = default; + LayoutLeft& operator=(LayoutLeft&&) = default; KOKKOS_INLINE_FUNCTION explicit constexpr LayoutLeft(size_t N0 = KOKKOS_IMPL_CTOR_DEFAULT_ARG, @@ -69,7 +73,8 @@ struct LayoutLeft { size_t N5 = KOKKOS_IMPL_CTOR_DEFAULT_ARG, size_t N6 = KOKKOS_IMPL_CTOR_DEFAULT_ARG, size_t N7 = KOKKOS_IMPL_CTOR_DEFAULT_ARG) - : dimension{N0, N1, N2, N3, N4, N5, N6, N7} {} + : dimension{N0, N1, N2, N3, N4, N5, N6, N7}, + stride(KOKKOS_IMPL_CTOR_DEFAULT_ARG) {} friend bool operator==(const LayoutLeft& left, const LayoutLeft& right) { for (unsigned int rank = 0; rank < ARRAY_LAYOUT_MAX_RANK; ++rank) @@ -101,13 +106,17 @@ struct LayoutRight { using array_layout = LayoutRight; size_t dimension[ARRAY_LAYOUT_MAX_RANK]; + // we don't have a constructor to set the stride directly + // but we will deprecate the class anyway (or at least using an instance of + // this class) when switching the internal implementation to use mdspan + size_t stride; enum : bool { is_extent_constructible = true }; - LayoutRight(LayoutRight const&) = default; - LayoutRight(LayoutRight&&) = default; + LayoutRight(LayoutRight const&) = default; + LayoutRight(LayoutRight&&) = default; LayoutRight& operator=(LayoutRight const&) = default; - LayoutRight& operator=(LayoutRight&&) = default; + LayoutRight& operator=(LayoutRight&&) = default; KOKKOS_INLINE_FUNCTION explicit constexpr LayoutRight(size_t N0 = KOKKOS_IMPL_CTOR_DEFAULT_ARG, @@ -118,7 +127,8 @@ struct LayoutRight { size_t N5 = KOKKOS_IMPL_CTOR_DEFAULT_ARG, size_t N6 = KOKKOS_IMPL_CTOR_DEFAULT_ARG, size_t N7 = KOKKOS_IMPL_CTOR_DEFAULT_ARG) - : dimension{N0, N1, N2, N3, N4, N5, N6, N7} {} + : dimension{N0, N1, N2, N3, N4, N5, N6, N7}, + stride{KOKKOS_IMPL_CTOR_DEFAULT_ARG} {} friend bool operator==(const LayoutRight& left, const LayoutRight& right) { for (unsigned int rank = 0; rank < ARRAY_LAYOUT_MAX_RANK; ++rank) @@ -144,10 +154,10 @@ struct LayoutStride { enum : bool { is_extent_constructible = false }; - LayoutStride(LayoutStride const&) = default; - LayoutStride(LayoutStride&&) = default; + LayoutStride(LayoutStride const&) = default; + LayoutStride(LayoutStride&&) = default; LayoutStride& operator=(LayoutStride const&) = default; - LayoutStride& operator=(LayoutStride&&) = default; + LayoutStride& operator=(LayoutStride&&) = default; /** \brief Compute strides from ordered dimensions. * @@ -191,8 +201,8 @@ struct LayoutStride { size_t N5 = KOKKOS_IMPL_CTOR_DEFAULT_ARG, size_t S5 = 0, size_t N6 = KOKKOS_IMPL_CTOR_DEFAULT_ARG, size_t S6 = 0, size_t N7 = KOKKOS_IMPL_CTOR_DEFAULT_ARG, size_t S7 = 0) - : dimension{N0, N1, N2, N3, N4, N5, N6, N7}, stride{S0, S1, S2, S3, - S4, S5, S6, S7} {} + : dimension{N0, N1, N2, N3, N4, N5, N6, N7}, + stride{S0, S1, S2, S3, S4, S5, S6, S7} {} friend bool operator==(const LayoutStride& left, const LayoutStride& right) { for (unsigned int rank = 0; rank < ARRAY_LAYOUT_MAX_RANK; ++rank) diff --git a/lib/kokkos/core/src/Kokkos_Macros.hpp b/lib/kokkos/core/src/Kokkos_Macros.hpp index 0a0acd303f..97b78a3c64 100644 --- a/lib/kokkos/core/src/Kokkos_Macros.hpp +++ b/lib/kokkos/core/src/Kokkos_Macros.hpp @@ -27,7 +27,7 @@ * KOKKOS_ENABLE_OPENMPTARGET Kokkos::Experimental::OpenMPTarget * execution space * KOKKOS_ENABLE_HIP Kokkos::HIP execution space - * KOKKOS_ENABLE_SYCL Kokkos::Experimental::SYCL execution space + * KOKKOS_ENABLE_SYCL Kokkos::SYCL execution space * KOKKOS_ENABLE_HWLOC HWLOC library is available. * KOKKOS_ENABLE_DEBUG_BOUNDS_CHECK Insert array bounds checks, is expensive! * KOKKOS_ENABLE_CUDA_UVM Use CUDA UVM for Cuda memory space. @@ -132,7 +132,7 @@ #define KOKKOS_CLASS_LAMBDA [ =, *this ] #endif -//#if !defined( __CUDA_ARCH__ ) // Not compiling Cuda code to 'ptx'. +// #if !defined( __CUDA_ARCH__ ) // Not compiling Cuda code to 'ptx'. // Intel compiler for host code. @@ -252,10 +252,10 @@ // CLANG compiler macros #if defined(KOKKOS_COMPILER_CLANG) -//#define KOKKOS_ENABLE_PRAGMA_UNROLL 1 -//#define KOKKOS_ENABLE_PRAGMA_IVDEP 1 -//#define KOKKOS_ENABLE_PRAGMA_LOOPCOUNT 1 -//#define KOKKOS_ENABLE_PRAGMA_VECTOR 1 +// #define KOKKOS_ENABLE_PRAGMA_UNROLL 1 +// #define KOKKOS_ENABLE_PRAGMA_IVDEP 1 +// #define KOKKOS_ENABLE_PRAGMA_LOOPCOUNT 1 +// #define KOKKOS_ENABLE_PRAGMA_VECTOR 1 #if !defined(KOKKOS_IMPL_HOST_FORCEINLINE_FUNCTION) #define KOKKOS_IMPL_HOST_FORCEINLINE_FUNCTION \ @@ -273,10 +273,10 @@ // GNU Compiler macros #if defined(KOKKOS_COMPILER_GNU) -//#define KOKKOS_ENABLE_PRAGMA_UNROLL 1 -//#define KOKKOS_ENABLE_PRAGMA_IVDEP 1 -//#define KOKKOS_ENABLE_PRAGMA_LOOPCOUNT 1 -//#define KOKKOS_ENABLE_PRAGMA_VECTOR 1 +// #define KOKKOS_ENABLE_PRAGMA_UNROLL 1 +// #define KOKKOS_ENABLE_PRAGMA_IVDEP 1 +// #define KOKKOS_ENABLE_PRAGMA_LOOPCOUNT 1 +// #define KOKKOS_ENABLE_PRAGMA_VECTOR 1 #if !defined(KOKKOS_IMPL_HOST_FORCEINLINE_FUNCTION) #define KOKKOS_IMPL_HOST_FORCEINLINE_FUNCTION \ @@ -298,7 +298,7 @@ #if defined(KOKKOS_COMPILER_NVHPC) #define KOKKOS_ENABLE_PRAGMA_UNROLL 1 #define KOKKOS_ENABLE_PRAGMA_IVDEP 1 -//#define KOKKOS_ENABLE_PRAGMA_LOOPCOUNT 1 +// #define KOKKOS_ENABLE_PRAGMA_LOOPCOUNT 1 #define KOKKOS_ENABLE_PRAGMA_VECTOR 1 #endif @@ -357,6 +357,21 @@ #define KOKKOS_IMPL_DEVICE_FUNCTION #endif +// FIXME_OPENACC FIXME_OPENMPTARGET +// Move to setup files once there is more content +// clang-format off +#if defined(KOKKOS_ENABLE_OPENACC) +#define KOKKOS_IMPL_RELOCATABLE_FUNCTION @"KOKKOS_RELOCATABLE_FUNCTION is not supported for the OpenACC backend" +#endif +#if defined(KOKKOS_ENABLE_OPENMPTARGET) +#define KOKKOS_IMPL_RELOCATABLE_FUNCTION @"KOKKOS_RELOCATABLE_FUNCTION is not supported for the OpenMPTarget backend" +#endif +// clang-format on + +#if !defined(KOKKOS_IMPL_RELOCATABLE_FUNCTION) +#define KOKKOS_IMPL_RELOCATABLE_FUNCTION +#endif + //---------------------------------------------------------------------------- // Define final version of functions. This is so that clang tidy can find these // macros more easily @@ -369,10 +384,14 @@ #define KOKKOS_FORCEINLINE_FUNCTION \ KOKKOS_IMPL_FORCEINLINE_FUNCTION \ __attribute__((annotate("KOKKOS_FORCEINLINE_FUNCTION"))) +#define KOKKOS_RELOCATABLE_FUNCTION \ + KOKKOS_IMPL_RELOCATABLE_FUNCTION \ + __attribute__((annotate("KOKKOS_RELOCATABLE_FUNCTION"))) #else #define KOKKOS_FUNCTION KOKKOS_IMPL_FUNCTION #define KOKKOS_INLINE_FUNCTION KOKKOS_IMPL_INLINE_FUNCTION #define KOKKOS_FORCEINLINE_FUNCTION KOKKOS_IMPL_FORCEINLINE_FUNCTION +#define KOKKOS_RELOCATABLE_FUNCTION KOKKOS_IMPL_RELOCATABLE_FUNCTION #endif //---------------------------------------------------------------------------- @@ -537,14 +556,17 @@ static constexpr bool kokkos_omp_on_host() { return false; } // If compiling with CUDA, we must use relocatable device code to enable the // task policy. +#ifdef KOKKOS_ENABLE_DEPRECATED_CODE_4 #if defined(KOKKOS_ENABLE_CUDA) #if defined(KOKKOS_ENABLE_CUDA_RELOCATABLE_DEVICE_CODE) #define KOKKOS_ENABLE_TASKDAG #endif // FIXME_SYCL Tasks not implemented -#elif !defined(KOKKOS_ENABLE_HIP) && !defined(KOKKOS_ENABLE_SYCL) +#elif !defined(KOKKOS_ENABLE_HIP) && !defined(KOKKOS_ENABLE_SYCL) && \ + !defined(KOKKOS_ENABLE_OPENMPTARGET) #define KOKKOS_ENABLE_TASKDAG #endif +#endif #if defined(KOKKOS_ENABLE_CUDA) && defined(KOKKOS_ENABLE_DEPRECATED_CODE_4) #define KOKKOS_ENABLE_CUDA_LDG_INTRINSIC @@ -582,9 +604,11 @@ static constexpr bool kokkos_omp_on_host() { return false; } // clang-format off #if defined(__NVCOMPILER) #define KOKKOS_IMPL_DISABLE_DEPRECATED_WARNINGS_PUSH() \ - _Pragma("diag_suppress 1216") + _Pragma("diag_suppress 1216") \ + _Pragma("diag_suppress deprecated_entity_with_custom_message") #define KOKKOS_IMPL_DISABLE_DEPRECATED_WARNINGS_POP() \ - _Pragma("diag_default 1216") + _Pragma("diag_default 1216") \ + _Pragma("diag_suppress deprecated_entity_with_custom_message") #elif defined(__EDG__) #define KOKKOS_IMPL_DISABLE_DEPRECATED_WARNINGS_PUSH() \ _Pragma("warning push") \ @@ -607,6 +631,18 @@ static constexpr bool kokkos_omp_on_host() { return false; } #define KOKKOS_IMPL_DISABLE_DEPRECATED_WARNINGS_PUSH() #define KOKKOS_IMPL_DISABLE_DEPRECATED_WARNINGS_POP() #endif + +#if defined(__NVCOMPILER) +#define KOKKOS_IMPL_DISABLE_UNREACHABLE_WARNINGS_PUSH() \ + _Pragma("diag_suppress code_is_unreachable") \ + _Pragma("diag_suppress initialization_not_reachable") +#define KOKKOS_IMPL_DISABLE_UNREACHABLE_WARNINGS_POP() \ + _Pragma("diag_default code_is_unreachable") \ + _Pragma("diag_default initialization_not_reachable") +#else +#define KOKKOS_IMPL_DISABLE_UNREACHABLE_WARNINGS_PUSH() +#define KOKKOS_IMPL_DISABLE_UNREACHABLE_WARNINGS_POP() +#endif // clang-format on #define KOKKOS_ATTRIBUTE_NODISCARD [[nodiscard]] diff --git a/lib/kokkos/core/src/Kokkos_MemoryPool.hpp b/lib/kokkos/core/src/Kokkos_MemoryPool.hpp index ce8c9e152f..f7e9e2a78c 100644 --- a/lib/kokkos/core/src/Kokkos_MemoryPool.hpp +++ b/lib/kokkos/core/src/Kokkos_MemoryPool.hpp @@ -196,9 +196,10 @@ class MemoryPool { stats.consumed_superblocks++; stats.consumed_blocks += block_used; - stats.consumed_bytes += block_used * block_size; + stats.consumed_bytes += static_cast(block_used) * block_size; stats.reserved_blocks += block_count - block_used; - stats.reserved_bytes += (block_count - block_used) * block_size; + stats.reserved_bytes += + static_cast(block_count - block_used) * block_size; } } @@ -234,9 +235,9 @@ class MemoryPool { //-------------------------------------------------------------------------- - KOKKOS_DEFAULTED_FUNCTION MemoryPool(MemoryPool &&) = default; - KOKKOS_DEFAULTED_FUNCTION MemoryPool(const MemoryPool &) = default; - KOKKOS_DEFAULTED_FUNCTION MemoryPool &operator=(MemoryPool &&) = default; + KOKKOS_DEFAULTED_FUNCTION MemoryPool(MemoryPool &&) = default; + KOKKOS_DEFAULTED_FUNCTION MemoryPool(const MemoryPool &) = default; + KOKKOS_DEFAULTED_FUNCTION MemoryPool &operator=(MemoryPool &&) = default; KOKKOS_DEFAULTED_FUNCTION MemoryPool &operator=(const MemoryPool &) = default; KOKKOS_INLINE_FUNCTION MemoryPool() diff --git a/lib/kokkos/core/src/Kokkos_NumericTraits.hpp b/lib/kokkos/core/src/Kokkos_NumericTraits.hpp index 118bf52c05..1304d3ba92 100644 --- a/lib/kokkos/core/src/Kokkos_NumericTraits.hpp +++ b/lib/kokkos/core/src/Kokkos_NumericTraits.hpp @@ -114,7 +114,7 @@ template <> struct signaling_NaN_helper { static constexpr long dou #endif template struct digits_helper {}; template <> struct digits_helper { static constexpr int value = 1; }; -template <> struct digits_helper { static constexpr int value = CHAR_BIT - std::is_signed::value; }; +template <> struct digits_helper { static constexpr int value = CHAR_BIT - std::is_signed_v; }; template <> struct digits_helper { static constexpr int value = CHAR_BIT - 1; }; template <> struct digits_helper { static constexpr int value = CHAR_BIT; }; template <> struct digits_helper { static constexpr int value = CHAR_BIT*sizeof(short)-1; }; diff --git a/lib/kokkos/core/src/Kokkos_Pair.hpp b/lib/kokkos/core/src/Kokkos_Pair.hpp index e569fefc14..c44d1f2310 100644 --- a/lib/kokkos/core/src/Kokkos_Pair.hpp +++ b/lib/kokkos/core/src/Kokkos_Pair.hpp @@ -449,7 +449,8 @@ struct KOKKOS_DEPRECATED pair { // Specialization of relational operators for Kokkos::pair. // -#if defined(KOKKOS_COMPILER_GNU) && (KOKKOS_COMPILER_GNU < 1110) +#if defined(KOKKOS_ENABLE_DEPRECATION_WARNINGS) && \ + defined(KOKKOS_COMPILER_GNU) && (KOKKOS_COMPILER_GNU < 1110) KOKKOS_IMPL_DISABLE_DEPRECATED_WARNINGS_PUSH() #endif template @@ -487,7 +488,8 @@ KOKKOS_DEPRECATED KOKKOS_FORCEINLINE_FUNCTION constexpr bool operator>=( const pair& lhs, const pair& rhs) { return !(lhs < rhs); } -#if defined(KOKKOS_COMPILER_GNU) && (KOKKOS_COMPILER_GNU < 1110) +#if defined(KOKKOS_ENABLE_DEPRECATION_WARNINGS) && \ + defined(KOKKOS_COMPILER_GNU) && (KOKKOS_COMPILER_GNU < 1110) KOKKOS_IMPL_DISABLE_DEPRECATED_WARNINGS_POP() #endif #endif diff --git a/lib/kokkos/core/src/Kokkos_Parallel.hpp b/lib/kokkos/core/src/Kokkos_Parallel.hpp index 122239df79..24349e95ae 100644 --- a/lib/kokkos/core/src/Kokkos_Parallel.hpp +++ b/lib/kokkos/core/src/Kokkos_Parallel.hpp @@ -72,19 +72,19 @@ struct FunctorPolicyExecutionSpace { static_assert( !is_detected::value || !is_detected::value || - std::is_same::value, + std::is_same_v, "A policy with an execution space and a functor with an execution space " "are given but the execution space types do not match!"); static_assert(!is_detected::value || !is_detected::value || - std::is_same::value, + std::is_same_v, "A policy with an execution space and a functor with a device " "type are given but the execution space types do not match!"); static_assert(!is_detected::value || !is_detected::value || - std::is_same::value, + std::is_same_v, "A functor with both an execution space and device type is " "given but their execution space types do not match!"); @@ -134,8 +134,10 @@ inline void parallel_for(const std::string& str, const ExecPolicy& policy, const FunctorType& functor) { uint64_t kpID = 0; - ExecPolicy inner_policy = policy; - Kokkos::Tools::Impl::begin_parallel_for(inner_policy, functor, str, kpID); + /** Request a tuned policy from the tools subsystem */ + const auto& response = + Kokkos::Tools::Impl::begin_parallel_for(policy, functor, str, kpID); + const auto& inner_policy = response.policy; auto closure = Kokkos::Impl::construct_with_shared_allocation_tracking_disabled< @@ -348,9 +350,11 @@ template ::value>> inline void parallel_scan(const std::string& str, const ExecutionPolicy& policy, const FunctorType& functor) { - uint64_t kpID = 0; - ExecutionPolicy inner_policy = policy; - Kokkos::Tools::Impl::begin_parallel_scan(inner_policy, functor, str, kpID); + uint64_t kpID = 0; + /** Request a tuned policy from the tools subsystem */ + const auto& response = + Kokkos::Tools::Impl::begin_parallel_scan(policy, functor, str, kpID); + const auto& inner_policy = response.policy; auto closure = Kokkos::Impl::construct_with_shared_allocation_tracking_disabled< diff --git a/lib/kokkos/core/src/Kokkos_Parallel_Reduce.hpp b/lib/kokkos/core/src/Kokkos_Parallel_Reduce.hpp index 53913266f1..3b89d184f2 100644 --- a/lib/kokkos/core/src/Kokkos_Parallel_Reduce.hpp +++ b/lib/kokkos/core/src/Kokkos_Parallel_Reduce.hpp @@ -73,7 +73,7 @@ struct Sum { template KOKKOS_DEDUCTION_GUIDE Sum(View const&) - ->Sum::memory_space>; + -> Sum::memory_space>; template struct Prod { @@ -118,7 +118,7 @@ struct Prod { template KOKKOS_DEDUCTION_GUIDE Prod(View const&) - ->Prod::memory_space>; + -> Prod::memory_space>; template struct Min { @@ -165,7 +165,7 @@ struct Min { template KOKKOS_DEDUCTION_GUIDE Min(View const&) - ->Min::memory_space>; + -> Min::memory_space>; template struct Max { @@ -213,7 +213,7 @@ struct Max { template KOKKOS_DEDUCTION_GUIDE Max(View const&) - ->Max::memory_space>; + -> Max::memory_space>; template struct LAnd { @@ -259,7 +259,7 @@ struct LAnd { template KOKKOS_DEDUCTION_GUIDE LAnd(View const&) - ->LAnd::memory_space>; + -> LAnd::memory_space>; template struct LOr { @@ -306,7 +306,7 @@ struct LOr { template KOKKOS_DEDUCTION_GUIDE LOr(View const&) - ->LOr::memory_space>; + -> LOr::memory_space>; template struct BAnd { @@ -353,7 +353,7 @@ struct BAnd { template KOKKOS_DEDUCTION_GUIDE BAnd(View const&) - ->BAnd::memory_space>; + -> BAnd::memory_space>; template struct BOr { @@ -400,7 +400,7 @@ struct BOr { template KOKKOS_DEDUCTION_GUIDE BOr(View const&) - ->BOr::memory_space>; + -> BOr::memory_space>; template struct ValLocScalar { @@ -438,7 +438,12 @@ struct MinLoc { // Required KOKKOS_INLINE_FUNCTION void join(value_type& dest, const value_type& src) const { - if (src.val < dest.val) dest = src; + if (src.val < dest.val) + dest = src; + else if (src.val == dest.val && + dest.loc == reduction_identity::min()) { + dest.loc = src.loc; + } } KOKKOS_INLINE_FUNCTION @@ -458,11 +463,10 @@ struct MinLoc { }; template -KOKKOS_DEDUCTION_GUIDE MinLoc( - View, Properties...> const&) - ->MinLoc, - Properties...>::memory_space>; +KOKKOS_DEDUCTION_GUIDE +MinLoc(View, Properties...> const&) -> MinLoc< + Scalar, Index, + typename View, Properties...>::memory_space>; template struct MaxLoc { @@ -494,7 +498,12 @@ struct MaxLoc { // Required KOKKOS_INLINE_FUNCTION void join(value_type& dest, const value_type& src) const { - if (src.val > dest.val) dest = src; + if (src.val > dest.val) + dest = src; + else if (src.val == dest.val && + dest.loc == reduction_identity::min()) { + dest.loc = src.loc; + } } KOKKOS_INLINE_FUNCTION @@ -514,11 +523,10 @@ struct MaxLoc { }; template -KOKKOS_DEDUCTION_GUIDE MaxLoc( - View, Properties...> const&) - ->MaxLoc, - Properties...>::memory_space>; +KOKKOS_DEDUCTION_GUIDE +MaxLoc(View, Properties...> const&) -> MaxLoc< + Scalar, Index, + typename View, Properties...>::memory_space>; template struct MinMaxScalar { @@ -580,8 +588,8 @@ struct MinMax { template KOKKOS_DEDUCTION_GUIDE MinMax(View, Properties...> const&) - ->MinMax, Properties...>::memory_space>; + -> MinMax, Properties...>::memory_space>; template struct MinMaxLocScalar { @@ -622,10 +630,16 @@ struct MinMaxLoc { if (src.min_val < dest.min_val) { dest.min_val = src.min_val; dest.min_loc = src.min_loc; + } else if (dest.min_val == src.min_val && + dest.min_loc == reduction_identity::min()) { + dest.min_loc = src.min_loc; } if (src.max_val > dest.max_val) { dest.max_val = src.max_val; dest.max_loc = src.max_loc; + } else if (dest.max_val == src.max_val && + dest.max_loc == reduction_identity::min()) { + dest.max_loc = src.max_loc; } } @@ -650,9 +664,9 @@ struct MinMaxLoc { template KOKKOS_DEDUCTION_GUIDE MinMaxLoc( View, Properties...> const&) - ->MinMaxLoc, - Properties...>::memory_space>; + -> MinMaxLoc, + Properties...>::memory_space>; // -------------------------------------------------- // reducers added to support std algorithms @@ -718,9 +732,9 @@ struct MaxFirstLoc { template KOKKOS_DEDUCTION_GUIDE MaxFirstLoc( View, Properties...> const&) - ->MaxFirstLoc, - Properties...>::memory_space>; + -> MaxFirstLoc, + Properties...>::memory_space>; // // MaxFirstLocCustomComparator @@ -788,9 +802,9 @@ template KOKKOS_DEDUCTION_GUIDE MaxFirstLocCustomComparator( View, Properties...> const&, ComparatorType) - ->MaxFirstLocCustomComparator, - Properties...>::memory_space>; + -> MaxFirstLocCustomComparator, + Properties...>::memory_space>; // // MinFirstLoc @@ -852,9 +866,9 @@ struct MinFirstLoc { template KOKKOS_DEDUCTION_GUIDE MinFirstLoc( View, Properties...> const&) - ->MinFirstLoc, - Properties...>::memory_space>; + -> MinFirstLoc, + Properties...>::memory_space>; // // MinFirstLocCustomComparator @@ -922,9 +936,9 @@ template KOKKOS_DEDUCTION_GUIDE MinFirstLocCustomComparator( View, Properties...> const&, ComparatorType) - ->MinFirstLocCustomComparator, - Properties...>::memory_space>; + -> MinFirstLocCustomComparator, + Properties...>::memory_space>; // // MinMaxFirstLastLoc @@ -997,9 +1011,9 @@ struct MinMaxFirstLastLoc { template KOKKOS_DEDUCTION_GUIDE MinMaxFirstLastLoc( View, Properties...> const&) - ->MinMaxFirstLastLoc, - Properties...>::memory_space>; + -> MinMaxFirstLastLoc, + Properties...>::memory_space>; // // MinMaxFirstLastLocCustomComparator @@ -1077,7 +1091,7 @@ template KOKKOS_DEDUCTION_GUIDE MinMaxFirstLastLocCustomComparator( View, Properties...> const&, ComparatorType) - ->MinMaxFirstLastLocCustomComparator< + -> MinMaxFirstLastLocCustomComparator< Scalar, Index, ComparatorType, typename View, Properties...>::memory_space>; @@ -1139,10 +1153,9 @@ struct FirstLoc { }; template -KOKKOS_DEDUCTION_GUIDE FirstLoc( - View, Properties...> const&) - ->FirstLoc, - Properties...>::memory_space>; +KOKKOS_DEDUCTION_GUIDE +FirstLoc(View, Properties...> const&) -> FirstLoc< + Index, typename View, Properties...>::memory_space>; // // LastLoc @@ -1202,8 +1215,8 @@ struct LastLoc { template KOKKOS_DEDUCTION_GUIDE LastLoc(View, Properties...> const&) - ->LastLoc, Properties...>::memory_space>; + -> LastLoc, + Properties...>::memory_space>; template struct StdIsPartScalar { @@ -1270,8 +1283,8 @@ struct StdIsPartitioned { template KOKKOS_DEDUCTION_GUIDE StdIsPartitioned( View, Properties...> const&) - ->StdIsPartitioned, - Properties...>::memory_space>; + -> StdIsPartitioned, + Properties...>::memory_space>; template struct StdPartPointScalar { @@ -1333,8 +1346,8 @@ struct StdPartitionPoint { template KOKKOS_DEDUCTION_GUIDE StdPartitionPoint( View, Properties...> const&) - ->StdPartitionPoint, - Properties...>::memory_space>; + -> StdPartitionPoint, + Properties...>::memory_space>; } // namespace Kokkos namespace Kokkos { @@ -1404,9 +1417,9 @@ struct ParallelReduceReturnValue< template struct ParallelReduceReturnValue< std::enable_if_t::value && - (!std::is_array::value && - !std::is_pointer::value) && - !Kokkos::is_reducer::value>, + (!std::is_array_v && + !std::is_pointer_v< + ReturnType>)&&!Kokkos::is_reducer::value>, ReturnType, FunctorType> { using return_type = Kokkos::View; @@ -1422,8 +1435,8 @@ struct ParallelReduceReturnValue< template struct ParallelReduceReturnValue< - std::enable_if_t<(std::is_array::value || - std::is_pointer::value)>, + std::enable_if_t<(std::is_array_v || + std::is_pointer_v)>, ReturnType, FunctorType> { using return_type = Kokkos::View, Kokkos::HostSpace, Kokkos::MemoryUnmanaged>; @@ -1434,7 +1447,7 @@ struct ParallelReduceReturnValue< static return_type return_value(ReturnType& return_val, const FunctorType& functor) { - if (std::is_array::value) + if (std::is_array_v) return return_type(return_val); else return return_type(return_val, functor.value_count); @@ -1467,8 +1480,7 @@ struct ParallelReducePolicyType< template struct ParallelReducePolicyType< - std::enable_if_t::value>, PolicyType, - FunctorType> { + std::enable_if_t>, PolicyType, FunctorType> { using execution_space = typename Impl::FunctorPolicyExecutionSpace::execution_space; @@ -1501,27 +1513,28 @@ struct ParallelReduceAdaptor { using PassedReducerType = typename return_value_adapter::reducer_type; uint64_t kpID = 0; - PolicyType inner_policy = policy; - Kokkos::Tools::Impl::begin_parallel_reduce( - inner_policy, functor, label, kpID); - using ReducerSelector = - Kokkos::Impl::if_c::value, + Kokkos::Impl::if_c, FunctorType, PassedReducerType>; using Analysis = FunctorAnalysis; - using CombinedFunctorReducerType = CombinedFunctorReducer; + + CombinedFunctorReducerType functor_reducer( + functor, typename Analysis::Reducer( + ReducerSelector::select(functor, return_value))); + const auto& response = Kokkos::Tools::Impl::begin_parallel_reduce< + typename return_value_adapter::reducer_type>(policy, functor_reducer, + label, kpID); + const auto& inner_policy = response.policy; + auto closure = construct_with_shared_allocation_tracking_disabled< Impl::ParallelReduce::execution_space>>( - CombinedFunctorReducerType( - functor, typename Analysis::Reducer( - ReducerSelector::select(functor, return_value))), - inner_policy, + functor_reducer, inner_policy, return_value_adapter::return_value(return_value, functor)); closure.execute(); @@ -1536,7 +1549,7 @@ struct ParallelReduceAdaptor { template static inline std::enable_if_t::value)> + std::is_pointer_v)> execute(const std::string& label, const PolicyType& policy, const FunctorType& functor, ReturnType& return_value) { execute_impl(label, policy, functor, return_value); @@ -1568,7 +1581,7 @@ struct ReducerHasTestReferenceFunction { static std::false_type test_func(...); enum { - value = std::is_same(nullptr))>::value + value = std::is_same_v(nullptr))> }; }; @@ -1611,7 +1624,7 @@ struct ParallelReduceFence { template static void fence(const ExecutionSpace& ex, const std::string& name, ArgsDeduced&&... args) { - if (Impl::parallel_reduce_needs_fence(ex, (ArgsDeduced &&) args...)) { + if (Impl::parallel_reduce_needs_fence(ex, (ArgsDeduced&&)args...)) { ex.fence(name); } } @@ -1663,11 +1676,11 @@ template inline std::enable_if_t::value && !(Kokkos::is_view::value || Kokkos::is_reducer::value || - std::is_pointer::value)> + std::is_pointer_v)> parallel_reduce(const std::string& label, const PolicyType& policy, const FunctorType& functor, ReturnType& return_value) { static_assert( - !std::is_const::value, + !std::is_const_v, "A const reduction result type is only allowed for a View, pointer or " "reducer return type!"); @@ -1684,11 +1697,11 @@ template inline std::enable_if_t::value && !(Kokkos::is_view::value || Kokkos::is_reducer::value || - std::is_pointer::value)> + std::is_pointer_v)> parallel_reduce(const PolicyType& policy, const FunctorType& functor, ReturnType& return_value) { static_assert( - !std::is_const::value, + !std::is_const_v, "A const reduction result type is only allowed for a View, pointer or " "reducer return type!"); @@ -1704,11 +1717,11 @@ parallel_reduce(const PolicyType& policy, const FunctorType& functor, template inline std::enable_if_t::value || Kokkos::is_reducer::value || - std::is_pointer::value)> + std::is_pointer_v)> parallel_reduce(const size_t& policy, const FunctorType& functor, ReturnType& return_value) { static_assert( - !std::is_const::value, + !std::is_const_v, "A const reduction result type is only allowed for a View, pointer or " "reducer return type!"); @@ -1728,11 +1741,11 @@ parallel_reduce(const size_t& policy, const FunctorType& functor, template inline std::enable_if_t::value || Kokkos::is_reducer::value || - std::is_pointer::value)> + std::is_pointer_v)> parallel_reduce(const std::string& label, const size_t& policy, const FunctorType& functor, ReturnType& return_value) { static_assert( - !std::is_const::value, + !std::is_const_v, "A const reduction result type is only allowed for a View, pointer or " "reducer return type!"); @@ -1754,7 +1767,7 @@ template inline std::enable_if_t::value && (Kokkos::is_view::value || Kokkos::is_reducer::value || - std::is_pointer::value)> + std::is_pointer_v)> parallel_reduce(const std::string& label, const PolicyType& policy, const FunctorType& functor, const ReturnType& return_value) { ReturnType return_value_impl = return_value; @@ -1771,7 +1784,7 @@ template inline std::enable_if_t::value && (Kokkos::is_view::value || Kokkos::is_reducer::value || - std::is_pointer::value)> + std::is_pointer_v)> parallel_reduce(const PolicyType& policy, const FunctorType& functor, const ReturnType& return_value) { ReturnType return_value_impl = return_value; @@ -1787,7 +1800,7 @@ parallel_reduce(const PolicyType& policy, const FunctorType& functor, template inline std::enable_if_t::value || Kokkos::is_reducer::value || - std::is_pointer::value> + std::is_pointer_v> parallel_reduce(const size_t& policy, const FunctorType& functor, const ReturnType& return_value) { using policy_type = @@ -1806,7 +1819,7 @@ parallel_reduce(const size_t& policy, const FunctorType& functor, template inline std::enable_if_t::value || Kokkos::is_reducer::value || - std::is_pointer::value> + std::is_pointer_v> parallel_reduce(const std::string& label, const size_t& policy, const FunctorType& functor, const ReturnType& return_value) { using policy_type = diff --git a/lib/kokkos/core/src/Kokkos_Profiling_ProfileSection.hpp b/lib/kokkos/core/src/Kokkos_Profiling_ProfileSection.hpp index e7a9ba0c7e..1759c2b4a1 100644 --- a/lib/kokkos/core/src/Kokkos_Profiling_ProfileSection.hpp +++ b/lib/kokkos/core/src/Kokkos_Profiling_ProfileSection.hpp @@ -32,7 +32,7 @@ class [[nodiscard]] ProfilingSection { uint32_t sectionID; public: - ProfilingSection(ProfilingSection const&) = delete; + ProfilingSection(ProfilingSection const&) = delete; ProfilingSection& operator=(ProfilingSection const&) = delete; #if defined(__has_cpp_attribute) && __has_cpp_attribute(nodiscard) >= 201907 diff --git a/lib/kokkos/core/src/Kokkos_Profiling_ScopedRegion.hpp b/lib/kokkos/core/src/Kokkos_Profiling_ScopedRegion.hpp index f45dfa324e..a4168b9401 100644 --- a/lib/kokkos/core/src/Kokkos_Profiling_ScopedRegion.hpp +++ b/lib/kokkos/core/src/Kokkos_Profiling_ScopedRegion.hpp @@ -30,7 +30,7 @@ namespace Kokkos::Profiling { class [[nodiscard]] ScopedRegion { public: - ScopedRegion(ScopedRegion const &) = delete; + ScopedRegion(ScopedRegion const &) = delete; ScopedRegion &operator=(ScopedRegion const &) = delete; #if defined(__has_cpp_attribute) && __has_cpp_attribute(nodiscard) >= 201907 diff --git a/lib/kokkos/core/src/Kokkos_ScratchSpace.hpp b/lib/kokkos/core/src/Kokkos_ScratchSpace.hpp index a925e32a33..f00e25fdb6 100644 --- a/lib/kokkos/core/src/Kokkos_ScratchSpace.hpp +++ b/lib/kokkos/core/src/Kokkos_ScratchSpace.hpp @@ -110,7 +110,7 @@ class ScratchMemorySpace { // Note: for team scratch m_offset is 0, since every // thread will get back the same shared pointer void* tmp = m_iter + m_offset * size; - uintptr_t increment = size * m_multiplier; + uintptr_t increment = static_cast(size) * m_multiplier; // Cast to uintptr_t to avoid problems with pointer arithmetic using SYCL const auto end_iter = diff --git a/lib/kokkos/core/src/Kokkos_TaskScheduler.hpp b/lib/kokkos/core/src/Kokkos_TaskScheduler.hpp index 869a5f8ec2..3edecb4502 100644 --- a/lib/kokkos/core/src/Kokkos_TaskScheduler.hpp +++ b/lib/kokkos/core/src/Kokkos_TaskScheduler.hpp @@ -14,11 +14,17 @@ // //@HEADER -#ifndef KOKKOS_IMPL_PUBLIC_INCLUDE #include + +#ifndef KOKKOS_IMPL_PUBLIC_INCLUDE static_assert(false, "Including non-public Kokkos header files is not allowed."); #endif + +#ifndef KOKKOS_ENABLE_DEPRECATED_CODE_4 +#error "The tasking framework is deprecated" +#endif + #ifndef KOKKOS_TASKSCHEDULER_HPP #define KOKKOS_TASKSCHEDULER_HPP @@ -44,6 +50,11 @@ static_assert(false, //---------------------------------------------------------------------------- //---------------------------------------------------------------------------- +#ifdef KOKKOS_ENABLE_DEPRECATION_WARNINGS +// We allow using deprecated classes in this file +KOKKOS_IMPL_DISABLE_DEPRECATED_WARNINGS_PUSH() +#endif + namespace Kokkos { namespace Impl { @@ -54,7 +65,7 @@ class TaskExec; } // end namespace Impl template -class BasicTaskScheduler : public Impl::TaskSchedulerBase { +class KOKKOS_DEPRECATED BasicTaskScheduler : public Impl::TaskSchedulerBase { public: using scheduler_type = BasicTaskScheduler; using execution_space = ExecSpace; @@ -494,8 +505,8 @@ namespace Kokkos { // Construct a TaskTeam execution policy template -Impl::TaskPolicyWithPredecessor> +KOKKOS_DEPRECATED Impl::TaskPolicyWithPredecessor< + Impl::TaskType::TaskTeam, Kokkos::BasicFuture> KOKKOS_INLINE_FUNCTION TaskTeam(Kokkos::BasicFuture arg_future, TaskPriority arg_priority = TaskPriority::Regular) { @@ -503,7 +514,8 @@ Impl::TaskPolicyWithPredecessor -Impl::TaskPolicyWithScheduler +KOKKOS_DEPRECATED Impl::TaskPolicyWithScheduler KOKKOS_INLINE_FUNCTION TaskTeam( Scheduler arg_scheduler, std::enable_if_t::value, TaskPriority> @@ -512,18 +524,18 @@ Impl::TaskPolicyWithScheduler } template -Impl::TaskPolicyWithScheduler +KOKKOS_DEPRECATED Impl::TaskPolicyWithScheduler< + Kokkos::Impl::TaskType::TaskTeam, Scheduler, PredecessorFuture> KOKKOS_INLINE_FUNCTION TaskTeam(Scheduler arg_scheduler, PredecessorFuture arg_future, std::enable_if_t::value && Kokkos::is_future::value, TaskPriority> arg_priority = TaskPriority::Regular) { - static_assert(std::is_same::value, - "Can't create a task policy from a scheduler and a future from " - "a different scheduler"); + static_assert( + std::is_same_v, + "Can't create a task policy from a scheduler and a future from " + "a different scheduler"); return {std::move(arg_scheduler), std::move(arg_future), arg_priority}; } @@ -531,8 +543,8 @@ Impl::TaskPolicyWithScheduler -Impl::TaskPolicyWithPredecessor> +KOKKOS_DEPRECATED Impl::TaskPolicyWithPredecessor< + Impl::TaskType::TaskSingle, Kokkos::BasicFuture> KOKKOS_INLINE_FUNCTION TaskSingle(Kokkos::BasicFuture arg_future, TaskPriority arg_priority = TaskPriority::Regular) { @@ -540,7 +552,8 @@ Impl::TaskPolicyWithPredecessor -Impl::TaskPolicyWithScheduler +KOKKOS_DEPRECATED Impl::TaskPolicyWithScheduler KOKKOS_INLINE_FUNCTION TaskSingle( Scheduler arg_scheduler, std::enable_if_t::value, TaskPriority> @@ -549,18 +562,18 @@ Impl::TaskPolicyWithScheduler } template -Impl::TaskPolicyWithScheduler +KOKKOS_DEPRECATED Impl::TaskPolicyWithScheduler< + Kokkos::Impl::TaskType::TaskSingle, Scheduler, PredecessorFuture> KOKKOS_INLINE_FUNCTION TaskSingle(Scheduler arg_scheduler, PredecessorFuture arg_future, std::enable_if_t::value && Kokkos::is_future::value, TaskPriority> arg_priority = TaskPriority::Regular) { - static_assert(std::is_same::value, - "Can't create a task policy from a scheduler and a future from " - "a different scheduler"); + static_assert( + std::is_same_v, + "Can't create a task policy from a scheduler and a future from " + "a different scheduler"); return {std::move(arg_scheduler), std::move(arg_future), arg_priority}; } @@ -575,7 +588,8 @@ Impl::TaskPolicyWithScheduler -typename Scheduler::template future_type_for_functor> +KOKKOS_DEPRECATED typename Scheduler::template future_type_for_functor< + std::decay_t> host_spawn(Impl::TaskPolicyWithScheduler arg_policy, FunctorType&& arg_functor) { @@ -606,7 +620,8 @@ host_spawn(Impl::TaskPolicyWithScheduler */ template -typename Scheduler::template future_type_for_functor> +KOKKOS_DEPRECATED typename Scheduler::template future_type_for_functor< + std::decay_t> KOKKOS_INLINE_FUNCTION task_spawn(Impl::TaskPolicyWithScheduler arg_policy, @@ -633,7 +648,7 @@ typename Scheduler::template future_type_for_functor> * 2) High, Normal, or Low priority */ template -void KOKKOS_INLINE_FUNCTION +KOKKOS_DEPRECATED void KOKKOS_INLINE_FUNCTION respawn(FunctorType* arg_self, T const& arg, TaskPriority const& arg_priority = TaskPriority::Regular) { static_assert(Kokkos::is_future::value || Kokkos::is_scheduler::value, @@ -656,7 +671,8 @@ respawn(FunctorType* arg_self, T const& arg, // Wait for all runnable tasks to complete template -inline void wait(BasicTaskScheduler const& scheduler) { +KOKKOS_DEPRECATED inline void wait( + BasicTaskScheduler const& scheduler) { using scheduler_type = BasicTaskScheduler; scheduler_type::specialization::execute(scheduler); // scheduler.m_queue->execute(); @@ -664,6 +680,10 @@ inline void wait(BasicTaskScheduler const& scheduler) { } // namespace Kokkos +#ifdef KOKKOS_ENABLE_DEPRECATION_WARNINGS +KOKKOS_IMPL_DISABLE_DEPRECATED_WARNINGS_POP() +#endif + //---------------------------------------------------------------------------- //---------------------------------------------------------------------------- diff --git a/lib/kokkos/core/src/Kokkos_TaskScheduler_fwd.hpp b/lib/kokkos/core/src/Kokkos_TaskScheduler_fwd.hpp index 203fb16eaf..83e1c06db9 100644 --- a/lib/kokkos/core/src/Kokkos_TaskScheduler_fwd.hpp +++ b/lib/kokkos/core/src/Kokkos_TaskScheduler_fwd.hpp @@ -31,31 +31,40 @@ static_assert(false, #include //---------------------------------------------------------------------------- +#ifdef KOKKOS_ENABLE_DEPRECATION_WARNINGS +// We allow using deprecated classes in this file +KOKKOS_IMPL_DISABLE_DEPRECATED_WARNINGS_PUSH() +#endif + namespace Kokkos { // Forward declarations used in Impl::TaskQueue template -class BasicFuture; +class KOKKOS_DEPRECATED BasicFuture; template -class SimpleTaskScheduler; +class KOKKOS_DEPRECATED SimpleTaskScheduler; template -class BasicTaskScheduler; +class KOKKOS_DEPRECATED BasicTaskScheduler; template -struct is_scheduler : public std::false_type {}; +struct KOKKOS_DEPRECATED is_scheduler : public std::false_type {}; template -struct is_scheduler> : public std::true_type { -}; +struct KOKKOS_DEPRECATED is_scheduler> + : public std::true_type {}; template -struct is_scheduler> : public std::true_type { -}; +struct KOKKOS_DEPRECATED is_scheduler> + : public std::true_type {}; -enum class TaskPriority : int { High = 0, Regular = 1, Low = 2 }; +enum class KOKKOS_DEPRECATED TaskPriority : int { + High = 0, + Regular = 1, + Low = 2 +}; } // namespace Kokkos @@ -141,28 +150,28 @@ using default_tasking_memory_space_for_execution_space_t = namespace Kokkos { template -using DeprecatedTaskScheduler = BasicTaskScheduler< +using DeprecatedTaskScheduler KOKKOS_DEPRECATED = BasicTaskScheduler< Space, Impl::TaskQueue< Space, Impl::default_tasking_memory_space_for_execution_space_t>>; template -using DeprecatedTaskSchedulerMultiple = BasicTaskScheduler< +using DeprecatedTaskSchedulerMultiple KOKKOS_DEPRECATED = BasicTaskScheduler< Space, Impl::TaskQueueMultiple< Space, Impl::default_tasking_memory_space_for_execution_space_t>>; template -using TaskScheduler = SimpleTaskScheduler< +using TaskScheduler KOKKOS_DEPRECATED = SimpleTaskScheduler< Space, Impl::SingleTaskQueue< Space, Impl::default_tasking_memory_space_for_execution_space_t, Impl::TaskQueueTraitsLockBased>>; template -using TaskSchedulerMultiple = SimpleTaskScheduler< +using TaskSchedulerMultiple KOKKOS_DEPRECATED = SimpleTaskScheduler< Space, Impl::MultipleTaskQueue< Space, Impl::default_tasking_memory_space_for_execution_space_t, @@ -172,7 +181,7 @@ using TaskSchedulerMultiple = SimpleTaskScheduler< Impl::default_tasking_memory_space_for_execution_space_t>>>>; template -using ChaseLevTaskScheduler = SimpleTaskScheduler< +using ChaseLevTaskScheduler KOKKOS_DEPRECATED = SimpleTaskScheduler< Space, Impl::MultipleTaskQueue< Space, Impl::default_tasking_memory_space_for_execution_space_t, @@ -182,7 +191,7 @@ using ChaseLevTaskScheduler = SimpleTaskScheduler< Impl::default_tasking_memory_space_for_execution_space_t>>>>; template -void wait(BasicTaskScheduler const&); +KOKKOS_DEPRECATED void wait(BasicTaskScheduler const&); namespace Impl { @@ -204,6 +213,10 @@ struct TaskPolicyData; } // namespace Kokkos +#ifdef KOKKOS_ENABLE_DEPRECATION_WARNINGS +KOKKOS_IMPL_DISABLE_DEPRECATED_WARNINGS_POP() +#endif + //---------------------------------------------------------------------------- #endif /* #if defined( KOKKOS_ENABLE_TASKDAG ) */ diff --git a/lib/kokkos/core/src/Kokkos_Timer.hpp b/lib/kokkos/core/src/Kokkos_Timer.hpp index a210b6ff18..ab31484d76 100644 --- a/lib/kokkos/core/src/Kokkos_Timer.hpp +++ b/lib/kokkos/core/src/Kokkos_Timer.hpp @@ -48,7 +48,7 @@ class Timer { inline Timer() { reset(); } - Timer(const Timer&) = delete; + Timer(const Timer&) = delete; Timer& operator=(const Timer&) = delete; inline double seconds() const { diff --git a/lib/kokkos/core/src/Kokkos_Tuners.hpp b/lib/kokkos/core/src/Kokkos_Tuners.hpp index f5ffc66af5..fcb061b378 100644 --- a/lib/kokkos/core/src/Kokkos_Tuners.hpp +++ b/lib/kokkos/core/src/Kokkos_Tuners.hpp @@ -52,6 +52,8 @@ VariableValue make_variable_value(size_t, int64_t); VariableValue make_variable_value(size_t, double); SetOrRange make_candidate_range(double lower, double upper, double step, bool openLower, bool openUpper); +SetOrRange make_candidate_range(int64_t lower, int64_t upper, int64_t step, + bool openLower, bool openUpper); size_t get_new_context_id(); void begin_context(size_t context_id); void end_context(size_t context_id); @@ -412,18 +414,19 @@ class TeamSizeTuner : public ExtendableTunerMixin { TunerType tuner; public: - TeamSizeTuner() = default; + TeamSizeTuner() = default; TeamSizeTuner& operator=(const TeamSizeTuner& other) = default; TeamSizeTuner(const TeamSizeTuner& other) = default; - TeamSizeTuner& operator=(TeamSizeTuner&& other) = default; - TeamSizeTuner(TeamSizeTuner&& other) = default; + TeamSizeTuner& operator=(TeamSizeTuner&& other) = default; + TeamSizeTuner(TeamSizeTuner&& other) = default; template TeamSizeTuner(const std::string& name, - Kokkos::TeamPolicy& policy, + const Kokkos::TeamPolicy& policy_in, const Functor& functor, const TagType& tag, ViableConfigurationCalculator calc) { - using PolicyType = Kokkos::TeamPolicy; + using PolicyType = Kokkos::TeamPolicy; + PolicyType policy(policy_in); auto initial_vector_length = policy.impl_vector_length(); if (initial_vector_length < 1) { policy.impl_set_vector_length(1); @@ -505,7 +508,8 @@ class TeamSizeTuner : public ExtendableTunerMixin { } template - void tune(Kokkos::TeamPolicy& policy) { + auto tune(const Kokkos::TeamPolicy& policy_in) { + Kokkos::TeamPolicy policy(policy_in); if (Kokkos::Tools::Experimental::have_tuning_tool()) { auto configuration = tuner.begin(); auto team_size = std::get<1>(configuration); @@ -515,6 +519,111 @@ class TeamSizeTuner : public ExtendableTunerMixin { policy.impl_set_vector_length(vector_length); } } + return policy; + } + void end() { + if (Kokkos::Tools::Experimental::have_tuning_tool()) { + tuner.end(); + } + } + + TunerType get_tuner() const { return tuner; } +}; +namespace Impl { +template +struct tuning_type_for; + +template <> +struct tuning_type_for { + static constexpr Kokkos::Tools::Experimental::ValueType value = + Kokkos::Tools::Experimental::ValueType::kokkos_value_double; + static double get( + const Kokkos::Tools::Experimental::VariableValue& value_struct) { + return value_struct.value.double_value; + } +}; +template <> +struct tuning_type_for { + static constexpr Kokkos::Tools::Experimental::ValueType value = + Kokkos::Tools::Experimental::ValueType::kokkos_value_int64; + static int64_t get( + const Kokkos::Tools::Experimental::VariableValue& value_struct) { + return value_struct.value.int_value; + } +}; +} // namespace Impl +template +class SingleDimensionalRangeTuner { + size_t id; + size_t context; + using tuning_util = Impl::tuning_type_for; + + Bound default_value; + + public: + SingleDimensionalRangeTuner() = default; + SingleDimensionalRangeTuner( + const std::string& name, + Kokkos::Tools::Experimental::StatisticalCategory category, + Bound default_val, Bound lower, Bound upper, Bound step = (Bound)0) { + default_value = default_val; + Kokkos::Tools::Experimental::VariableInfo info; + info.category = category; + info.candidates = make_candidate_range( + static_cast(lower), static_cast(upper), + static_cast(step), false, false); + info.valueQuantity = + Kokkos::Tools::Experimental::CandidateValueType::kokkos_value_range; + info.type = tuning_util::value; + id = Kokkos::Tools::Experimental::declare_output_type(name, info); + } + + Bound begin() { + context = Kokkos::Tools::Experimental::get_new_context_id(); + Kokkos::Tools::Experimental::begin_context(context); + auto tuned_value = + Kokkos::Tools::Experimental::make_variable_value(id, default_value); + Kokkos::Tools::Experimental::request_output_values(context, 1, + &tuned_value); + return tuning_util::get(tuned_value); + } + + void end() { Kokkos::Tools::Experimental::end_context(context); } + + template + void with_tuned_value(Functor& func) { + func(begin()); + end(); + } +}; + +class RangePolicyOccupancyTuner { + private: + using TunerType = SingleDimensionalRangeTuner; + TunerType tuner; + + public: + RangePolicyOccupancyTuner() = default; + template + RangePolicyOccupancyTuner(const std::string& name, + const Kokkos::RangePolicy&, + const Functor&, const TagType&, + ViableConfigurationCalculator) + : tuner(TunerType(name, + Kokkos::Tools::Experimental::StatisticalCategory:: + kokkos_value_ratio, + 100, 5, 100, 5)) {} + + template + auto tune(const Kokkos::RangePolicy& policy_in) { + Kokkos::RangePolicy policy(policy_in); + if (Kokkos::Tools::Experimental::have_tuning_tool()) { + auto occupancy = tuner.begin(); + policy.impl_set_desired_occupancy( + Kokkos::Experimental::DesiredOccupancy{static_cast(occupancy)}); + } + return policy; } void end() { if (Kokkos::Tools::Experimental::have_tuning_tool()) { @@ -578,11 +687,13 @@ struct MDRangeTuner : public ExtendableTunerMixin> { policy.impl_change_tile_size({std::get(tuple)...}); } template - void tune(Kokkos::MDRangePolicy& policy) { + auto tune(const Kokkos::MDRangePolicy& policy_in) { + Kokkos::MDRangePolicy policy(policy_in); if (Kokkos::Tools::Experimental::have_tuning_tool()) { auto configuration = tuner.begin(); set_policy_tile(policy, configuration, std::make_index_sequence{}); } + return policy; } void end() { if (Kokkos::Tools::Experimental::have_tuning_tool()) { diff --git a/lib/kokkos/core/src/Kokkos_TypeInfo.hpp b/lib/kokkos/core/src/Kokkos_TypeInfo.hpp new file mode 100644 index 0000000000..e5710da2e3 --- /dev/null +++ b/lib/kokkos/core/src/Kokkos_TypeInfo.hpp @@ -0,0 +1,103 @@ +//@HEADER +// ************************************************************************ +// +// Kokkos v. 4.0 +// Copyright (2022) National Technology & Engineering +// Solutions of Sandia, LLC (NTESS). +// +// Under the terms of Contract DE-NA0003525 with NTESS, +// the U.S. Government retains certain rights in this software. +// +// Part of Kokkos, under the Apache License v2.0 with LLVM Exceptions. +// See https://kokkos.org/LICENSE for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//@HEADER + +#ifndef KOKKOS_TYPE_INFO_HPP +#define KOKKOS_TYPE_INFO_HPP + +#include +#include +#include + +#include + +// Intel C++ Compiler Classic version 2021.2.0 works but 2021.1.2 doesn't +// Both have __INTEL_COMPILER defined to 2021 so using +// __INTEL_COMPILER_BUILD_DATE to discriminate. +// Experimenting on the compiler explorer gave +// icc version | __INTEL_COMPILER | __INTEL_COMPILER_BUILD_DATE +// 2021.1.2 | 2021 | 20201208 +// 2021.2.0 | 2021 | 20210228 +// NVCC versions less than 11.3.0 segfault when that header is included +// NVCC+MSVC doesn't work at all - it simply reports "T" inside type_name +#if (!defined(KOKKOS_COMPILER_INTEL) || \ + (__INTEL_COMPILER_BUILD_DATE >= 20210228)) && \ + (!defined(KOKKOS_COMPILER_NVCC) || (KOKKOS_COMPILER_NVCC >= 1130)) && \ + (!(defined(KOKKOS_COMPILER_NVCC) && defined(KOKKOS_COMPILER_MSVC))) + +#define KOKKOS_ENABLE_IMPL_TYPEINFO + +namespace Kokkos::Impl { + +template +constexpr std::array to_array(std::string_view src) { + std::array dst{}; + for (size_t i = 0; i < N; ++i) { + dst[i] = src[i]; + } + return dst; +} + +template +constexpr auto type_name() { +#if defined(__clang__) + constexpr std::string_view func = __PRETTY_FUNCTION__; + constexpr std::string_view prefix{"[T = "}; + constexpr std::string_view suffix{"]"}; +#elif defined(__GNUC__) + constexpr std::string_view func = __PRETTY_FUNCTION__; + constexpr std::string_view prefix{"[with T = "}; + constexpr std::string_view suffix{"]"}; +#elif defined(_MSC_VER) + constexpr std::string_view func = __FUNCSIG__; + constexpr std::string_view prefix{"type_name<"}; + constexpr std::string_view suffix{">(void)"}; +#else +#error bug +#endif + constexpr auto beg = func.find(prefix) + prefix.size(); + constexpr auto end = func.rfind(suffix); + static_assert(beg != std::string_view::npos); + static_assert(end != std::string_view::npos); + return to_array(func.substr(beg, end)); +} + +template +class TypeInfo { + static constexpr auto value_ = type_name(); + + public: + static constexpr std::string_view name() noexcept { + return {value_.data(), value_.size()}; + } +}; + +} // namespace Kokkos::Impl + +#else // out of luck, using Intel C++ Compiler Classic + +namespace Kokkos::Impl { + +template +class TypeInfo { + public: + static constexpr std::string_view name() noexcept { return "not supported"; } +}; + +} // namespace Kokkos::Impl + +#endif + +#endif diff --git a/lib/kokkos/core/src/Kokkos_View.hpp b/lib/kokkos/core/src/Kokkos_View.hpp index 04d1fcf151..d5b352876c 100644 --- a/lib/kokkos/core/src/Kokkos_View.hpp +++ b/lib/kokkos/core/src/Kokkos_View.hpp @@ -22,2016 +22,10 @@ static_assert(false, #ifndef KOKKOS_VIEW_HPP #define KOKKOS_VIEW_HPP -#include -#include -#include -#include - -#include -#include -#include -#include -#include - -#include -#include - -#ifdef KOKKOS_ENABLE_IMPL_MDSPAN -#include -#include -#include -#endif -#include - -//---------------------------------------------------------------------------- -//---------------------------------------------------------------------------- - -namespace Kokkos { -namespace Impl { - -template -struct ViewArrayAnalysis; - -template ::non_const_value_type> -struct ViewDataAnalysis; - -template -class ViewMapping { - public: - enum : bool { is_assignable_data_type = false }; - enum : bool { is_assignable = false }; -}; - -template -constexpr KOKKOS_INLINE_FUNCTION std::size_t count_valid_integers( - const IntType i0, const IntType i1, const IntType i2, const IntType i3, - const IntType i4, const IntType i5, const IntType i6, const IntType i7) { - static_assert(std::is_integral::value, - "count_valid_integers() must have integer arguments."); - - return (i0 != KOKKOS_INVALID_INDEX) + (i1 != KOKKOS_INVALID_INDEX) + - (i2 != KOKKOS_INVALID_INDEX) + (i3 != KOKKOS_INVALID_INDEX) + - (i4 != KOKKOS_INVALID_INDEX) + (i5 != KOKKOS_INVALID_INDEX) + - (i6 != KOKKOS_INVALID_INDEX) + (i7 != KOKKOS_INVALID_INDEX); -} - -// FIXME Ideally, we would not instantiate this function for every possible View -// type. We should be able to only pass "extent" when we use mdspan. -template -KOKKOS_INLINE_FUNCTION void runtime_check_rank( - const View&, const bool is_void_spec, const size_t i0, const size_t i1, - const size_t i2, const size_t i3, const size_t i4, const size_t i5, - const size_t i6, const size_t i7, const char* label) { - (void)(label); - - if (is_void_spec) { - const size_t num_passed_args = - count_valid_integers(i0, i1, i2, i3, i4, i5, i6, i7); - // We either allow to pass as many extents as the dynamic rank is, or - // as many extents as the total rank is. In the latter case, the given - // extents for the static dimensions must match the - // compile-time extents. - constexpr int rank = View::rank(); - constexpr int dyn_rank = View::rank_dynamic(); - const bool n_args_is_dyn_rank = num_passed_args == dyn_rank; - const bool n_args_is_rank = num_passed_args == rank; - - if constexpr (rank != dyn_rank) { - if (n_args_is_rank) { - size_t new_extents[8] = {i0, i1, i2, i3, i4, i5, i6, i7}; - for (int i = dyn_rank; i < rank; ++i) - if (new_extents[i] != View::static_extent(i)) { - KOKKOS_IF_ON_HOST( - const std::string message = - "The specified run-time extent for Kokkos::View '" + - std::string(label) + - "' does not match the compile-time extent in dimension " + - std::to_string(i) + ". The given extent is " + - std::to_string(new_extents[i]) + " but should be " + - std::to_string(View::static_extent(i)) + ".\n"; - Kokkos::abort(message.c_str());) - KOKKOS_IF_ON_DEVICE( - Kokkos::abort( - "The specified run-time extents for a Kokkos::View " - "do not match the compile-time extents.");) - } - } - } - - if (!n_args_is_dyn_rank && !n_args_is_rank) { - KOKKOS_IF_ON_HOST( - const std::string message = - "Constructor for Kokkos::View '" + std::string(label) + - "' has mismatched number of arguments. The number " - "of arguments = " + - std::to_string(num_passed_args) + - " neither matches the dynamic rank = " + - std::to_string(dyn_rank) + - " nor the total rank = " + std::to_string(rank) + "\n"; - Kokkos::abort(message.c_str());) - KOKKOS_IF_ON_DEVICE(Kokkos::abort("Constructor for Kokkos View has " - "mismatched number of arguments.");) - } - } -} - -} /* namespace Impl */ -} /* namespace Kokkos */ - -// Class to provide a uniform type -namespace Kokkos { -namespace Impl { -template -struct ViewUniformType; -} -} // namespace Kokkos - -//---------------------------------------------------------------------------- -//---------------------------------------------------------------------------- - -namespace Kokkos { - -/** \class ViewTraits - * \brief Traits class for accessing attributes of a View. - * - * This is an implementation detail of View. It is only of interest - * to developers implementing a new specialization of View. - * - * Template argument options: - * - View< DataType > - * - View< DataType , Space > - * - View< DataType , Space , MemoryTraits > - * - View< DataType , ArrayLayout > - * - View< DataType , ArrayLayout , Space > - * - View< DataType , ArrayLayout , MemoryTraits > - * - View< DataType , ArrayLayout , Space , MemoryTraits > - * - View< DataType , MemoryTraits > - */ - -template -struct ViewTraits; - -template <> -struct ViewTraits { - using execution_space = void; - using memory_space = void; - using HostMirrorSpace = void; - using array_layout = void; - using memory_traits = void; - using specialize = void; - using hooks_policy = void; -}; - -template -struct ViewTraits { - // Ignore an extraneous 'void' - using execution_space = typename ViewTraits::execution_space; - using memory_space = typename ViewTraits::memory_space; - using HostMirrorSpace = typename ViewTraits::HostMirrorSpace; - using array_layout = typename ViewTraits::array_layout; - using memory_traits = typename ViewTraits::memory_traits; - using specialize = typename ViewTraits::specialize; - using hooks_policy = typename ViewTraits::hooks_policy; -}; - -template -struct ViewTraits< - std::enable_if_t::value>, - HooksPolicy, Prop...> { - using execution_space = typename ViewTraits::execution_space; - using memory_space = typename ViewTraits::memory_space; - using HostMirrorSpace = typename ViewTraits::HostMirrorSpace; - using array_layout = typename ViewTraits::array_layout; - using memory_traits = typename ViewTraits::memory_traits; - using specialize = typename ViewTraits::specialize; - using hooks_policy = HooksPolicy; -}; - -template -struct ViewTraits::value>, - ArrayLayout, Prop...> { - // Specify layout, keep subsequent space and memory traits arguments - - using execution_space = typename ViewTraits::execution_space; - using memory_space = typename ViewTraits::memory_space; - using HostMirrorSpace = typename ViewTraits::HostMirrorSpace; - using array_layout = ArrayLayout; - using memory_traits = typename ViewTraits::memory_traits; - using specialize = typename ViewTraits::specialize; - using hooks_policy = typename ViewTraits::hooks_policy; -}; - -template -struct ViewTraits::value>, Space, - Prop...> { - // Specify Space, memory traits should be the only subsequent argument. - - static_assert( - std::is_same::execution_space, - void>::value && - std::is_same::memory_space, - void>::value && - std::is_same::HostMirrorSpace, - void>::value && - std::is_same::array_layout, - void>::value, - "Only one View Execution or Memory Space template argument"); - - using execution_space = typename Space::execution_space; - using memory_space = typename Space::memory_space; - using HostMirrorSpace = - typename Kokkos::Impl::HostMirror::Space::memory_space; - using array_layout = typename execution_space::array_layout; - using memory_traits = typename ViewTraits::memory_traits; - using specialize = typename ViewTraits::specialize; - using hooks_policy = typename ViewTraits::hooks_policy; -}; - -template -struct ViewTraits< - std::enable_if_t::value>, - MemoryTraits, Prop...> { - // Specify memory trait, should not be any subsequent arguments - - static_assert( - std::is_same::execution_space, - void>::value && - std::is_same::memory_space, - void>::value && - std::is_same::array_layout, - void>::value && - std::is_same::memory_traits, - void>::value && - std::is_same::hooks_policy, - void>::value, - "MemoryTrait is the final optional template argument for a View"); - - using execution_space = void; - using memory_space = void; - using HostMirrorSpace = void; - using array_layout = void; - using memory_traits = MemoryTraits; - using specialize = void; - using hooks_policy = void; -}; - -template -struct ViewTraits { - private: - // Unpack the properties arguments - using prop = ViewTraits; - - using ExecutionSpace = - std::conditional_t::value, - typename prop::execution_space, - Kokkos::DefaultExecutionSpace>; - - using MemorySpace = - std::conditional_t::value, - typename prop::memory_space, - typename ExecutionSpace::memory_space>; - - using ArrayLayout = - std::conditional_t::value, - typename prop::array_layout, - typename ExecutionSpace::array_layout>; - - using HostMirrorSpace = std::conditional_t< - !std::is_void::value, - typename prop::HostMirrorSpace, - typename Kokkos::Impl::HostMirror::Space>; - - using MemoryTraits = - std::conditional_t::value, - typename prop::memory_traits, - typename Kokkos::MemoryManaged>; - - using HooksPolicy = - std::conditional_t::value, - typename prop::hooks_policy, - Kokkos::Experimental::DefaultViewHooks>; - - // Analyze data type's properties, - // May be specialized based upon the layout and value type - using data_analysis = Kokkos::Impl::ViewDataAnalysis; - - public: - //------------------------------------ - // Data type traits: - - using data_type = typename data_analysis::type; - using const_data_type = typename data_analysis::const_type; - using non_const_data_type = typename data_analysis::non_const_type; - - //------------------------------------ - // Compatible array of trivial type traits: - - using scalar_array_type = typename data_analysis::scalar_array_type; - using const_scalar_array_type = - typename data_analysis::const_scalar_array_type; - using non_const_scalar_array_type = - typename data_analysis::non_const_scalar_array_type; - - //------------------------------------ - // Value type traits: - - using value_type = typename data_analysis::value_type; - using const_value_type = typename data_analysis::const_value_type; - using non_const_value_type = typename data_analysis::non_const_value_type; - - //------------------------------------ - // Mapping traits: - - using array_layout = ArrayLayout; - using dimension = typename data_analysis::dimension; - - using specialize = std::conditional_t< - std::is_void::value, - typename prop::specialize, - typename data_analysis::specialize>; /* mapping specialization tag */ - - static constexpr unsigned rank = dimension::rank; - static constexpr unsigned rank_dynamic = dimension::rank_dynamic; - - //------------------------------------ - // Execution space, memory space, memory access traits, and host mirror space. - - using execution_space = ExecutionSpace; - using memory_space = MemorySpace; - using device_type = Kokkos::Device; - using memory_traits = MemoryTraits; - using host_mirror_space = HostMirrorSpace; - using hooks_policy = HooksPolicy; - - using size_type = typename MemorySpace::size_type; - - enum { is_hostspace = std::is_same::value }; - enum { is_managed = MemoryTraits::is_unmanaged == 0 }; - enum { is_random_access = MemoryTraits::is_random_access == 1 }; - - //------------------------------------ -}; - -#ifdef KOKKOS_ENABLE_IMPL_MDSPAN -namespace Impl { -struct UnsupportedKokkosArrayLayout; - -template -struct MDSpanViewTraits { - using mdspan_type = UnsupportedKokkosArrayLayout; -}; - -// "Natural" mdspan for a view if the View's ArrayLayout is supported. -template -struct MDSpanViewTraits::type>> { - using index_type = std::size_t; - using extents_type = - typename Impl::ExtentsFromDataType::type; - using mdspan_layout_type = - typename Impl::LayoutFromArrayLayout::type; - using accessor_type = Impl::SpaceAwareAccessor< - typename Traits::memory_space, - Kokkos::default_accessor>; - using mdspan_type = mdspan; -}; -} // namespace Impl -#endif // KOKKOS_ENABLE_IMPL_MDSPAN - -/** \class View - * \brief View to an array of data. - * - * A View represents an array of one or more dimensions. - * For details, please refer to Kokkos' tutorial materials. - * - * \section Kokkos_View_TemplateParameters Template parameters - * - * This class has both required and optional template parameters. The - * \c DataType parameter must always be provided, and must always be - * first. The parameters \c Arg1Type, \c Arg2Type, and \c Arg3Type are - * placeholders for different template parameters. The default value - * of the fifth template parameter \c Specialize suffices for most use - * cases. When explaining the template parameters, we won't refer to - * \c Arg1Type, \c Arg2Type, and \c Arg3Type; instead, we will refer - * to the valid categories of template parameters, in whatever order - * they may occur. - * - * Valid ways in which template arguments may be specified: - * - View< DataType > - * - View< DataType , Layout > - * - View< DataType , Layout , Space > - * - View< DataType , Layout , Space , MemoryTraits > - * - View< DataType , Space > - * - View< DataType , Space , MemoryTraits > - * - View< DataType , MemoryTraits > - * - * \tparam DataType (required) This indicates both the type of each - * entry of the array, and the combination of compile-time and - * run-time array dimension(s). For example, double* - * indicates a one-dimensional array of \c double with run-time - * dimension, and int*[3] a two-dimensional array of \c int - * with run-time first dimension and compile-time second dimension - * (of 3). In general, the run-time dimensions (if any) must go - * first, followed by zero or more compile-time dimensions. For - * more examples, please refer to the tutorial materials. - * - * \tparam Space (required) The memory space. - * - * \tparam Layout (optional) The array's layout in memory. For - * example, LayoutLeft indicates a column-major (Fortran style) - * layout, and LayoutRight a row-major (C style) layout. If not - * specified, this defaults to the preferred layout for the - * Space. - * - * \tparam MemoryTraits (optional) Assertion of the user's intended - * access behavior. For example, RandomAccess indicates read-only - * access with limited spatial locality, and Unmanaged lets users - * wrap externally allocated memory in a View without automatic - * deallocation. - * - * \section Kokkos_View_MT MemoryTraits discussion - * - * \subsection Kokkos_View_MT_Interp MemoryTraits interpretation depends on - * Space - * - * Some \c MemoryTraits options may have different interpretations for - * different \c Space types. For example, with the Cuda device, - * \c RandomAccess tells Kokkos to fetch the data through the texture - * cache, whereas the non-GPU devices have no such hardware construct. - * - * \subsection Kokkos_View_MT_PrefUse Preferred use of MemoryTraits - * - * Users should defer applying the optional \c MemoryTraits parameter - * until the point at which they actually plan to rely on it in a - * computational kernel. This minimizes the number of template - * parameters exposed in their code, which reduces the cost of - * compilation. Users may always assign a View without specified - * \c MemoryTraits to a compatible View with that specification. - * For example: - * \code - * // Pass in the simplest types of View possible. - * void - * doSomething (View out, - * View in) - * { - * // Assign the "generic" View in to a RandomAccess View in_rr. - * // Note that RandomAccess View objects must have const data. - * View in_rr = in; - * // ... do something with in_rr and out ... - * } - * \endcode - */ - -} // namespace Kokkos - -namespace Kokkos { - -template -struct is_always_assignable_impl; - -template -struct is_always_assignable_impl, - Kokkos::View> { - using mapping_type = Kokkos::Impl::ViewMapping< - typename Kokkos::View::traits, - typename Kokkos::View::traits, - typename Kokkos::View::traits::specialize>; - - constexpr static bool value = - mapping_type::is_assignable && - static_cast(Kokkos::View::rank_dynamic) >= - static_cast(Kokkos::View::rank_dynamic); -}; - -template -using is_always_assignable = is_always_assignable_impl< - std::remove_reference_t, - std::remove_const_t>>; - -template -inline constexpr bool is_always_assignable_v = - is_always_assignable::value; - -template -constexpr bool is_assignable(const Kokkos::View& dst, - const Kokkos::View& src) { - using DstTraits = typename Kokkos::View::traits; - using SrcTraits = typename Kokkos::View::traits; - using mapping_type = - Kokkos::Impl::ViewMapping; - - return is_always_assignable_v, - Kokkos::View> || - (mapping_type::is_assignable && - ((DstTraits::dimension::rank_dynamic >= 1) || - (dst.static_extent(0) == src.extent(0))) && - ((DstTraits::dimension::rank_dynamic >= 2) || - (dst.static_extent(1) == src.extent(1))) && - ((DstTraits::dimension::rank_dynamic >= 3) || - (dst.static_extent(2) == src.extent(2))) && - ((DstTraits::dimension::rank_dynamic >= 4) || - (dst.static_extent(3) == src.extent(3))) && - ((DstTraits::dimension::rank_dynamic >= 5) || - (dst.static_extent(4) == src.extent(4))) && - ((DstTraits::dimension::rank_dynamic >= 6) || - (dst.static_extent(5) == src.extent(5))) && - ((DstTraits::dimension::rank_dynamic >= 7) || - (dst.static_extent(6) == src.extent(6))) && - ((DstTraits::dimension::rank_dynamic >= 8) || - (dst.static_extent(7) == src.extent(7)))); -} - -} /* namespace Kokkos */ - -//---------------------------------------------------------------------------- -//---------------------------------------------------------------------------- - -#include - -//---------------------------------------------------------------------------- -//---------------------------------------------------------------------------- - -namespace Kokkos { - -// FIXME_OPENMPTARGET - The `declare target` is needed for the Intel GPUs with -// the OpenMPTarget backend -#if defined(KOKKOS_ENABLE_OPENMPTARGET) && defined(KOKKOS_COMPILER_INTEL_LLVM) -#pragma omp declare target +#if defined(KOKKOS_ENABLE_IMPL_MDSPAN) && !defined(KOKKOS_COMPILER_INTEL) +#include #endif -inline constexpr Kokkos::ALL_t ALL{}; +#include -#if defined(KOKKOS_ENABLE_OPENMPTARGET) && defined(KOKKOS_COMPILER_INTEL_LLVM) -#pragma omp end declare target -#endif - -inline constexpr Kokkos::Impl::SequentialHostInit_t SequentialHostInit{}; - -inline constexpr Kokkos::Impl::WithoutInitializing_t WithoutInitializing{}; - -inline constexpr Kokkos::Impl::AllowPadding_t AllowPadding{}; - -/** \brief Create View allocation parameter bundle from argument list. - * - * Valid argument list members are: - * 1) label as a "string" or std::string - * 2) memory space instance of the View::memory_space type - * 3) execution space instance compatible with the View::memory_space - * 4) Kokkos::WithoutInitializing to bypass initialization - * 4) Kokkos::AllowPadding to allow allocation to pad dimensions for memory - * alignment - */ -template -inline Impl::ViewCtorProp::type...> -view_alloc(Args const&... args) { - using return_type = - Impl::ViewCtorProp::type...>; - - static_assert(!return_type::has_pointer, - "Cannot give pointer-to-memory for view allocation"); - - return return_type(args...); -} - -template -KOKKOS_INLINE_FUNCTION - Impl::ViewCtorProp::type...> - view_wrap(Args const&... args) { - using return_type = - Impl::ViewCtorProp::type...>; - - static_assert(!return_type::has_memory_space && - !return_type::has_execution_space && - !return_type::has_label && return_type::has_pointer, - "Must only give pointer-to-memory for view wrapping"); - - return return_type(args...); -} - -} /* namespace Kokkos */ - -//---------------------------------------------------------------------------- -//---------------------------------------------------------------------------- - -namespace Kokkos { - -template -class View; - -template -struct is_view : public std::false_type {}; - -template -struct is_view> : public std::true_type {}; - -template -struct is_view> : public std::true_type {}; - -template -inline constexpr bool is_view_v = is_view::value; - -template -class View : public ViewTraits { - private: - template - friend class View; - template - friend class Kokkos::Impl::ViewMapping; - - using view_tracker_type = Kokkos::Impl::ViewTracker; - - public: - using traits = ViewTraits; - - private: - using map_type = - Kokkos::Impl::ViewMapping; - template - friend struct Kokkos::Impl::ViewTracker; - using hooks_policy = typename traits::hooks_policy; - - view_tracker_type m_track; - map_type m_map; - - public: - //---------------------------------------- - /** \brief Compatible view of array of scalar types */ - using array_type = - View; - - /** \brief Compatible view of const data type */ - using const_type = - View; - - /** \brief Compatible view of non-const data type */ - using non_const_type = - View; - - /** \brief Compatible HostMirror view */ - using HostMirror = - View, - typename traits::hooks_policy>; - - /** \brief Compatible HostMirror view */ - using host_mirror_type = - View; - - /** \brief Unified types */ - using uniform_type = typename Impl::ViewUniformType::type; - using uniform_const_type = - typename Impl::ViewUniformType::const_type; - using uniform_runtime_type = - typename Impl::ViewUniformType::runtime_type; - using uniform_runtime_const_type = - typename Impl::ViewUniformType::runtime_const_type; - using uniform_nomemspace_type = - typename Impl::ViewUniformType::nomemspace_type; - using uniform_const_nomemspace_type = - typename Impl::ViewUniformType::const_nomemspace_type; - using uniform_runtime_nomemspace_type = - typename Impl::ViewUniformType::runtime_nomemspace_type; - using uniform_runtime_const_nomemspace_type = - typename Impl::ViewUniformType::runtime_const_nomemspace_type; - - //---------------------------------------- - // Domain rank and extents - - static constexpr Impl::integral_constant - rank = {}; - static constexpr Impl::integral_constant - rank_dynamic = {}; -#ifdef KOKKOS_ENABLE_DEPRECATED_CODE_4 - enum {Rank KOKKOS_DEPRECATED_WITH_COMMENT("Use rank instead.") = - map_type::Rank}; -#endif - - template - KOKKOS_INLINE_FUNCTION constexpr std::enable_if_t< - std::is_integral::value, size_t> - extent(const iType& r) const noexcept { - return m_map.extent(r); - } - - static KOKKOS_INLINE_FUNCTION constexpr size_t static_extent( - const unsigned r) noexcept { - return map_type::static_extent(r); - } - - template - KOKKOS_INLINE_FUNCTION constexpr std::enable_if_t< - std::is_integral::value, int> - extent_int(const iType& r) const noexcept { - return static_cast(m_map.extent(r)); - } - - KOKKOS_INLINE_FUNCTION constexpr typename traits::array_layout layout() - const { - return m_map.layout(); - } - - //---------------------------------------- - /* Deprecate all 'dimension' functions in favor of - * ISO/C++ vocabulary 'extent'. - */ - - KOKKOS_INLINE_FUNCTION constexpr size_t size() const { - return m_map.dimension_0() * m_map.dimension_1() * m_map.dimension_2() * - m_map.dimension_3() * m_map.dimension_4() * m_map.dimension_5() * - m_map.dimension_6() * m_map.dimension_7(); - } - - KOKKOS_INLINE_FUNCTION constexpr size_t stride_0() const { - return m_map.stride_0(); - } - KOKKOS_INLINE_FUNCTION constexpr size_t stride_1() const { - return m_map.stride_1(); - } - KOKKOS_INLINE_FUNCTION constexpr size_t stride_2() const { - return m_map.stride_2(); - } - KOKKOS_INLINE_FUNCTION constexpr size_t stride_3() const { - return m_map.stride_3(); - } - KOKKOS_INLINE_FUNCTION constexpr size_t stride_4() const { - return m_map.stride_4(); - } - KOKKOS_INLINE_FUNCTION constexpr size_t stride_5() const { - return m_map.stride_5(); - } - KOKKOS_INLINE_FUNCTION constexpr size_t stride_6() const { - return m_map.stride_6(); - } - KOKKOS_INLINE_FUNCTION constexpr size_t stride_7() const { - return m_map.stride_7(); - } - - template - KOKKOS_INLINE_FUNCTION constexpr std::enable_if_t< - std::is_integral::value, size_t> - stride(iType r) const { - return ( - r == 0 - ? m_map.stride_0() - : (r == 1 - ? m_map.stride_1() - : (r == 2 - ? m_map.stride_2() - : (r == 3 - ? m_map.stride_3() - : (r == 4 - ? m_map.stride_4() - : (r == 5 - ? m_map.stride_5() - : (r == 6 - ? m_map.stride_6() - : m_map.stride_7()))))))); - } - - template - KOKKOS_INLINE_FUNCTION void stride(iType* const s) const { - m_map.stride(s); - } - - //---------------------------------------- - // Range span is the span which contains all members. - - using reference_type = typename map_type::reference_type; - using pointer_type = typename map_type::pointer_type; - - enum { - reference_type_is_lvalue_reference = - std::is_lvalue_reference::value - }; - - KOKKOS_INLINE_FUNCTION constexpr size_t span() const { return m_map.span(); } - KOKKOS_INLINE_FUNCTION bool span_is_contiguous() const { - return m_map.span_is_contiguous(); - } - KOKKOS_INLINE_FUNCTION constexpr bool is_allocated() const { - return m_map.data() != nullptr; - } - KOKKOS_INLINE_FUNCTION constexpr pointer_type data() const { - return m_map.data(); - } - - //---------------------------------------- - // Allow specializations to query their specialized map - - KOKKOS_INLINE_FUNCTION - const Kokkos::Impl::ViewMapping& - impl_map() const { - return m_map; - } - KOKKOS_INLINE_FUNCTION - const Kokkos::Impl::SharedAllocationTracker& impl_track() const { - return m_track.m_tracker; - } - //---------------------------------------- - - private: - static constexpr bool is_layout_left = - std::is_same::value; - - static constexpr bool is_layout_right = - std::is_same::value; - - static constexpr bool is_layout_stride = - std::is_same::value; - - static constexpr bool is_default_map = - std::is_void::value && - (is_layout_left || is_layout_right || is_layout_stride); - -#if defined(KOKKOS_ENABLE_DEBUG_BOUNDS_CHECK) - -#define KOKKOS_IMPL_VIEW_OPERATOR_VERIFY(...) \ - Kokkos::Impl::runtime_check_memory_access_violation< \ - typename traits::memory_space>( \ - "Kokkos::View ERROR: attempt to access inaccessible memory space", \ - __VA_ARGS__); \ - Kokkos::Impl::view_verify_operator_bounds( \ - __VA_ARGS__); - -#else - -#define KOKKOS_IMPL_VIEW_OPERATOR_VERIFY(...) \ - Kokkos::Impl::runtime_check_memory_access_violation< \ - typename traits::memory_space>( \ - "Kokkos::View ERROR: attempt to access inaccessible memory space", \ - __VA_ARGS__); - -#endif - - template - static KOKKOS_FUNCTION void check_access_member_function_valid_args(Is...) { - static_assert(rank <= sizeof...(Is)); - static_assert(sizeof...(Is) <= 8); - static_assert(Kokkos::Impl::are_integral::value); - } - - template - static KOKKOS_FUNCTION void check_operator_parens_valid_args(Is...) { - static_assert(rank == sizeof...(Is)); - static_assert(Kokkos::Impl::are_integral::value); - } - - public: - //------------------------------ - // Rank 1 default map operator() - - template - KOKKOS_FORCEINLINE_FUNCTION - std::enable_if_t<(Kokkos::Impl::always_true::value && // - (1 == rank) && is_default_map && !is_layout_stride), - reference_type> - operator()(I0 i0) const { - check_operator_parens_valid_args(i0); - KOKKOS_IMPL_VIEW_OPERATOR_VERIFY(m_track, m_map, i0) - return m_map.m_impl_handle[i0]; - } - - template - KOKKOS_FORCEINLINE_FUNCTION - std::enable_if_t<(Kokkos::Impl::always_true::value && // - (1 == rank) && is_default_map && is_layout_stride), - reference_type> - operator()(I0 i0) const { - check_operator_parens_valid_args(i0); - KOKKOS_IMPL_VIEW_OPERATOR_VERIFY(m_track, m_map, i0) - return m_map.m_impl_handle[m_map.m_impl_offset.m_stride.S0 * i0]; - } - - //------------------------------ - // Rank 1 operator[] - - template - KOKKOS_FORCEINLINE_FUNCTION std::enable_if_t< - ((1 == rank) && Kokkos::Impl::are_integral::value && !is_default_map), - reference_type> - operator[](I0 i0) const { - KOKKOS_IMPL_VIEW_OPERATOR_VERIFY(m_track, m_map, i0) - return m_map.reference(i0); - } - - template - KOKKOS_FORCEINLINE_FUNCTION - std::enable_if_t<((1 == rank) && Kokkos::Impl::are_integral::value && - is_default_map && !is_layout_stride), - reference_type> - operator[](I0 i0) const { - KOKKOS_IMPL_VIEW_OPERATOR_VERIFY(m_track, m_map, i0) - return m_map.m_impl_handle[i0]; - } - - template - KOKKOS_FORCEINLINE_FUNCTION - std::enable_if_t<((1 == rank) && Kokkos::Impl::are_integral::value && - is_default_map && is_layout_stride), - reference_type> - operator[](I0 i0) const { - KOKKOS_IMPL_VIEW_OPERATOR_VERIFY(m_track, m_map, i0) - return m_map.m_impl_handle[m_map.m_impl_offset.m_stride.S0 * i0]; - } - - //------------------------------ - // Rank 2 default map operator() - - template - KOKKOS_FORCEINLINE_FUNCTION std::enable_if_t< - (Kokkos::Impl::always_true::value && // - (2 == rank) && is_default_map && - (is_layout_left || is_layout_right || is_layout_stride)), - reference_type> - operator()(I0 i0, I1 i1) const { - check_operator_parens_valid_args(i0, i1); - KOKKOS_IMPL_VIEW_OPERATOR_VERIFY(m_track, m_map, i0, i1) - if constexpr (is_layout_left) { - if constexpr (rank_dynamic == 0) - return m_map.m_impl_handle[i0 + m_map.m_impl_offset.m_dim.N0 * i1]; - else - return m_map.m_impl_handle[i0 + m_map.m_impl_offset.m_stride * i1]; - } else if constexpr (is_layout_right) { - if constexpr (rank_dynamic == 0) - return m_map.m_impl_handle[i1 + m_map.m_impl_offset.m_dim.N1 * i0]; - else - return m_map.m_impl_handle[i1 + m_map.m_impl_offset.m_stride * i0]; - } else { - static_assert(is_layout_stride); - return m_map.m_impl_handle[i0 * m_map.m_impl_offset.m_stride.S0 + - i1 * m_map.m_impl_offset.m_stride.S1]; - } -#if defined KOKKOS_COMPILER_INTEL - __builtin_unreachable(); -#endif - } - - // Rank 0 -> 8 operator() except for rank-1 and rank-2 with default map which - // have "inlined" versions above - - template - KOKKOS_FORCEINLINE_FUNCTION std::enable_if_t< - (Kokkos::Impl::always_true::value && // - (2 != rank) && (1 != rank) && (0 != rank) && is_default_map), - reference_type> - operator()(Is... indices) const { - check_operator_parens_valid_args(indices...); - KOKKOS_IMPL_VIEW_OPERATOR_VERIFY(m_track, m_map, indices...) - return m_map.m_impl_handle[m_map.m_impl_offset(indices...)]; - } - - template - KOKKOS_FORCEINLINE_FUNCTION - std::enable_if_t<(Kokkos::Impl::always_true::value && // - ((0 == rank) || !is_default_map)), - reference_type> - operator()(Is... indices) const { - check_operator_parens_valid_args(indices...); - KOKKOS_IMPL_VIEW_OPERATOR_VERIFY(m_track, m_map, indices...) - return m_map.reference(indices...); - } - - //------------------------------ - // Rank 0 - - template - KOKKOS_FORCEINLINE_FUNCTION std::enable_if_t< - (Kokkos::Impl::always_true::value && (0 == rank)), reference_type> - access(Is... extra) const { - check_access_member_function_valid_args(extra...); - KOKKOS_IMPL_VIEW_OPERATOR_VERIFY(m_track, m_map, extra...) - return m_map.reference(); - } - - //------------------------------ - // Rank 1 - - template - KOKKOS_FORCEINLINE_FUNCTION - std::enable_if_t<(Kokkos::Impl::always_true::value && - (1 == rank) && !is_default_map), - reference_type> - access(I0 i0, Is... extra) const { - check_access_member_function_valid_args(i0, extra...); - KOKKOS_IMPL_VIEW_OPERATOR_VERIFY(m_track, m_map, i0, extra...) - return m_map.reference(i0); - } - - template - KOKKOS_FORCEINLINE_FUNCTION - std::enable_if_t<(Kokkos::Impl::always_true::value && - (1 == rank) && is_default_map && !is_layout_stride), - reference_type> - access(I0 i0, Is... extra) const { - check_access_member_function_valid_args(i0, extra...); - KOKKOS_IMPL_VIEW_OPERATOR_VERIFY(m_track, m_map, i0, extra...) - return m_map.m_impl_handle[i0]; - } - - template - KOKKOS_FORCEINLINE_FUNCTION - std::enable_if_t<(Kokkos::Impl::always_true::value && - (1 == rank) && is_default_map && is_layout_stride), - reference_type> - access(I0 i0, Is... extra) const { - check_access_member_function_valid_args(i0, extra...); - KOKKOS_IMPL_VIEW_OPERATOR_VERIFY(m_track, m_map, i0, extra...) - return m_map.m_impl_handle[m_map.m_impl_offset.m_stride.S0 * i0]; - } - - //------------------------------ - // Rank 2 - - template - KOKKOS_FORCEINLINE_FUNCTION - std::enable_if_t<(Kokkos::Impl::always_true::value && - (2 == rank) && !is_default_map), - reference_type> - access(I0 i0, I1 i1, Is... extra) const { - check_access_member_function_valid_args(i0, i1, extra...); - KOKKOS_IMPL_VIEW_OPERATOR_VERIFY(m_track, m_map, i0, i1, extra...) - return m_map.reference(i0, i1); - } - - template - KOKKOS_FORCEINLINE_FUNCTION std::enable_if_t< - (Kokkos::Impl::always_true::value && (2 == rank) && - is_default_map && - (is_layout_left || is_layout_right || is_layout_stride)), - reference_type> - access(I0 i0, I1 i1, Is... extra) const { - check_access_member_function_valid_args(i0, i1, extra...); - KOKKOS_IMPL_VIEW_OPERATOR_VERIFY(m_track, m_map, i0, i1, extra...) - if constexpr (is_layout_left) { - if constexpr (rank_dynamic == 0) - return m_map.m_impl_handle[i0 + m_map.m_impl_offset.m_dim.N0 * i1]; - else - return m_map.m_impl_handle[i0 + m_map.m_impl_offset.m_stride * i1]; - } else if constexpr (is_layout_right) { - if constexpr (rank_dynamic == 0) - return m_map.m_impl_handle[i1 + m_map.m_impl_offset.m_dim.N1 * i0]; - else - return m_map.m_impl_handle[i1 + m_map.m_impl_offset.m_stride * i0]; - } else { - static_assert(is_layout_stride); - return m_map.m_impl_handle[i0 * m_map.m_impl_offset.m_stride.S0 + - i1 * m_map.m_impl_offset.m_stride.S1]; - } -#if defined KOKKOS_COMPILER_INTEL - __builtin_unreachable(); -#endif - } - - //------------------------------ - // Rank 3 - - template - KOKKOS_FORCEINLINE_FUNCTION - std::enable_if_t<(Kokkos::Impl::always_true::value && - (3 == rank) && is_default_map), - reference_type> - access(I0 i0, I1 i1, I2 i2, Is... extra) const { - check_access_member_function_valid_args(i0, i1, i2, extra...); - KOKKOS_IMPL_VIEW_OPERATOR_VERIFY(m_track, m_map, i0, i1, i2, extra...) - return m_map.m_impl_handle[m_map.m_impl_offset(i0, i1, i2)]; - } - - template - KOKKOS_FORCEINLINE_FUNCTION - std::enable_if_t<(Kokkos::Impl::always_true::value && - (3 == rank) && !is_default_map), - reference_type> - access(I0 i0, I1 i1, I2 i2, Is... extra) const { - check_access_member_function_valid_args(i0, i1, i2, extra...); - KOKKOS_IMPL_VIEW_OPERATOR_VERIFY(m_track, m_map, i0, i1, i2, extra...) - return m_map.reference(i0, i1, i2); - } - - //------------------------------ - // Rank 4 - - template - KOKKOS_FORCEINLINE_FUNCTION std::enable_if_t< - (Kokkos::Impl::always_true::value && (4 == rank) && - is_default_map), - reference_type> - access(I0 i0, I1 i1, I2 i2, I3 i3, Is... extra) const { - check_access_member_function_valid_args(i0, i1, i2, i3, extra...); - KOKKOS_IMPL_VIEW_OPERATOR_VERIFY(m_track, m_map, i0, i1, i2, i3, extra...) - return m_map.m_impl_handle[m_map.m_impl_offset(i0, i1, i2, i3)]; - } - - template - KOKKOS_FORCEINLINE_FUNCTION std::enable_if_t< - (Kokkos::Impl::always_true::value && (4 == rank) && - !is_default_map), - reference_type> - access(I0 i0, I1 i1, I2 i2, I3 i3, Is... extra) const { - check_access_member_function_valid_args(i0, i1, i2, i3, extra...); - KOKKOS_IMPL_VIEW_OPERATOR_VERIFY(m_track, m_map, i0, i1, i2, i3, extra...) - return m_map.reference(i0, i1, i2, i3); - } - - //------------------------------ - // Rank 5 - - template - KOKKOS_FORCEINLINE_FUNCTION std::enable_if_t< - (Kokkos::Impl::always_true::value && - (5 == rank) && is_default_map), - reference_type> - access(I0 i0, I1 i1, I2 i2, I3 i3, I4 i4, Is... extra) const { - check_access_member_function_valid_args(i0, i1, i2, i3, i4, extra...); - KOKKOS_IMPL_VIEW_OPERATOR_VERIFY(m_track, m_map, i0, i1, i2, i3, i4, - extra...) - return m_map.m_impl_handle[m_map.m_impl_offset(i0, i1, i2, i3, i4)]; - } - - template - KOKKOS_FORCEINLINE_FUNCTION std::enable_if_t< - (Kokkos::Impl::always_true::value && - (5 == rank) && !is_default_map), - reference_type> - access(I0 i0, I1 i1, I2 i2, I3 i3, I4 i4, Is... extra) const { - check_access_member_function_valid_args(i0, i1, i2, i3, i4, extra...); - KOKKOS_IMPL_VIEW_OPERATOR_VERIFY(m_track, m_map, i0, i1, i2, i3, i4, - extra...) - return m_map.reference(i0, i1, i2, i3, i4); - } - - //------------------------------ - // Rank 6 - - template - KOKKOS_FORCEINLINE_FUNCTION std::enable_if_t< - (Kokkos::Impl::always_true::value && - (6 == rank) && is_default_map), - reference_type> - access(I0 i0, I1 i1, I2 i2, I3 i3, I4 i4, I5 i5, Is... extra) const { - check_access_member_function_valid_args(i0, i1, i2, i3, i4, i5, extra...); - KOKKOS_IMPL_VIEW_OPERATOR_VERIFY(m_track, m_map, i0, i1, i2, i3, i4, i5, - extra...) - return m_map.m_impl_handle[m_map.m_impl_offset(i0, i1, i2, i3, i4, i5)]; - } - - template - KOKKOS_FORCEINLINE_FUNCTION std::enable_if_t< - (Kokkos::Impl::always_true::value && - (6 == rank) && !is_default_map), - reference_type> - access(I0 i0, I1 i1, I2 i2, I3 i3, I4 i4, I5 i5, Is... extra) const { - check_access_member_function_valid_args(i0, i1, i2, i3, i4, i5, extra...); - KOKKOS_IMPL_VIEW_OPERATOR_VERIFY(m_track, m_map, i0, i1, i2, i3, i4, i5, - extra...) - return m_map.reference(i0, i1, i2, i3, i4, i5); - } - - //------------------------------ - // Rank 7 - - template - KOKKOS_FORCEINLINE_FUNCTION std::enable_if_t< - (Kokkos::Impl::always_true::value && - (7 == rank) && is_default_map), - reference_type> - access(I0 i0, I1 i1, I2 i2, I3 i3, I4 i4, I5 i5, I6 i6, Is... extra) const { - check_access_member_function_valid_args(i0, i1, i2, i3, i4, i5, i6, - extra...); - KOKKOS_IMPL_VIEW_OPERATOR_VERIFY(m_track, m_map, i0, i1, i2, i3, i4, i5, i6, - extra...) - return m_map.m_impl_handle[m_map.m_impl_offset(i0, i1, i2, i3, i4, i5, i6)]; - } - - template - KOKKOS_FORCEINLINE_FUNCTION std::enable_if_t< - (Kokkos::Impl::always_true::value && - (7 == rank) && !is_default_map), - reference_type> - access(I0 i0, I1 i1, I2 i2, I3 i3, I4 i4, I5 i5, I6 i6, Is... extra) const { - check_access_member_function_valid_args(i0, i1, i2, i3, i4, i5, i6, - extra...); - KOKKOS_IMPL_VIEW_OPERATOR_VERIFY(m_track, m_map, i0, i1, i2, i3, i4, i5, i6, - extra...) - return m_map.reference(i0, i1, i2, i3, i4, i5, i6); - } - - //------------------------------ - // Rank 8 - - template - KOKKOS_FORCEINLINE_FUNCTION - std::enable_if_t<(Kokkos::Impl::always_true::value && - (8 == rank) && is_default_map), - reference_type> - access(I0 i0, I1 i1, I2 i2, I3 i3, I4 i4, I5 i5, I6 i6, I7 i7, - Is... extra) const { - check_access_member_function_valid_args(i0, i1, i2, i3, i4, i5, i6, i7, - extra...); - KOKKOS_IMPL_VIEW_OPERATOR_VERIFY(m_track, m_map, i0, i1, i2, i3, i4, i5, i6, - i7, extra...) - return m_map - .m_impl_handle[m_map.m_impl_offset(i0, i1, i2, i3, i4, i5, i6, i7)]; - } - - template - KOKKOS_FORCEINLINE_FUNCTION - std::enable_if_t<(Kokkos::Impl::always_true::value && - (8 == rank) && !is_default_map), - reference_type> - access(I0 i0, I1 i1, I2 i2, I3 i3, I4 i4, I5 i5, I6 i6, I7 i7, - Is... extra) const { - check_access_member_function_valid_args(i0, i1, i2, i3, i4, i5, i6, i7, - extra...); - KOKKOS_IMPL_VIEW_OPERATOR_VERIFY(m_track, m_map, i0, i1, i2, i3, i4, i5, i6, - i7, extra...) - return m_map.reference(i0, i1, i2, i3, i4, i5, i6, i7); - } - -#undef KOKKOS_IMPL_VIEW_OPERATOR_VERIFY - - //---------------------------------------- - // Standard destructor, constructors, and assignment operators - - KOKKOS_DEFAULTED_FUNCTION - ~View() = default; - - KOKKOS_DEFAULTED_FUNCTION - View() = default; - - KOKKOS_FUNCTION - View(const View& other) : m_track(other.m_track), m_map(other.m_map) { - KOKKOS_IF_ON_HOST((hooks_policy::copy_construct(*this, other);)) - } - - KOKKOS_FUNCTION - View(View&& other) - : m_track{std::move(other.m_track)}, m_map{std::move(other.m_map)} { - KOKKOS_IF_ON_HOST((hooks_policy::move_construct(*this, other);)) - } - - KOKKOS_FUNCTION - View& operator=(const View& other) { - m_map = other.m_map; - m_track = other.m_track; - - KOKKOS_IF_ON_HOST((hooks_policy::copy_assign(*this, other);)) - - return *this; - } - - KOKKOS_FUNCTION - View& operator=(View&& other) { - m_map = std::move(other.m_map); - m_track = std::move(other.m_track); - - KOKKOS_IF_ON_HOST((hooks_policy::move_assign(*this, other);)) - - return *this; - } - - //---------------------------------------- - // Compatible view copy constructor and assignment - // may assign unmanaged from managed. - - template - KOKKOS_INLINE_FUNCTION View( - const View& rhs, - std::enable_if_t::traits, - typename traits::specialize>::is_assignable_data_type>* = nullptr) - : m_track(rhs), m_map() { - using SrcTraits = typename View::traits; - using Mapping = Kokkos::Impl::ViewMapping; - static_assert(Mapping::is_assignable, - "Incompatible View copy construction"); - Mapping::assign(m_map, rhs.m_map, rhs.m_track.m_tracker); - } - - template - KOKKOS_INLINE_FUNCTION std::enable_if_t< - Kokkos::Impl::ViewMapping< - traits, typename View::traits, - typename traits::specialize>::is_assignable_data_type, - View>& - operator=(const View& rhs) { - using SrcTraits = typename View::traits; - using Mapping = Kokkos::Impl::ViewMapping; - static_assert(Mapping::is_assignable, "Incompatible View copy assignment"); - Mapping::assign(m_map, rhs.m_map, rhs.m_track.m_tracker); - m_track.assign(rhs); - return *this; - } - - //---------------------------------------- - // Compatible subview constructor - // may assign unmanaged from managed. - - template - KOKKOS_INLINE_FUNCTION View(const View& src_view, const Arg0 arg0, - Args... args) - : m_track(src_view), m_map() { - using SrcType = View; - - using Mapping = Kokkos::Impl::ViewMapping; - - using DstType = typename Mapping::type; - - static_assert( - Kokkos::Impl::ViewMapping::is_assignable, - "Subview construction requires compatible view and subview arguments"); - - Mapping::assign(m_map, src_view.m_map, arg0, args...); - } - - //---------------------------------------- - // Allocation tracking properties - - KOKKOS_INLINE_FUNCTION - int use_count() const { return m_track.m_tracker.use_count(); } - - inline const std::string label() const { - return m_track.m_tracker - .template get_label(); - } - - public: - //---------------------------------------- - // Allocation according to allocation properties and array layout - - template - explicit inline View( - const Impl::ViewCtorProp& arg_prop, - std::enable_if_t::has_pointer, - typename traits::array_layout> const& arg_layout) - : m_track(), m_map() { - // Copy the input allocation properties with possibly defaulted properties - // We need to split it in two to avoid MSVC compiler errors - auto prop_copy_tmp = - Impl::with_properties_if_unset(arg_prop, std::string{}); - auto prop_copy = Impl::with_properties_if_unset( - prop_copy_tmp, typename traits::device_type::memory_space{}, - typename traits::device_type::execution_space{}); - using alloc_prop = decltype(prop_copy); - - static_assert(traits::is_managed, - "View allocation constructor requires managed memory"); - - if (alloc_prop::initialize && - !alloc_prop::execution_space::impl_is_initialized()) { - // If initializing view data then - // the execution space must be initialized. - Kokkos::Impl::throw_runtime_exception( - "Constructing View and initializing data with uninitialized " - "execution space"); - } - -#ifdef KOKKOS_ENABLE_DEBUG_BOUNDS_CHECK - if constexpr (std::is_same_v || - std::is_same_v || - std::is_same_v) { - size_t i0 = arg_layout.dimension[0]; - size_t i1 = arg_layout.dimension[1]; - size_t i2 = arg_layout.dimension[2]; - size_t i3 = arg_layout.dimension[3]; - size_t i4 = arg_layout.dimension[4]; - size_t i5 = arg_layout.dimension[5]; - size_t i6 = arg_layout.dimension[6]; - size_t i7 = arg_layout.dimension[7]; - - const std::string& alloc_name = - Impl::get_property(prop_copy); - Impl::runtime_check_rank( - *this, std::is_same::value, i0, i1, - i2, i3, i4, i5, i6, i7, alloc_name.c_str()); - } -#endif - - Kokkos::Impl::SharedAllocationRecord<>* record = m_map.allocate_shared( - prop_copy, arg_layout, Impl::ViewCtorProp::has_execution_space); - - // Setup and initialization complete, start tracking - m_track.m_tracker.assign_allocated_record_to_uninitialized(record); - } - - KOKKOS_INLINE_FUNCTION - void assign_data(pointer_type arg_data) { - m_track.m_tracker.clear(); - m_map.assign_data(arg_data); - } - - // Wrap memory according to properties and array layout - template - explicit KOKKOS_INLINE_FUNCTION View( - const Impl::ViewCtorProp& arg_prop, - std::enable_if_t::has_pointer, - typename traits::array_layout> const& arg_layout) - : m_track() // No memory tracking - , - m_map(arg_prop, arg_layout) { - static_assert( - std::is_same::pointer_type>::value, - "Constructing View to wrap user memory must supply matching pointer " - "type"); - -#ifdef KOKKOS_ENABLE_DEBUG_BOUNDS_CHECK - if constexpr (std::is_same_v || - std::is_same_v || - std::is_same_v) { - size_t i0 = arg_layout.dimension[0]; - size_t i1 = arg_layout.dimension[1]; - size_t i2 = arg_layout.dimension[2]; - size_t i3 = arg_layout.dimension[3]; - size_t i4 = arg_layout.dimension[4]; - size_t i5 = arg_layout.dimension[5]; - size_t i6 = arg_layout.dimension[6]; - size_t i7 = arg_layout.dimension[7]; - - Impl::runtime_check_rank( - *this, std::is_same::value, i0, i1, - i2, i3, i4, i5, i6, i7, "UNMANAGED"); - } -#endif - } - - // Simple dimension-only layout - template - explicit inline View( - const Impl::ViewCtorProp& arg_prop, - std::enable_if_t::has_pointer, size_t> const - arg_N0 = KOKKOS_IMPL_CTOR_DEFAULT_ARG, - const size_t arg_N1 = KOKKOS_IMPL_CTOR_DEFAULT_ARG, - const size_t arg_N2 = KOKKOS_IMPL_CTOR_DEFAULT_ARG, - const size_t arg_N3 = KOKKOS_IMPL_CTOR_DEFAULT_ARG, - const size_t arg_N4 = KOKKOS_IMPL_CTOR_DEFAULT_ARG, - const size_t arg_N5 = KOKKOS_IMPL_CTOR_DEFAULT_ARG, - const size_t arg_N6 = KOKKOS_IMPL_CTOR_DEFAULT_ARG, - const size_t arg_N7 = KOKKOS_IMPL_CTOR_DEFAULT_ARG) - : View(arg_prop, - typename traits::array_layout(arg_N0, arg_N1, arg_N2, arg_N3, - arg_N4, arg_N5, arg_N6, arg_N7)) { - static_assert(traits::array_layout::is_extent_constructible, - "Layout is not constructible from extent arguments. Use " - "overload taking a layout object instead."); - } - - template - explicit KOKKOS_INLINE_FUNCTION View( - const Impl::ViewCtorProp& arg_prop, - std::enable_if_t::has_pointer, size_t> const - arg_N0 = KOKKOS_IMPL_CTOR_DEFAULT_ARG, - const size_t arg_N1 = KOKKOS_IMPL_CTOR_DEFAULT_ARG, - const size_t arg_N2 = KOKKOS_IMPL_CTOR_DEFAULT_ARG, - const size_t arg_N3 = KOKKOS_IMPL_CTOR_DEFAULT_ARG, - const size_t arg_N4 = KOKKOS_IMPL_CTOR_DEFAULT_ARG, - const size_t arg_N5 = KOKKOS_IMPL_CTOR_DEFAULT_ARG, - const size_t arg_N6 = KOKKOS_IMPL_CTOR_DEFAULT_ARG, - const size_t arg_N7 = KOKKOS_IMPL_CTOR_DEFAULT_ARG) - : View(arg_prop, - typename traits::array_layout(arg_N0, arg_N1, arg_N2, arg_N3, - arg_N4, arg_N5, arg_N6, arg_N7)) { - static_assert(traits::array_layout::is_extent_constructible, - "Layout is not constructible from extent arguments. Use " - "overload taking a layout object instead."); - } - - // Allocate with label and layout - template - explicit inline View( - const Label& arg_label, - std::enable_if_t::value, - typename traits::array_layout> const& arg_layout) - : View(Impl::ViewCtorProp(arg_label), arg_layout) {} - - // Allocate label and layout, must disambiguate from subview constructor. - template - explicit inline View( - const Label& arg_label, - std::enable_if_t::value, const size_t> - arg_N0 = KOKKOS_IMPL_CTOR_DEFAULT_ARG, - const size_t arg_N1 = KOKKOS_IMPL_CTOR_DEFAULT_ARG, - const size_t arg_N2 = KOKKOS_IMPL_CTOR_DEFAULT_ARG, - const size_t arg_N3 = KOKKOS_IMPL_CTOR_DEFAULT_ARG, - const size_t arg_N4 = KOKKOS_IMPL_CTOR_DEFAULT_ARG, - const size_t arg_N5 = KOKKOS_IMPL_CTOR_DEFAULT_ARG, - const size_t arg_N6 = KOKKOS_IMPL_CTOR_DEFAULT_ARG, - const size_t arg_N7 = KOKKOS_IMPL_CTOR_DEFAULT_ARG) - : View(Impl::ViewCtorProp(arg_label), - typename traits::array_layout(arg_N0, arg_N1, arg_N2, arg_N3, - arg_N4, arg_N5, arg_N6, arg_N7)) { - static_assert(traits::array_layout::is_extent_constructible, - "Layout is not constructible from extent arguments. Use " - "overload taking a layout object instead."); - } - - // Construct view from ViewTracker and map - // This should be the preferred method because future extensions may need to - // use the ViewTracker class. - template - KOKKOS_INLINE_FUNCTION View( - const view_tracker_type& track, - const Kokkos::Impl::ViewMapping& map) - : m_track(track), m_map() { - using Mapping = - Kokkos::Impl::ViewMapping; - static_assert(Mapping::is_assignable, - "Incompatible View copy construction"); - Mapping::assign(m_map, map, track.m_tracker); - } - - // Construct View from internal shared allocation tracker object and map - // This is here for backwards compatibility for classes that derive from - // Kokkos::View - template - KOKKOS_INLINE_FUNCTION View( - const typename view_tracker_type::track_type& track, - const Kokkos::Impl::ViewMapping& map) - : m_track(track), m_map() { - using Mapping = - Kokkos::Impl::ViewMapping; - static_assert(Mapping::is_assignable, - "Incompatible View copy construction"); - Mapping::assign(m_map, map, track); - } - - //---------------------------------------- - // Memory span required to wrap these dimensions. - static constexpr size_t required_allocation_size( - typename traits::array_layout const& layout) { - return map_type::memory_span(layout); - } - - static constexpr size_t required_allocation_size( - const size_t arg_N0 = 0, const size_t arg_N1 = 0, const size_t arg_N2 = 0, - const size_t arg_N3 = 0, const size_t arg_N4 = 0, const size_t arg_N5 = 0, - const size_t arg_N6 = 0, const size_t arg_N7 = 0) { - static_assert(traits::array_layout::is_extent_constructible, - "Layout is not constructible from extent arguments. Use " - "overload taking a layout object instead."); - return map_type::memory_span(typename traits::array_layout( - arg_N0, arg_N1, arg_N2, arg_N3, arg_N4, arg_N5, arg_N6, arg_N7)); - } - - explicit KOKKOS_INLINE_FUNCTION View( - pointer_type arg_ptr, const size_t arg_N0 = KOKKOS_IMPL_CTOR_DEFAULT_ARG, - const size_t arg_N1 = KOKKOS_IMPL_CTOR_DEFAULT_ARG, - const size_t arg_N2 = KOKKOS_IMPL_CTOR_DEFAULT_ARG, - const size_t arg_N3 = KOKKOS_IMPL_CTOR_DEFAULT_ARG, - const size_t arg_N4 = KOKKOS_IMPL_CTOR_DEFAULT_ARG, - const size_t arg_N5 = KOKKOS_IMPL_CTOR_DEFAULT_ARG, - const size_t arg_N6 = KOKKOS_IMPL_CTOR_DEFAULT_ARG, - const size_t arg_N7 = KOKKOS_IMPL_CTOR_DEFAULT_ARG) - : View(Impl::ViewCtorProp(arg_ptr), - typename traits::array_layout(arg_N0, arg_N1, arg_N2, arg_N3, - arg_N4, arg_N5, arg_N6, arg_N7)) { - static_assert(traits::array_layout::is_extent_constructible, - "Layout is not constructible from extent arguments. Use " - "overload taking a layout object instead."); - } - - explicit KOKKOS_INLINE_FUNCTION View( - pointer_type arg_ptr, const typename traits::array_layout& arg_layout) - : View(Impl::ViewCtorProp(arg_ptr), arg_layout) {} - - //---------------------------------------- - // Shared scratch memory constructor - - static KOKKOS_INLINE_FUNCTION size_t - shmem_size(const size_t arg_N0 = KOKKOS_INVALID_INDEX, - const size_t arg_N1 = KOKKOS_INVALID_INDEX, - const size_t arg_N2 = KOKKOS_INVALID_INDEX, - const size_t arg_N3 = KOKKOS_INVALID_INDEX, - const size_t arg_N4 = KOKKOS_INVALID_INDEX, - const size_t arg_N5 = KOKKOS_INVALID_INDEX, - const size_t arg_N6 = KOKKOS_INVALID_INDEX, - const size_t arg_N7 = KOKKOS_INVALID_INDEX) { - static_assert(traits::array_layout::is_extent_constructible, - "Layout is not constructible from extent arguments. Use " - "overload taking a layout object instead."); - const size_t num_passed_args = Impl::count_valid_integers( - arg_N0, arg_N1, arg_N2, arg_N3, arg_N4, arg_N5, arg_N6, arg_N7); - - if (std::is_void::value && - num_passed_args != rank_dynamic) { - Kokkos::abort( - "Kokkos::View::shmem_size() rank_dynamic != number of arguments.\n"); - } - - return View::shmem_size(typename traits::array_layout( - arg_N0, arg_N1, arg_N2, arg_N3, arg_N4, arg_N5, arg_N6, arg_N7)); - } - - private: - // Want to be able to align to minimum scratch alignment or sizeof or alignof - // elements - static constexpr size_t scratch_value_alignment = - max({sizeof(typename traits::value_type), - alignof(typename traits::value_type), - static_cast( - traits::execution_space::scratch_memory_space::ALIGN)}); - - public: - static KOKKOS_INLINE_FUNCTION size_t - shmem_size(typename traits::array_layout const& arg_layout) { - return map_type::memory_span(arg_layout) + scratch_value_alignment; - } - - explicit KOKKOS_INLINE_FUNCTION View( - const typename traits::execution_space::scratch_memory_space& arg_space, - const typename traits::array_layout& arg_layout) - : View(Impl::ViewCtorProp(reinterpret_cast( - arg_space.get_shmem_aligned(map_type::memory_span(arg_layout), - scratch_value_alignment))), - arg_layout) {} - - explicit KOKKOS_INLINE_FUNCTION View( - const typename traits::execution_space::scratch_memory_space& arg_space, - const size_t arg_N0 = KOKKOS_IMPL_CTOR_DEFAULT_ARG, - const size_t arg_N1 = KOKKOS_IMPL_CTOR_DEFAULT_ARG, - const size_t arg_N2 = KOKKOS_IMPL_CTOR_DEFAULT_ARG, - const size_t arg_N3 = KOKKOS_IMPL_CTOR_DEFAULT_ARG, - const size_t arg_N4 = KOKKOS_IMPL_CTOR_DEFAULT_ARG, - const size_t arg_N5 = KOKKOS_IMPL_CTOR_DEFAULT_ARG, - const size_t arg_N6 = KOKKOS_IMPL_CTOR_DEFAULT_ARG, - const size_t arg_N7 = KOKKOS_IMPL_CTOR_DEFAULT_ARG) - : View(Impl::ViewCtorProp( - reinterpret_cast(arg_space.get_shmem_aligned( - map_type::memory_span(typename traits::array_layout( - arg_N0, arg_N1, arg_N2, arg_N3, arg_N4, arg_N5, arg_N6, - arg_N7)), - scratch_value_alignment))), - typename traits::array_layout(arg_N0, arg_N1, arg_N2, arg_N3, - arg_N4, arg_N5, arg_N6, arg_N7)) { - static_assert(traits::array_layout::is_extent_constructible, - "Layout is not constructible from extent arguments. Use " - "overload taking a layout object instead."); - } - - //---------------------------------------- - // MDSpan converting constructors -#ifdef KOKKOS_ENABLE_IMPL_MDSPAN - template ::mdspan_type> - KOKKOS_INLINE_FUNCTION -#ifndef KOKKOS_ENABLE_CXX17 - explicit(traits::is_managed) -#endif - View(const typename Impl::MDSpanViewTraits::mdspan_type& mds, - std::enable_if_t< - !std::is_same_v>* = - nullptr) - : View(mds.data_handle(), - Impl::array_layout_from_mapping< - typename traits::array_layout, - typename Impl::MDSpanViewTraits::mdspan_type>( - mds.mapping())) { - } - - template - KOKKOS_INLINE_FUNCTION -#ifndef KOKKOS_ENABLE_CXX17 - explicit(!std::is_convertible_v< - Kokkos::mdspan, - typename Impl::MDSpanViewTraits::mdspan_type>) -#endif - View(const Kokkos::mdspan& mds) - : View(typename Impl::MDSpanViewTraits::mdspan_type(mds)) { - } - - //---------------------------------------- - // Conversion to MDSpan - template ::mdspan_type, - typename = std::enable_if_t, - std::false_type, - std::is_assignable, - ImplNaturalMDSpanType>>::value>> - KOKKOS_INLINE_FUNCTION constexpr operator mdspan< - OtherElementType, OtherExtents, OtherLayoutPolicy, OtherAccessor>() { - using mdspan_type = typename Impl::MDSpanViewTraits::mdspan_type; - return mdspan_type{data(), - Impl::mapping_from_view_mapping(m_map)}; - } - - template >, - typename = std::enable_if_t>> - KOKKOS_INLINE_FUNCTION constexpr auto to_mdspan( - const OtherAccessorType& other_accessor = - typename Impl::MDSpanViewTraits::accessor_type()) { - using mdspan_type = typename Impl::MDSpanViewTraits::mdspan_type; - using ret_mdspan_type = - mdspan; - return ret_mdspan_type{data(), - Impl::mapping_from_view_mapping(m_map), - other_accessor}; - } -#endif // KOKKOS_ENABLE_IMPL_MDSPAN -}; - -template -KOKKOS_INLINE_FUNCTION constexpr unsigned rank(const View&) { - return View::rank(); -} - -namespace Impl { - -template -struct RankDataType { - using type = typename RankDataType::type*; -}; - -template -struct RankDataType { - using type = ValueType; -}; - -template -KOKKOS_FUNCTION std::enable_if_t< - N == View::rank() && - std::is_same::specialize, void>::value, - View> -as_view_of_rank_n(View v) { - return v; -} - -// Placeholder implementation to compile generic code for DynRankView; should -// never be called -template -KOKKOS_FUNCTION std::enable_if_t< - N != View::rank() && - std::is_same::specialize, void>::value, - View::value_type, N>::type, - Args...>> -as_view_of_rank_n(View) { - Kokkos::abort("Trying to get at a View of the wrong rank"); - return {}; -} - -template -void apply_to_view_of_static_rank(Function&& f, View a) { - f(a); -} - -} // namespace Impl -//---------------------------------------------------------------------------- -//---------------------------------------------------------------------------- - -namespace Impl { -template -struct TypeListToViewTraits; - -template -struct TypeListToViewTraits> { - using type = ViewTraits; -}; - -// It is not safe to assume that subviews of views with the Aligned memory trait -// are also aligned. Hence, just remove that attribute for subviews. -template -struct RemoveAlignedMemoryTrait { - private: - using type_list_in = Kokkos::Impl::type_list; - using memory_traits = typename ViewTraits::memory_traits; - using type_list_in_wo_memory_traits = - typename Kokkos::Impl::type_list_remove_first::type; - using new_memory_traits = - Kokkos::MemoryTraits; - using new_type_list = typename Kokkos::Impl::concat_type_list< - type_list_in_wo_memory_traits, - Kokkos::Impl::type_list>::type; - - public: - using type = typename TypeListToViewTraits::type; -}; -} // namespace Impl - -template -KOKKOS_INLINE_FUNCTION auto subview(const View& src, Args... args) { - static_assert(View::rank == sizeof...(Args), - "subview requires one argument for each source View rank"); - - return typename Kokkos::Impl::ViewMapping< - void /* deduce subview type from source view traits */ - , - typename Impl::RemoveAlignedMemoryTrait::type, - Args...>::type(src, args...); -} - -#ifdef KOKKOS_ENABLE_DEPRECATED_CODE_4 -template -KOKKOS_DEPRECATED KOKKOS_INLINE_FUNCTION auto subview(const View& src, - Args... args) { - static_assert(View::rank == sizeof...(Args), - "subview requires one argument for each source View rank"); - static_assert(Kokkos::is_memory_traits::value); - - return typename Kokkos::Impl::ViewMapping< - void /* deduce subview type from source view traits */ - , - typename Impl::RemoveAlignedMemoryTrait::type, - Args...>::type(src, args...); -} -#endif - -template -using Subview = decltype(subview(std::declval(), std::declval()...)); - -} /* namespace Kokkos */ - -//---------------------------------------------------------------------------- -//---------------------------------------------------------------------------- - -namespace Kokkos { - -template -KOKKOS_INLINE_FUNCTION bool operator==(const View& lhs, - const View& rhs) { - // Same data, layout, dimensions - using lhs_traits = ViewTraits; - using rhs_traits = ViewTraits; - - return std::is_same::value && - std::is_same::value && - std::is_same::value && - View::rank() == View::rank() && - lhs.data() == rhs.data() && lhs.span() == rhs.span() && - lhs.extent(0) == rhs.extent(0) && lhs.extent(1) == rhs.extent(1) && - lhs.extent(2) == rhs.extent(2) && lhs.extent(3) == rhs.extent(3) && - lhs.extent(4) == rhs.extent(4) && lhs.extent(5) == rhs.extent(5) && - lhs.extent(6) == rhs.extent(6) && lhs.extent(7) == rhs.extent(7); -} - -template -KOKKOS_INLINE_FUNCTION bool operator!=(const View& lhs, - const View& rhs) { - return !(operator==(lhs, rhs)); -} - -} /* namespace Kokkos */ - -//---------------------------------------------------------------------------- -//---------------------------------------------------------------------------- - -namespace Kokkos { -namespace Impl { - -template -struct CommonViewValueType; - -template -struct CommonViewValueType { - using value_type = std::common_type_t; -}; - -template -struct CommonViewAllocProp; - -template -struct CommonViewAllocProp { - using value_type = ValueType; - using scalar_array_type = ValueType; - - template - KOKKOS_INLINE_FUNCTION CommonViewAllocProp(const Views&...) {} -}; - -template -struct DeduceCommonViewAllocProp; - -// Base case must provide types for: -// 1. specialize 2. value_type 3. is_view 4. prop_type -template -struct DeduceCommonViewAllocProp { - using specialize = typename FirstView::traits::specialize; - - using value_type = typename FirstView::traits::value_type; - - enum : bool { is_view = is_view::value }; - - using prop_type = CommonViewAllocProp; -}; - -template -struct DeduceCommonViewAllocProp { - using NextTraits = DeduceCommonViewAllocProp; - - using first_specialize = typename FirstView::traits::specialize; - using first_value_type = typename FirstView::traits::value_type; - - enum : bool { first_is_view = is_view::value }; - - using next_specialize = typename NextTraits::specialize; - using next_value_type = typename NextTraits::value_type; - - enum : bool { next_is_view = NextTraits::is_view }; - - // common types - - // determine specialize type - // if first and next specialize differ, but are not the same specialize, error - // out - static_assert(!(!std::is_same::value && - !std::is_void::value && - !std::is_void::value), - "Kokkos DeduceCommonViewAllocProp ERROR: Only one non-void " - "specialize trait allowed"); - - // otherwise choose non-void specialize if either/both are non-void - using specialize = std::conditional_t< - std::is_same::value, first_specialize, - std::conditional_t<(std::is_void::value && - !std::is_void::value), - next_specialize, first_specialize>>; - - using value_type = typename CommonViewValueType::value_type; - - enum : bool { is_view = (first_is_view && next_is_view) }; - - using prop_type = CommonViewAllocProp; -}; - -} // end namespace Impl - -template -using DeducedCommonPropsType = - typename Impl::DeduceCommonViewAllocProp::prop_type; - -// This function is required in certain scenarios where users customize -// Kokkos View internals. One example are dynamic length embedded ensemble -// types. The function is used to propagate necessary information -// (like the ensemble size) when creating new views. -// However, most of the time it is called with a single view. -// Furthermore, the propagated information is not just for view allocations. -// From what I can tell, the type of functionality provided by -// common_view_alloc_prop is the equivalent of propagating accessors in mdspan, -// a mechanism we will eventually use to replace this clunky approach here, when -// we are finally mdspan based. -// TODO: get rid of this when we have mdspan -template -KOKKOS_INLINE_FUNCTION DeducedCommonPropsType common_view_alloc_prop( - Views const&... views) { - return DeducedCommonPropsType(views...); -} - -} // namespace Kokkos - -#include -#include - -//---------------------------------------------------------------------------- -//---------------------------------------------------------------------------- - -#endif /* #ifndef KOKKOS_VIEW_HPP */ +#endif /* KOKKOS_VIEW_HPP */ diff --git a/lib/kokkos/core/src/Kokkos_WorkGraphPolicy.hpp b/lib/kokkos/core/src/Kokkos_WorkGraphPolicy.hpp index efa56a086e..4d22634281 100644 --- a/lib/kokkos/core/src/Kokkos_WorkGraphPolicy.hpp +++ b/lib/kokkos/core/src/Kokkos_WorkGraphPolicy.hpp @@ -120,7 +120,7 @@ class WorkGraphPolicy : public Kokkos::Impl::PolicyTraits { (std::int32_t)BEGIN_TOKEN))) { // Attempt to claim ready work index succeeded, // update the hint and return work index - atomic_increment(begin_hint); + atomic_inc(begin_hint); return w; } // arrive here when ready_queue[i] == BEGIN_TOKEN @@ -169,7 +169,7 @@ class WorkGraphPolicy : public Kokkos::Impl::PolicyTraits { void operator()(const TagCount, int i) const noexcept { std::int32_t* const count_queue = &m_queue[m_graph.numRows()]; - atomic_increment(count_queue + m_graph.entries[i]); + atomic_inc(count_queue + m_graph.entries[i]); } KOKKOS_INLINE_FUNCTION diff --git a/lib/kokkos/core/src/OpenACC/Kokkos_OpenACC.cpp b/lib/kokkos/core/src/OpenACC/Kokkos_OpenACC.cpp index 99daf379b6..37fcfb7a1d 100644 --- a/lib/kokkos/core/src/OpenACC/Kokkos_OpenACC.cpp +++ b/lib/kokkos/core/src/OpenACC/Kokkos_OpenACC.cpp @@ -23,7 +23,19 @@ #include #include +#if defined(KOKKOS_IMPL_ARCH_NVIDIA_GPU) +#include +#elif defined(KOKKOS_ARCH_AMD_GPU) +// FIXME_OPENACC - hip_runtime_api.h contains two implementations: one for AMD +// GPUs and the other for NVIDIA GPUs; below macro is needed to choose AMD GPUs. +#define __HIP_PLATFORM_AMD__ +#include +#elif defined(KOKKOS_ENABLE_OPENACC_FORCE_HOST_AS_DEVICE) +#include +#endif + #include +#include Kokkos::Experimental::OpenACC::OpenACC() : m_space_instance( @@ -46,6 +58,8 @@ Kokkos::Experimental::OpenACC::OpenACC(int async_arg) void Kokkos::Experimental::OpenACC::impl_initialize( InitializationSettings const& settings) { + Impl::OpenACCInternal::m_concurrency = + 256000; // FIXME_OPENACC - random guess when cannot compute if (Impl::OpenACC_Traits::may_fallback_to_host && acc_get_num_devices(Impl::OpenACC_Traits::dev_type) == 0 && !settings.has_device_id()) { @@ -59,11 +73,46 @@ void Kokkos::Experimental::OpenACC::impl_initialize( acc_get_device_num(acc_device_host); } else { using Kokkos::Impl::get_visible_devices; + acc_set_device_type(Impl::OpenACC_Traits::dev_type); std::vector const& visible_devices = get_visible_devices(); using Kokkos::Impl::get_gpu; int const dev_num = get_gpu(settings).value_or(visible_devices[0]); acc_set_device_num(dev_num, Impl::OpenACC_Traits::dev_type); Impl::OpenACCInternal::m_acc_device_num = dev_num; +#if defined(KOKKOS_IMPL_ARCH_NVIDIA_GPU) + cudaDeviceProp deviceProp; + cudaError error = cudaGetDeviceProperties(&deviceProp, dev_num); + if (error != cudaSuccess) { + std::ostringstream msg; + msg << "Error: During OpenACC backend initialization, failed to retrieve " + << "CUDA device properties: (" << cudaGetErrorName(error) + << "): " << cudaGetErrorString(error); + Kokkos::Impl::host_abort(msg.str().c_str()); + } + Impl::OpenACCInternal::m_concurrency = + deviceProp.maxThreadsPerMultiProcessor * deviceProp.multiProcessorCount; +#elif defined(KOKKOS_ARCH_AMD_GPU) + hipDeviceProp_t deviceProp; + hipError_t error = hipGetDeviceProperties(&deviceProp, dev_num); + if (error != hipSuccess) { + std::ostringstream msg; + msg << "Error: During OpenACC backend initialization, failed to retrieve " + << "HIP device properties: (" << hipGetErrorName(error) + << "): " << hipGetErrorString(error); + Kokkos::Impl::host_abort(msg.str().c_str()); + } + Impl::OpenACCInternal::m_concurrency = + deviceProp.maxThreadsPerMultiProcessor * deviceProp.multiProcessorCount; +#elif defined(KOKKOS_ENABLE_OPENACC_FORCE_HOST_AS_DEVICE) + Impl::OpenACCInternal::m_concurrency = std::thread::hardware_concurrency(); + if (Impl::OpenACCInternal::m_concurrency == 0) { + Kokkos::Impl::host_abort( + "Error: During OpenACC backend initialization, failed to retrieve " + "CPU hardware concurrency"); + } +#else + // FIXME_OPENACC: Compute Impl::OpenACCInternal::m_concurrency correctly. +#endif } Impl::OpenACCInternal::singleton().initialize(); } @@ -86,6 +135,12 @@ void Kokkos::Experimental::OpenACC::print_configuration(std::ostream& os, os << "yes\n"; #else os << "no\n"; +#endif + os << " KOKKOS_ENABLE_OPENACC_FORCE_HOST_AS_DEVICE: "; +#if defined(KOKKOS_ENABLE_OPENACC_FORCE_HOST_AS_DEVICE) + os << "yes\n"; +#else + os << "no\n"; #endif m_space_instance->print_configuration(os, verbose); } diff --git a/lib/kokkos/core/src/OpenACC/Kokkos_OpenACC.hpp b/lib/kokkos/core/src/OpenACC/Kokkos_OpenACC.hpp index 5155bee33d..aee696bd34 100644 --- a/lib/kokkos/core/src/OpenACC/Kokkos_OpenACC.hpp +++ b/lib/kokkos/core/src/OpenACC/Kokkos_OpenACC.hpp @@ -42,6 +42,7 @@ static_assert(false, // LLVM/Clacc compiler does not need this. #ifndef KOKKOS_COMPILER_CLANG #define KOKKOS_ENABLE_OPENACC_COLLAPSE_HIERARCHICAL_CONSTRUCTS +#define KOKKOS_ENABLE_OPENACC_COLLAPSE_MDRANGE_LOOPS #endif namespace Kokkos::Experimental::Impl { @@ -87,9 +88,9 @@ class OpenACC { static char const* name() { return "OpenACC"; } #ifdef KOKKOS_ENABLE_DEPRECATED_CODE_4 - static int concurrency() { return 256000; } // FIXME_OPENACC + static int concurrency(); #else - int concurrency() const { return 256000; } // FIXME_OPENACC + int concurrency() const; #endif #ifdef KOKKOS_ENABLE_DEPRECATED_CODE_4 KOKKOS_DEPRECATED static bool in_parallel() { diff --git a/lib/kokkos/core/src/OpenACC/Kokkos_OpenACCSpace.hpp b/lib/kokkos/core/src/OpenACC/Kokkos_OpenACCSpace.hpp index 4e7170cbbd..75cef98a8d 100644 --- a/lib/kokkos/core/src/OpenACC/Kokkos_OpenACCSpace.hpp +++ b/lib/kokkos/core/src/OpenACC/Kokkos_OpenACCSpace.hpp @@ -85,16 +85,26 @@ class OpenACCSpace { template <> struct Kokkos::Impl::MemorySpaceAccess { +#if defined(KOKKOS_ENABLE_OPENACC_FORCE_HOST_AS_DEVICE) + enum : bool{assignable = true}; + enum : bool{accessible = true}; +#else enum : bool { assignable = false }; enum : bool { accessible = false }; +#endif enum : bool { deepcopy = true }; }; template <> struct Kokkos::Impl::MemorySpaceAccess { +#if defined(KOKKOS_ENABLE_OPENACC_FORCE_HOST_AS_DEVICE) + enum : bool{assignable = true}; + enum : bool{accessible = true}; +#else enum : bool { assignable = false }; enum : bool { accessible = false }; +#endif enum : bool { deepcopy = true }; }; diff --git a/lib/kokkos/core/src/OpenACC/Kokkos_OpenACC_FunctorAdapter.hpp b/lib/kokkos/core/src/OpenACC/Kokkos_OpenACC_FunctorAdapter.hpp index 82d38586eb..1373f8fa7a 100644 --- a/lib/kokkos/core/src/OpenACC/Kokkos_OpenACC_FunctorAdapter.hpp +++ b/lib/kokkos/core/src/OpenACC/Kokkos_OpenACC_FunctorAdapter.hpp @@ -38,7 +38,7 @@ class FunctorAdapter; \ KOKKOS_IMPL_ACC_PRAGMA(routine CLAUSE) \ template \ - KOKKOS_FUNCTION void operator()(Args &&... args) const { \ + KOKKOS_FUNCTION void operator()(Args &&...args) const { \ if constexpr (std::is_void_v) { \ m_functor(static_cast(args)...); \ } else { \ diff --git a/lib/kokkos/core/src/OpenACC/Kokkos_OpenACC_Instance.cpp b/lib/kokkos/core/src/OpenACC/Kokkos_OpenACC_Instance.cpp index 10a76fbd31..1dad499c1b 100644 --- a/lib/kokkos/core/src/OpenACC/Kokkos_OpenACC_Instance.cpp +++ b/lib/kokkos/core/src/OpenACC/Kokkos_OpenACC_Instance.cpp @@ -27,6 +27,7 @@ // Arbitrary value to denote that we don't know yet what device to use. int Kokkos::Experimental::Impl::OpenACCInternal::m_acc_device_num = -1; +int Kokkos::Experimental::Impl::OpenACCInternal::m_concurrency = -1; Kokkos::Experimental::Impl::OpenACCInternal& Kokkos::Experimental::Impl::OpenACCInternal::singleton() { @@ -78,8 +79,18 @@ void Kokkos::Experimental::Impl::OpenACCInternal::fence( [&]() { acc_wait(m_async_arg); }); } -uint32_t Kokkos::Experimental::Impl::OpenACCInternal::instance_id() const - noexcept { +uint32_t Kokkos::Experimental::Impl::OpenACCInternal::instance_id() + const noexcept { return Kokkos::Tools::Experimental::Impl::idForInstance( reinterpret_cast(this)); } + +#ifdef KOKKOS_ENABLE_DEPRECATED_CODE_4 +int Kokkos::Experimental::OpenACC::concurrency() { + return Impl::OpenACCInternal::m_concurrency; +} +#else +int Kokkos::Experimental::OpenACC::concurrency() const { + return Impl::OpenACCInternal::m_concurrency; +} +#endif diff --git a/lib/kokkos/core/src/OpenACC/Kokkos_OpenACC_Instance.hpp b/lib/kokkos/core/src/OpenACC/Kokkos_OpenACC_Instance.hpp index c3d7236872..343d9921a9 100644 --- a/lib/kokkos/core/src/OpenACC/Kokkos_OpenACC_Instance.hpp +++ b/lib/kokkos/core/src/OpenACC/Kokkos_OpenACC_Instance.hpp @@ -30,11 +30,12 @@ namespace Kokkos::Experimental::Impl { class OpenACCInternal { bool m_is_initialized = false; - OpenACCInternal(const OpenACCInternal&) = default; + OpenACCInternal(const OpenACCInternal&) = default; OpenACCInternal& operator=(const OpenACCInternal&) = default; public: static int m_acc_device_num; + static int m_concurrency; int m_async_arg = acc_async_noval; OpenACCInternal() = default; diff --git a/lib/kokkos/core/src/OpenACC/Kokkos_OpenACC_ParallelFor_MDRange.hpp b/lib/kokkos/core/src/OpenACC/Kokkos_OpenACC_ParallelFor_MDRange.hpp index 550436fe7b..629d26928e 100644 --- a/lib/kokkos/core/src/OpenACC/Kokkos_OpenACC_ParallelFor_MDRange.hpp +++ b/lib/kokkos/core/src/OpenACC/Kokkos_OpenACC_ParallelFor_MDRange.hpp @@ -30,10 +30,23 @@ void OpenACCParallelForMDRangePolicy(OpenACCCollapse, OpenACCIterateLeft, OpenACCMDRangeBegin<2> const& begin, OpenACCMDRangeEnd<2> const& end, int async_arg) { - int begin1 = begin[1]; - int end1 = end[1]; - int begin0 = begin[0]; - int end0 = end[0]; + auto begin1 = begin[1]; + auto end1 = end[1]; + auto begin0 = begin[0]; + auto end0 = end[0]; +#if defined(KOKKOS_ENABLE_OPENACC_COLLAPSE_MDRANGE_LOOPS) + auto dim1 = end1 - begin1; + auto dim0 = end0 - begin0; + auto nIter = dim1 * dim0; +// clang-format off +#pragma acc parallel loop gang vector copyin(functor) async(async_arg) + // clang-format on + for (decltype(nIter) m = 0; m < nIter; ++m) { + auto i1 = m / dim0 + begin1; + auto i0 = m % dim0 + begin0; + functor(i0, i1); + } +#else // clang-format off #pragma acc parallel loop gang vector collapse(2) copyin(functor) async(async_arg) // clang-format on @@ -42,6 +55,7 @@ void OpenACCParallelForMDRangePolicy(OpenACCCollapse, OpenACCIterateLeft, functor(i0, i1); } } +#endif } template @@ -50,10 +64,23 @@ void OpenACCParallelForMDRangePolicy(OpenACCCollapse, OpenACCIterateRight, OpenACCMDRangeBegin<2> const& begin, OpenACCMDRangeEnd<2> const& end, int async_arg) { - int begin0 = begin[0]; - int end0 = end[0]; - int begin1 = begin[1]; - int end1 = end[1]; + auto begin0 = begin[0]; + auto end0 = end[0]; + auto begin1 = begin[1]; + auto end1 = end[1]; +#if defined(KOKKOS_ENABLE_OPENACC_COLLAPSE_MDRANGE_LOOPS) + auto dim1 = end1 - begin1; + auto dim0 = end0 - begin0; + auto nIter = dim1 * dim0; +// clang-format off +#pragma acc parallel loop gang vector copyin(functor) async(async_arg) + // clang-format on + for (decltype(nIter) m = 0; m < nIter; ++m) { + auto i0 = m / dim1 + begin0; + auto i1 = m % dim1 + begin1; + functor(i0, i1); + } +#else // clang-format off #pragma acc parallel loop gang vector collapse(2) copyin(functor) async(async_arg) // clang-format on @@ -62,6 +89,7 @@ void OpenACCParallelForMDRangePolicy(OpenACCCollapse, OpenACCIterateRight, functor(i0, i1); } } +#endif } template @@ -71,12 +99,12 @@ void OpenACCParallelForMDRangePolicy(OpenACCTile, OpenACCIterateLeft, OpenACCMDRangeEnd<2> const& end, OpenACCMDRangeTile<2> const& tile, int async_arg) { - int tile0 = tile[0]; - int tile1 = tile[1]; - int begin1 = begin[1]; - int end1 = end[1]; - int begin0 = begin[0]; - int end0 = end[0]; + auto tile0 = tile[0]; + auto tile1 = tile[1]; + auto begin1 = begin[1]; + auto end1 = end[1]; + auto begin0 = begin[0]; + auto end0 = end[0]; // clang-format off #pragma acc parallel loop gang vector tile(tile0,tile1) copyin(functor) async(async_arg) // clang-format on @@ -94,12 +122,12 @@ void OpenACCParallelForMDRangePolicy(OpenACCTile, OpenACCIterateRight, OpenACCMDRangeEnd<2> const& end, OpenACCMDRangeTile<2> const& tile, int async_arg) { - int tile1 = tile[1]; - int tile0 = tile[0]; - int begin0 = begin[0]; - int end0 = end[0]; - int begin1 = begin[1]; - int end1 = end[1]; + auto tile1 = tile[1]; + auto tile0 = tile[0]; + auto begin0 = begin[0]; + auto end0 = end[0]; + auto begin1 = begin[1]; + auto end1 = end[1]; // clang-format off #pragma acc parallel loop gang vector tile(tile1,tile0) copyin(functor) async(async_arg) // clang-format on @@ -116,12 +144,29 @@ void OpenACCParallelForMDRangePolicy(OpenACCCollapse, OpenACCIterateLeft, OpenACCMDRangeBegin<3> const& begin, OpenACCMDRangeEnd<3> const& end, int async_arg) { - int begin2 = begin[2]; - int end2 = end[2]; - int begin1 = begin[1]; - int end1 = end[1]; - int begin0 = begin[0]; - int end0 = end[0]; + auto begin2 = begin[2]; + auto end2 = end[2]; + auto begin1 = begin[1]; + auto end1 = end[1]; + auto begin0 = begin[0]; + auto end0 = end[0]; +#if defined(KOKKOS_ENABLE_OPENACC_COLLAPSE_MDRANGE_LOOPS) + auto dim2 = end2 - begin2; + auto dim1 = end1 - begin1; + auto dim0 = end0 - begin0; + auto nIter = dim2 * dim1 * dim0; +// clang-format off +#pragma acc parallel loop gang vector copyin(functor) async(async_arg) + // clang-format on + for (decltype(nIter) m = 0; m < nIter; ++m) { + auto tmp1 = dim1 * dim0; + auto i2 = m / tmp1 + begin2; + auto tmp2 = m % tmp1; + auto i1 = tmp2 / dim0 + begin1; + auto i0 = tmp2 % dim0 + begin0; + functor(i0, i1, i2); + } +#else // clang-format off #pragma acc parallel loop gang vector collapse(3) copyin(functor) async(async_arg) // clang-format on @@ -132,6 +177,7 @@ void OpenACCParallelForMDRangePolicy(OpenACCCollapse, OpenACCIterateLeft, } } } +#endif } template @@ -140,12 +186,29 @@ void OpenACCParallelForMDRangePolicy(OpenACCCollapse, OpenACCIterateRight, OpenACCMDRangeBegin<3> const& begin, OpenACCMDRangeEnd<3> const& end, int async_arg) { - int begin0 = begin[0]; - int end0 = end[0]; - int begin1 = begin[1]; - int end1 = end[1]; - int begin2 = begin[2]; - int end2 = end[2]; + auto begin0 = begin[0]; + auto end0 = end[0]; + auto begin1 = begin[1]; + auto end1 = end[1]; + auto begin2 = begin[2]; + auto end2 = end[2]; +#if defined(KOKKOS_ENABLE_OPENACC_COLLAPSE_MDRANGE_LOOPS) + auto dim2 = end2 - begin2; + auto dim1 = end1 - begin1; + auto dim0 = end0 - begin0; + auto nIter = dim2 * dim1 * dim0; +// clang-format off +#pragma acc parallel loop gang vector copyin(functor) async(async_arg) + // clang-format on + for (decltype(nIter) m = 0; m < nIter; ++m) { + auto tmp1 = dim2 * dim1; + auto i0 = m / tmp1 + begin0; + auto tmp2 = m % tmp1; + auto i1 = tmp2 / dim2 + begin1; + auto i2 = tmp2 % dim2 + begin2; + functor(i0, i1, i2); + } +#else // clang-format off #pragma acc parallel loop gang vector collapse(3) copyin(functor) async(async_arg) // clang-format on @@ -156,6 +219,7 @@ void OpenACCParallelForMDRangePolicy(OpenACCCollapse, OpenACCIterateRight, } } } +#endif } template @@ -165,15 +229,15 @@ void OpenACCParallelForMDRangePolicy(OpenACCTile, OpenACCIterateLeft, OpenACCMDRangeEnd<3> const& end, OpenACCMDRangeTile<3> const& tile, int async_arg) { - int tile0 = tile[0]; - int tile1 = tile[1]; - int tile2 = tile[2]; - int begin2 = begin[2]; - int end2 = end[2]; - int begin1 = begin[1]; - int end1 = end[1]; - int begin0 = begin[0]; - int end0 = end[0]; + auto tile0 = tile[0]; + auto tile1 = tile[1]; + auto tile2 = tile[2]; + auto begin2 = begin[2]; + auto end2 = end[2]; + auto begin1 = begin[1]; + auto end1 = end[1]; + auto begin0 = begin[0]; + auto end0 = end[0]; // clang-format off #pragma acc parallel loop gang vector tile(tile0,tile1,tile2) copyin(functor) async(async_arg) // clang-format on @@ -193,15 +257,15 @@ void OpenACCParallelForMDRangePolicy(OpenACCTile, OpenACCIterateRight, OpenACCMDRangeEnd<3> const& end, OpenACCMDRangeTile<3> const& tile, int async_arg) { - int tile2 = tile[2]; - int tile1 = tile[1]; - int tile0 = tile[0]; - int begin0 = begin[0]; - int end0 = end[0]; - int begin1 = begin[1]; - int end1 = end[1]; - int begin2 = begin[2]; - int end2 = end[2]; + auto tile2 = tile[2]; + auto tile1 = tile[1]; + auto tile0 = tile[0]; + auto begin0 = begin[0]; + auto end0 = end[0]; + auto begin1 = begin[1]; + auto end1 = end[1]; + auto begin2 = begin[2]; + auto end2 = end[2]; // clang-format off #pragma acc parallel loop gang vector tile(tile2,tile1,tile0) copyin(functor) async(async_arg) // clang-format on @@ -220,14 +284,35 @@ void OpenACCParallelForMDRangePolicy(OpenACCCollapse, OpenACCIterateLeft, OpenACCMDRangeBegin<4> const& begin, OpenACCMDRangeEnd<4> const& end, int async_arg) { - int begin3 = begin[3]; - int end3 = end[3]; - int begin2 = begin[2]; - int end2 = end[2]; - int begin1 = begin[1]; - int end1 = end[1]; - int begin0 = begin[0]; - int end0 = end[0]; + auto begin3 = begin[3]; + auto end3 = end[3]; + auto begin2 = begin[2]; + auto end2 = end[2]; + auto begin1 = begin[1]; + auto end1 = end[1]; + auto begin0 = begin[0]; + auto end0 = end[0]; +#if defined(KOKKOS_ENABLE_OPENACC_COLLAPSE_MDRANGE_LOOPS) + auto dim3 = end3 - begin3; + auto dim2 = end2 - begin2; + auto dim1 = end1 - begin1; + auto dim0 = end0 - begin0; + auto nIter = dim3 * dim2 * dim1 * dim0; +// clang-format off +#pragma acc parallel loop gang vector copyin(functor) async(async_arg) + // clang-format on + for (decltype(nIter) m = 0; m < nIter; ++m) { + auto tmp1 = dim2 * dim1 * dim0; + auto i3 = m / tmp1 + begin3; + auto tmp2 = m % tmp1; + tmp1 = dim1 * dim0; + auto i2 = tmp2 / tmp1 + begin2; + tmp2 = tmp2 % tmp1; + auto i1 = tmp2 / dim0 + begin1; + auto i0 = tmp2 % dim0 + begin0; + functor(i0, i1, i2, i3); + } +#else // clang-format off #pragma acc parallel loop gang vector collapse(4) copyin(functor) async(async_arg) // clang-format on @@ -240,6 +325,7 @@ void OpenACCParallelForMDRangePolicy(OpenACCCollapse, OpenACCIterateLeft, } } } +#endif } template @@ -248,14 +334,35 @@ void OpenACCParallelForMDRangePolicy(OpenACCCollapse, OpenACCIterateRight, OpenACCMDRangeBegin<4> const& begin, OpenACCMDRangeEnd<4> const& end, int async_arg) { - int begin0 = begin[0]; - int end0 = end[0]; - int begin1 = begin[1]; - int end1 = end[1]; - int begin2 = begin[2]; - int end2 = end[2]; - int begin3 = begin[3]; - int end3 = end[3]; + auto begin0 = begin[0]; + auto end0 = end[0]; + auto begin1 = begin[1]; + auto end1 = end[1]; + auto begin2 = begin[2]; + auto end2 = end[2]; + auto begin3 = begin[3]; + auto end3 = end[3]; +#if defined(KOKKOS_ENABLE_OPENACC_COLLAPSE_MDRANGE_LOOPS) + auto dim3 = end3 - begin3; + auto dim2 = end2 - begin2; + auto dim1 = end1 - begin1; + auto dim0 = end0 - begin0; + auto nIter = dim3 * dim2 * dim1 * dim0; +// clang-format off +#pragma acc parallel loop gang vector copyin(functor) async(async_arg) + // clang-format on + for (decltype(nIter) m = 0; m < nIter; ++m) { + auto tmp1 = dim3 * dim2 * dim1; + auto i0 = m / tmp1 + begin0; + auto tmp2 = m % tmp1; + tmp1 = dim3 * dim2; + auto i1 = tmp2 / tmp1 + begin1; + tmp2 = tmp2 % tmp1; + auto i2 = tmp2 / dim3 + begin2; + auto i3 = tmp2 % dim3 + begin3; + functor(i0, i1, i2, i3); + } +#else // clang-format off #pragma acc parallel loop gang vector collapse(4) copyin(functor) async(async_arg) // clang-format on @@ -268,6 +375,7 @@ void OpenACCParallelForMDRangePolicy(OpenACCCollapse, OpenACCIterateRight, } } } +#endif } template @@ -277,18 +385,18 @@ void OpenACCParallelForMDRangePolicy(OpenACCTile, OpenACCIterateLeft, OpenACCMDRangeEnd<4> const& end, OpenACCMDRangeTile<4> const& tile, int async_arg) { - int tile0 = tile[0]; - int tile1 = tile[1]; - int tile2 = tile[2]; - int tile3 = tile[3]; - int begin3 = begin[3]; - int end3 = end[3]; - int begin2 = begin[2]; - int end2 = end[2]; - int begin1 = begin[1]; - int end1 = end[1]; - int begin0 = begin[0]; - int end0 = end[0]; + auto tile0 = tile[0]; + auto tile1 = tile[1]; + auto tile2 = tile[2]; + auto tile3 = tile[3]; + auto begin3 = begin[3]; + auto end3 = end[3]; + auto begin2 = begin[2]; + auto end2 = end[2]; + auto begin1 = begin[1]; + auto end1 = end[1]; + auto begin0 = begin[0]; + auto end0 = end[0]; // clang-format off #pragma acc parallel loop gang vector tile(tile0,tile1,tile2,tile3) copyin(functor) async(async_arg) // clang-format on @@ -310,18 +418,18 @@ void OpenACCParallelForMDRangePolicy(OpenACCTile, OpenACCIterateRight, OpenACCMDRangeEnd<4> const& end, OpenACCMDRangeTile<4> const& tile, int async_arg) { - int tile3 = tile[3]; - int tile2 = tile[2]; - int tile1 = tile[1]; - int tile0 = tile[0]; - int begin0 = begin[0]; - int end0 = end[0]; - int begin1 = begin[1]; - int end1 = end[1]; - int begin2 = begin[2]; - int end2 = end[2]; - int begin3 = begin[3]; - int end3 = end[3]; + auto tile3 = tile[3]; + auto tile2 = tile[2]; + auto tile1 = tile[1]; + auto tile0 = tile[0]; + auto begin0 = begin[0]; + auto end0 = end[0]; + auto begin1 = begin[1]; + auto end1 = end[1]; + auto begin2 = begin[2]; + auto end2 = end[2]; + auto begin3 = begin[3]; + auto end3 = end[3]; // clang-format off #pragma acc parallel loop gang vector tile(tile3,tile2,tile1,tile0) copyin(functor) async(async_arg) // clang-format on @@ -342,16 +450,41 @@ void OpenACCParallelForMDRangePolicy(OpenACCCollapse, OpenACCIterateLeft, OpenACCMDRangeBegin<5> const& begin, OpenACCMDRangeEnd<5> const& end, int async_arg) { - int begin4 = begin[4]; - int end4 = end[4]; - int begin3 = begin[3]; - int end3 = end[3]; - int begin2 = begin[2]; - int end2 = end[2]; - int begin1 = begin[1]; - int end1 = end[1]; - int begin0 = begin[0]; - int end0 = end[0]; + auto begin4 = begin[4]; + auto end4 = end[4]; + auto begin3 = begin[3]; + auto end3 = end[3]; + auto begin2 = begin[2]; + auto end2 = end[2]; + auto begin1 = begin[1]; + auto end1 = end[1]; + auto begin0 = begin[0]; + auto end0 = end[0]; +#if defined(KOKKOS_ENABLE_OPENACC_COLLAPSE_MDRANGE_LOOPS) + auto dim4 = end4 - begin4; + auto dim3 = end3 - begin3; + auto dim2 = end2 - begin2; + auto dim1 = end1 - begin1; + auto dim0 = end0 - begin0; + auto nIter = dim4 * dim3 * dim2 * dim1 * dim0; +// clang-format off +#pragma acc parallel loop gang vector copyin(functor) async(async_arg) + // clang-format on + for (decltype(nIter) m = 0; m < nIter; ++m) { + auto tmp1 = dim3 * dim2 * dim1 * dim0; + auto i4 = m / tmp1 + begin4; + auto tmp2 = m % tmp1; + tmp1 = dim2 * dim1 * dim0; + auto i3 = tmp2 / tmp1 + begin3; + tmp2 = tmp2 % tmp1; + tmp1 = dim1 * dim0; + auto i2 = tmp2 / tmp1 + begin2; + tmp2 = tmp2 % tmp1; + auto i1 = tmp2 / dim0 + begin1; + auto i0 = tmp2 % dim0 + begin0; + functor(i0, i1, i2, i3, i4); + } +#else // clang-format off #pragma acc parallel loop gang vector collapse(5) copyin(functor) async(async_arg) // clang-format on @@ -366,6 +499,7 @@ void OpenACCParallelForMDRangePolicy(OpenACCCollapse, OpenACCIterateLeft, } } } +#endif } template @@ -374,16 +508,41 @@ void OpenACCParallelForMDRangePolicy(OpenACCCollapse, OpenACCIterateRight, OpenACCMDRangeBegin<5> const& begin, OpenACCMDRangeEnd<5> const& end, int async_arg) { - int begin0 = begin[0]; - int end0 = end[0]; - int begin1 = begin[1]; - int end1 = end[1]; - int begin2 = begin[2]; - int end2 = end[2]; - int begin3 = begin[3]; - int end3 = end[3]; - int begin4 = begin[4]; - int end4 = end[4]; + auto begin0 = begin[0]; + auto end0 = end[0]; + auto begin1 = begin[1]; + auto end1 = end[1]; + auto begin2 = begin[2]; + auto end2 = end[2]; + auto begin3 = begin[3]; + auto end3 = end[3]; + auto begin4 = begin[4]; + auto end4 = end[4]; +#if defined(KOKKOS_ENABLE_OPENACC_COLLAPSE_MDRANGE_LOOPS) + auto dim4 = end4 - begin4; + auto dim3 = end3 - begin3; + auto dim2 = end2 - begin2; + auto dim1 = end1 - begin1; + auto dim0 = end0 - begin0; + auto nIter = dim4 * dim3 * dim2 * dim1 * dim0; +// clang-format off +#pragma acc parallel loop gang vector copyin(functor) async(async_arg) + // clang-format on + for (decltype(nIter) m = 0; m < nIter; ++m) { + auto tmp1 = dim4 * dim3 * dim2 * dim1; + auto i0 = m / tmp1 + begin0; + auto tmp2 = m % tmp1; + tmp1 = dim4 * dim3 * dim2; + auto i1 = tmp2 / tmp1 + begin1; + tmp2 = tmp2 % tmp1; + tmp1 = dim4 * dim3; + auto i2 = tmp2 / tmp1 + begin2; + tmp2 = tmp2 % tmp1; + auto i3 = tmp2 / dim4 + begin3; + auto i4 = tmp2 % dim4 + begin4; + functor(i0, i1, i2, i3, i4); + } +#else // clang-format off #pragma acc parallel loop gang vector collapse(5) copyin(functor) async(async_arg) // clang-format on @@ -398,6 +557,7 @@ void OpenACCParallelForMDRangePolicy(OpenACCCollapse, OpenACCIterateRight, } } } +#endif } template @@ -407,21 +567,21 @@ void OpenACCParallelForMDRangePolicy(OpenACCTile, OpenACCIterateLeft, OpenACCMDRangeEnd<5> const& end, OpenACCMDRangeTile<5> const& tile, int async_arg) { - int tile0 = tile[0]; - int tile1 = tile[1]; - int tile2 = tile[2]; - int tile3 = tile[3]; - int tile4 = tile[4]; - int begin4 = begin[4]; - int end4 = end[4]; - int begin3 = begin[3]; - int end3 = end[3]; - int begin2 = begin[2]; - int end2 = end[2]; - int begin1 = begin[1]; - int end1 = end[1]; - int begin0 = begin[0]; - int end0 = end[0]; + auto tile0 = tile[0]; + auto tile1 = tile[1]; + auto tile2 = tile[2]; + auto tile3 = tile[3]; + auto tile4 = tile[4]; + auto begin4 = begin[4]; + auto end4 = end[4]; + auto begin3 = begin[3]; + auto end3 = end[3]; + auto begin2 = begin[2]; + auto end2 = end[2]; + auto begin1 = begin[1]; + auto end1 = end[1]; + auto begin0 = begin[0]; + auto end0 = end[0]; // clang-format off #pragma acc parallel loop gang vector tile(tile0,tile1,tile2,tile3,tile4) copyin(functor) async(async_arg) // clang-format on @@ -445,21 +605,21 @@ void OpenACCParallelForMDRangePolicy(OpenACCTile, OpenACCIterateRight, OpenACCMDRangeEnd<5> const& end, OpenACCMDRangeTile<5> const& tile, int async_arg) { - int tile4 = tile[4]; - int tile3 = tile[3]; - int tile2 = tile[2]; - int tile1 = tile[1]; - int tile0 = tile[0]; - int begin0 = begin[0]; - int end0 = end[0]; - int begin1 = begin[1]; - int end1 = end[1]; - int begin2 = begin[2]; - int end2 = end[2]; - int begin3 = begin[3]; - int end3 = end[3]; - int begin4 = begin[4]; - int end4 = end[4]; + auto tile4 = tile[4]; + auto tile3 = tile[3]; + auto tile2 = tile[2]; + auto tile1 = tile[1]; + auto tile0 = tile[0]; + auto begin0 = begin[0]; + auto end0 = end[0]; + auto begin1 = begin[1]; + auto end1 = end[1]; + auto begin2 = begin[2]; + auto end2 = end[2]; + auto begin3 = begin[3]; + auto end3 = end[3]; + auto begin4 = begin[4]; + auto end4 = end[4]; // clang-format off #pragma acc parallel loop gang vector tile(tile4,tile3,tile2,tile1,tile0) copyin(functor) async(async_arg) // clang-format on @@ -482,18 +642,47 @@ void OpenACCParallelForMDRangePolicy(OpenACCCollapse, OpenACCIterateLeft, OpenACCMDRangeBegin<6> const& begin, OpenACCMDRangeEnd<6> const& end, int async_arg) { - int begin5 = begin[5]; - int end5 = end[5]; - int begin4 = begin[4]; - int end4 = end[4]; - int begin3 = begin[3]; - int end3 = end[3]; - int begin2 = begin[2]; - int end2 = end[2]; - int begin1 = begin[1]; - int end1 = end[1]; - int begin0 = begin[0]; - int end0 = end[0]; + auto begin5 = begin[5]; + auto end5 = end[5]; + auto begin4 = begin[4]; + auto end4 = end[4]; + auto begin3 = begin[3]; + auto end3 = end[3]; + auto begin2 = begin[2]; + auto end2 = end[2]; + auto begin1 = begin[1]; + auto end1 = end[1]; + auto begin0 = begin[0]; + auto end0 = end[0]; +#if defined(KOKKOS_ENABLE_OPENACC_COLLAPSE_MDRANGE_LOOPS) + auto dim5 = end5 - begin5; + auto dim4 = end4 - begin4; + auto dim3 = end3 - begin3; + auto dim2 = end2 - begin2; + auto dim1 = end1 - begin1; + auto dim0 = end0 - begin0; + auto nIter = dim5 * dim4 * dim3 * dim2 * dim1 * dim0; +// clang-format off +#pragma acc parallel loop gang vector copyin(functor) async(async_arg) + // clang-format on + for (decltype(nIter) m = 0; m < nIter; ++m) { + auto tmp1 = dim4 * dim3 * dim2 * dim1 * dim0; + auto i5 = m / tmp1 + begin5; + auto tmp2 = m % tmp1; + tmp1 = dim3 * dim2 * dim1 * dim0; + auto i4 = tmp2 / tmp1 + begin4; + tmp2 = tmp2 % tmp1; + tmp1 = dim2 * dim1 * dim0; + auto i3 = tmp2 / tmp1 + begin3; + tmp2 = tmp2 % tmp1; + tmp1 = dim1 * dim0; + auto i2 = tmp2 / tmp1 + begin2; + tmp2 = tmp2 % tmp1; + auto i1 = tmp2 / dim0 + begin1; + auto i0 = tmp2 % dim0 + begin0; + functor(i0, i1, i2, i3, i4, i5); + } +#else // clang-format off #pragma acc parallel loop gang vector collapse(6) copyin(functor) async(async_arg) // clang-format on @@ -510,6 +699,7 @@ void OpenACCParallelForMDRangePolicy(OpenACCCollapse, OpenACCIterateLeft, } } } +#endif } template @@ -518,18 +708,47 @@ void OpenACCParallelForMDRangePolicy(OpenACCCollapse, OpenACCIterateRight, OpenACCMDRangeBegin<6> const& begin, OpenACCMDRangeEnd<6> const& end, int async_arg) { - int begin0 = begin[0]; - int end0 = end[0]; - int begin1 = begin[1]; - int end1 = end[1]; - int begin2 = begin[2]; - int end2 = end[2]; - int begin3 = begin[3]; - int end3 = end[3]; - int begin4 = begin[4]; - int end4 = end[4]; - int begin5 = begin[5]; - int end5 = end[5]; + auto begin0 = begin[0]; + auto end0 = end[0]; + auto begin1 = begin[1]; + auto end1 = end[1]; + auto begin2 = begin[2]; + auto end2 = end[2]; + auto begin3 = begin[3]; + auto end3 = end[3]; + auto begin4 = begin[4]; + auto end4 = end[4]; + auto begin5 = begin[5]; + auto end5 = end[5]; +#if defined(KOKKOS_ENABLE_OPENACC_COLLAPSE_MDRANGE_LOOPS) + auto dim5 = end5 - begin5; + auto dim4 = end4 - begin4; + auto dim3 = end3 - begin3; + auto dim2 = end2 - begin2; + auto dim1 = end1 - begin1; + auto dim0 = end0 - begin0; + auto nIter = dim5 * dim4 * dim3 * dim2 * dim1 * dim0; +// clang-format off +#pragma acc parallel loop gang vector copyin(functor) async(async_arg) + // clang-format on + for (decltype(nIter) m = 0; m < nIter; ++m) { + auto tmp1 = dim5 * dim4 * dim3 * dim2 * dim1; + auto i0 = m / tmp1 + begin0; + auto tmp2 = m % tmp1; + tmp1 = dim5 * dim4 * dim3 * dim2; + auto i1 = tmp2 / tmp1 + begin1; + tmp2 = tmp2 % tmp1; + tmp1 = dim5 * dim4 * dim3; + auto i2 = tmp2 / tmp1 + begin2; + tmp2 = tmp2 % tmp1; + tmp1 = dim5 * dim4; + auto i3 = tmp2 / tmp1 + begin3; + tmp2 = tmp2 % tmp1; + auto i4 = tmp2 / dim5 + begin4; + auto i5 = tmp2 % dim5 + begin5; + functor(i0, i1, i2, i3, i4, i5); + } +#else // clang-format off #pragma acc parallel loop gang vector collapse(6) copyin(functor) async(async_arg) // clang-format on @@ -546,6 +765,7 @@ void OpenACCParallelForMDRangePolicy(OpenACCCollapse, OpenACCIterateRight, } } } +#endif } template @@ -555,24 +775,24 @@ void OpenACCParallelForMDRangePolicy(OpenACCTile, OpenACCIterateLeft, OpenACCMDRangeEnd<6> const& end, OpenACCMDRangeTile<6> const& tile, int async_arg) { - int tile0 = tile[0]; - int tile1 = tile[1]; - int tile2 = tile[2]; - int tile3 = tile[3]; - int tile4 = tile[4]; - int tile5 = tile[5]; - int begin5 = begin[5]; - int end5 = end[5]; - int begin4 = begin[4]; - int end4 = end[4]; - int begin3 = begin[3]; - int end3 = end[3]; - int begin2 = begin[2]; - int end2 = end[2]; - int begin1 = begin[1]; - int end1 = end[1]; - int begin0 = begin[0]; - int end0 = end[0]; + auto tile0 = tile[0]; + auto tile1 = tile[1]; + auto tile2 = tile[2]; + auto tile3 = tile[3]; + auto tile4 = tile[4]; + auto tile5 = tile[5]; + auto begin5 = begin[5]; + auto end5 = end[5]; + auto begin4 = begin[4]; + auto end4 = end[4]; + auto begin3 = begin[3]; + auto end3 = end[3]; + auto begin2 = begin[2]; + auto end2 = end[2]; + auto begin1 = begin[1]; + auto end1 = end[1]; + auto begin0 = begin[0]; + auto end0 = end[0]; // clang-format off #pragma acc parallel loop gang vector tile(tile0,tile1,tile2,tile3,tile4,tile5) copyin(functor) async(async_arg) // clang-format on @@ -598,24 +818,24 @@ void OpenACCParallelForMDRangePolicy(OpenACCTile, OpenACCIterateRight, OpenACCMDRangeEnd<6> const& end, OpenACCMDRangeTile<6> const& tile, int async_arg) { - int tile5 = tile[5]; - int tile4 = tile[4]; - int tile3 = tile[3]; - int tile2 = tile[2]; - int tile1 = tile[1]; - int tile0 = tile[0]; - int begin0 = begin[0]; - int end0 = end[0]; - int begin1 = begin[1]; - int end1 = end[1]; - int begin2 = begin[2]; - int end2 = end[2]; - int begin3 = begin[3]; - int end3 = end[3]; - int begin4 = begin[4]; - int end4 = end[4]; - int begin5 = begin[5]; - int end5 = end[5]; + auto tile5 = tile[5]; + auto tile4 = tile[4]; + auto tile3 = tile[3]; + auto tile2 = tile[2]; + auto tile1 = tile[1]; + auto tile0 = tile[0]; + auto begin0 = begin[0]; + auto end0 = end[0]; + auto begin1 = begin[1]; + auto end1 = end[1]; + auto begin2 = begin[2]; + auto end2 = end[2]; + auto begin3 = begin[3]; + auto end3 = end[3]; + auto begin4 = begin[4]; + auto end4 = end[4]; + auto begin5 = begin[5]; + auto end5 = end[5]; // clang-format off #pragma acc parallel loop gang vector tile(tile5,tile4,tile3,tile2,tile1,tile0) copyin(functor) async(async_arg) // clang-format on diff --git a/lib/kokkos/core/src/OpenACC/Kokkos_OpenACC_ParallelReduce_MDRange.hpp b/lib/kokkos/core/src/OpenACC/Kokkos_OpenACC_ParallelReduce_MDRange.hpp index 5afb5e75d3..2b5631d6f8 100644 --- a/lib/kokkos/core/src/OpenACC/Kokkos_OpenACC_ParallelReduce_MDRange.hpp +++ b/lib/kokkos/core/src/OpenACC/Kokkos_OpenACC_ParallelReduce_MDRange.hpp @@ -113,6 +113,404 @@ class Kokkos::Impl::ParallelReduce \ + void OpenACCParallelReduce##REDUCER(OpenACCIterateLeft, ValueType& aval, \ + Functor const& afunctor, \ + OpenACCMDRangeBegin<2> const& begin, \ + OpenACCMDRangeEnd<2> const& end, \ + int async_arg) { \ + auto val = aval; \ + auto const functor(afunctor); \ + auto begin1 = begin[1]; \ + auto end1 = end[1]; \ + auto begin0 = begin[0]; \ + auto end0 = end[0]; \ + auto dim1 = end1 - begin1; \ + auto dim0 = end0 - begin0; \ + auto nIter = dim1 * dim0; \ + /* clang-format off */ \ + KOKKOS_IMPL_ACC_PRAGMA(parallel loop gang vector reduction(OPERATOR:val) copyin(functor) async(async_arg)) \ + /* clang-format on */ \ + for (decltype(nIter) m = 0; m < nIter; ++m) { \ + auto i1 = m / dim0 + begin1; \ + auto i0 = m % dim0 + begin0; \ + functor(i0, i1, val); \ + } \ + acc_wait(async_arg); \ + aval = val; \ + } \ + \ + template \ + void OpenACCParallelReduce##REDUCER(OpenACCIterateRight, ValueType& aval, \ + Functor const& afunctor, \ + OpenACCMDRangeBegin<2> const& begin, \ + OpenACCMDRangeEnd<2> const& end, \ + int async_arg) { \ + auto val = aval; \ + auto const functor(afunctor); \ + auto begin0 = begin[0]; \ + auto end0 = end[0]; \ + auto begin1 = begin[1]; \ + auto end1 = end[1]; \ + auto dim1 = end1 - begin1; \ + auto dim0 = end0 - begin0; \ + auto nIter = dim1 * dim0; \ + /* clang-format off */ \ + KOKKOS_IMPL_ACC_PRAGMA(parallel loop gang vector reduction(OPERATOR:val) copyin(functor) async(async_arg)) \ + /* clang-format on */ \ + for (decltype(nIter) m = 0; m < nIter; ++m) { \ + auto i0 = m / dim1 + begin0; \ + auto i1 = m % dim1 + begin1; \ + functor(i0, i1, val); \ + } \ + acc_wait(async_arg); \ + aval = val; \ + } \ + \ + template \ + void OpenACCParallelReduce##REDUCER(OpenACCIterateLeft, ValueType& aval, \ + Functor const& afunctor, \ + OpenACCMDRangeBegin<3> const& begin, \ + OpenACCMDRangeEnd<3> const& end, \ + int async_arg) { \ + auto val = aval; \ + auto const functor(afunctor); \ + auto begin2 = begin[2]; \ + auto end2 = end[2]; \ + auto begin1 = begin[1]; \ + auto end1 = end[1]; \ + auto begin0 = begin[0]; \ + auto end0 = end[0]; \ + auto dim2 = end2 - begin2; \ + auto dim1 = end1 - begin1; \ + auto dim0 = end0 - begin0; \ + auto nIter = dim2 * dim1 * dim0; \ + /* clang-format off */ \ + KOKKOS_IMPL_ACC_PRAGMA(parallel loop gang vector reduction(OPERATOR:val) copyin(functor) async(async_arg)) \ + /* clang-format on */ \ + for (decltype(nIter) m = 0; m < nIter; ++m) { \ + auto tmp1 = dim1 * dim0; \ + auto i2 = m / tmp1 + begin2; \ + auto tmp2 = m % tmp1; \ + auto i1 = tmp2 / dim0 + begin1; \ + auto i0 = tmp2 % dim0 + begin0; \ + functor(i0, i1, i2, val); \ + } \ + acc_wait(async_arg); \ + aval = val; \ + } \ + \ + template \ + void OpenACCParallelReduce##REDUCER(OpenACCIterateRight, ValueType& aval, \ + Functor const& afunctor, \ + OpenACCMDRangeBegin<3> const& begin, \ + OpenACCMDRangeEnd<3> const& end, \ + int async_arg) { \ + auto val = aval; \ + auto const functor(afunctor); \ + auto begin0 = begin[0]; \ + auto end0 = end[0]; \ + auto begin1 = begin[1]; \ + auto end1 = end[1]; \ + auto begin2 = begin[2]; \ + auto end2 = end[2]; \ + auto dim2 = end2 - begin2; \ + auto dim1 = end1 - begin1; \ + auto dim0 = end0 - begin0; \ + auto nIter = dim2 * dim1 * dim0; \ + /* clang-format off */ \ + KOKKOS_IMPL_ACC_PRAGMA(parallel loop gang vector reduction(OPERATOR:val) copyin(functor) async(async_arg)) \ + /* clang-format on */ \ + for (decltype(nIter) m = 0; m < nIter; ++m) { \ + auto tmp1 = dim2 * dim1; \ + auto i0 = m / tmp1 + begin0; \ + auto tmp2 = m % tmp1; \ + auto i1 = tmp2 / dim2 + begin1; \ + auto i2 = tmp2 % dim2 + begin2; \ + functor(i0, i1, i2, val); \ + } \ + acc_wait(async_arg); \ + aval = val; \ + } \ + \ + template \ + void OpenACCParallelReduce##REDUCER(OpenACCIterateLeft, ValueType& aval, \ + Functor const& afunctor, \ + OpenACCMDRangeBegin<4> const& begin, \ + OpenACCMDRangeEnd<4> const& end, \ + int async_arg) { \ + auto val = aval; \ + auto const functor(afunctor); \ + auto begin3 = begin[3]; \ + auto end3 = end[3]; \ + auto begin2 = begin[2]; \ + auto end2 = end[2]; \ + auto begin1 = begin[1]; \ + auto end1 = end[1]; \ + auto begin0 = begin[0]; \ + auto end0 = end[0]; \ + auto dim3 = end3 - begin3; \ + auto dim2 = end2 - begin2; \ + auto dim1 = end1 - begin1; \ + auto dim0 = end0 - begin0; \ + auto nIter = dim3 * dim2 * dim1 * dim0; \ + /* clang-format off */ \ + KOKKOS_IMPL_ACC_PRAGMA(parallel loop gang vector reduction(OPERATOR:val) copyin(functor) async(async_arg)) \ + /* clang-format on */ \ + for (decltype(nIter) m = 0; m < nIter; ++m) { \ + auto tmp1 = dim2 * dim1 * dim0; \ + auto i3 = m / tmp1 + begin3; \ + auto tmp2 = m % tmp1; \ + tmp1 = dim1 * dim0; \ + auto i2 = tmp2 / tmp1 + begin2; \ + tmp2 = tmp2 % tmp1; \ + auto i1 = tmp2 / dim0 + begin1; \ + auto i0 = tmp2 % dim0 + begin0; \ + functor(i0, i1, i2, i3, val); \ + } \ + acc_wait(async_arg); \ + aval = val; \ + } \ + \ + template \ + void OpenACCParallelReduce##REDUCER(OpenACCIterateRight, ValueType& aval, \ + Functor const& afunctor, \ + OpenACCMDRangeBegin<4> const& begin, \ + OpenACCMDRangeEnd<4> const& end, \ + int async_arg) { \ + auto val = aval; \ + auto const functor(afunctor); \ + auto begin0 = begin[0]; \ + auto end0 = end[0]; \ + auto begin1 = begin[1]; \ + auto end1 = end[1]; \ + auto begin2 = begin[2]; \ + auto end2 = end[2]; \ + auto begin3 = begin[3]; \ + auto end3 = end[3]; \ + auto dim3 = end3 - begin3; \ + auto dim2 = end2 - begin2; \ + auto dim1 = end1 - begin1; \ + auto dim0 = end0 - begin0; \ + auto nIter = dim3 * dim2 * dim1 * dim0; \ + /* clang-format off */ \ + KOKKOS_IMPL_ACC_PRAGMA(parallel loop gang vector reduction(OPERATOR:val) copyin(functor) async(async_arg)) \ + /* clang-format on */ \ + for (decltype(nIter) m = 0; m < nIter; ++m) { \ + auto tmp1 = dim3 * dim2 * dim1; \ + auto i0 = m / tmp1 + begin0; \ + auto tmp2 = m % tmp1; \ + tmp1 = dim3 * dim2; \ + auto i1 = tmp2 / tmp1 + begin1; \ + tmp2 = tmp2 % tmp1; \ + auto i2 = tmp2 / dim3 + begin2; \ + auto i3 = tmp2 % dim3 + begin3; \ + functor(i0, i1, i2, i3, val); \ + } \ + acc_wait(async_arg); \ + aval = val; \ + } \ + \ + template \ + void OpenACCParallelReduce##REDUCER(OpenACCIterateLeft, ValueType& aval, \ + Functor const& afunctor, \ + OpenACCMDRangeBegin<5> const& begin, \ + OpenACCMDRangeEnd<5> const& end, \ + int async_arg) { \ + auto val = aval; \ + auto const functor(afunctor); \ + auto begin4 = begin[4]; \ + auto end4 = end[4]; \ + auto begin3 = begin[3]; \ + auto end3 = end[3]; \ + auto begin2 = begin[2]; \ + auto end2 = end[2]; \ + auto begin1 = begin[1]; \ + auto end1 = end[1]; \ + auto begin0 = begin[0]; \ + auto end0 = end[0]; \ + auto dim4 = end4 - begin4; \ + auto dim3 = end3 - begin3; \ + auto dim2 = end2 - begin2; \ + auto dim1 = end1 - begin1; \ + auto dim0 = end0 - begin0; \ + auto nIter = dim4 * dim3 * dim2 * dim1 * dim0; \ + /* clang-format off */ \ + KOKKOS_IMPL_ACC_PRAGMA(parallel loop gang vector reduction(OPERATOR:val) copyin(functor) async(async_arg)) \ + /* clang-format on */ \ + for (decltype(nIter) m = 0; m < nIter; ++m) { \ + auto tmp1 = dim3 * dim2 * dim1 * dim0; \ + auto i4 = m / tmp1 + begin4; \ + auto tmp2 = m % tmp1; \ + tmp1 = dim2 * dim1 * dim0; \ + auto i3 = tmp2 / tmp1 + begin3; \ + tmp2 = tmp2 % tmp1; \ + tmp1 = dim1 * dim0; \ + auto i2 = tmp2 / tmp1 + begin2; \ + tmp2 = tmp2 % tmp1; \ + auto i1 = tmp2 / dim0 + begin1; \ + auto i0 = tmp2 % dim0 + begin0; \ + functor(i0, i1, i2, i3, i4, val); \ + } \ + acc_wait(async_arg); \ + aval = val; \ + } \ + \ + template \ + void OpenACCParallelReduce##REDUCER(OpenACCIterateRight, ValueType& aval, \ + Functor const& afunctor, \ + OpenACCMDRangeBegin<5> const& begin, \ + OpenACCMDRangeEnd<5> const& end, \ + int async_arg) { \ + auto val = aval; \ + auto const functor(afunctor); \ + auto begin0 = begin[0]; \ + auto end0 = end[0]; \ + auto begin1 = begin[1]; \ + auto end1 = end[1]; \ + auto begin2 = begin[2]; \ + auto end2 = end[2]; \ + auto begin3 = begin[3]; \ + auto end3 = end[3]; \ + auto begin4 = begin[4]; \ + auto end4 = end[4]; \ + auto dim4 = end4 - begin4; \ + auto dim3 = end3 - begin3; \ + auto dim2 = end2 - begin2; \ + auto dim1 = end1 - begin1; \ + auto dim0 = end0 - begin0; \ + auto nIter = dim4 * dim3 * dim2 * dim1 * dim0; \ + /* clang-format off */ \ + KOKKOS_IMPL_ACC_PRAGMA(parallel loop gang vector reduction(OPERATOR:val) copyin(functor) async(async_arg)) \ + /* clang-format on */ \ + for (decltype(nIter) m = 0; m < nIter; ++m) { \ + auto tmp1 = dim4 * dim3 * dim2 * dim1; \ + auto i0 = m / tmp1 + begin0; \ + auto tmp2 = m % tmp1; \ + tmp1 = dim4 * dim3 * dim2; \ + auto i1 = tmp2 / tmp1 + begin1; \ + tmp2 = tmp2 % tmp1; \ + tmp1 = dim4 * dim3; \ + auto i2 = tmp2 / tmp1 + begin2; \ + tmp2 = tmp2 % tmp1; \ + auto i3 = tmp2 / dim4 + begin3; \ + auto i4 = tmp2 % dim4 + begin4; \ + functor(i0, i1, i2, i3, i4, val); \ + } \ + acc_wait(async_arg); \ + aval = val; \ + } \ + \ + template \ + void OpenACCParallelReduce##REDUCER(OpenACCIterateLeft, ValueType& aval, \ + Functor const& afunctor, \ + OpenACCMDRangeBegin<6> const& begin, \ + OpenACCMDRangeEnd<6> const& end, \ + int async_arg) { \ + auto val = aval; \ + auto const functor(afunctor); \ + auto begin5 = begin[5]; \ + auto end5 = end[5]; \ + auto begin4 = begin[4]; \ + auto end4 = end[4]; \ + auto begin3 = begin[3]; \ + auto end3 = end[3]; \ + auto begin2 = begin[2]; \ + auto end2 = end[2]; \ + auto begin1 = begin[1]; \ + auto end1 = end[1]; \ + auto begin0 = begin[0]; \ + auto end0 = end[0]; \ + auto dim5 = end5 - begin5; \ + auto dim4 = end4 - begin4; \ + auto dim3 = end3 - begin3; \ + auto dim2 = end2 - begin2; \ + auto dim1 = end1 - begin1; \ + auto dim0 = end0 - begin0; \ + auto nIter = dim5 * dim4 * dim3 * dim2 * dim1 * dim0; \ + /* clang-format off */ \ + KOKKOS_IMPL_ACC_PRAGMA(parallel loop gang vector reduction(OPERATOR:val) copyin(functor) async(async_arg)) \ + /* clang-format on */ \ + for (decltype(nIter) m = 0; m < nIter; ++m) { \ + auto tmp1 = dim4 * dim3 * dim2 * dim1 * dim0; \ + auto i5 = m / tmp1 + begin5; \ + auto tmp2 = m % tmp1; \ + tmp1 = dim3 * dim2 * dim1 * dim0; \ + auto i4 = tmp2 / tmp1 + begin4; \ + tmp2 = tmp2 % tmp1; \ + tmp1 = dim2 * dim1 * dim0; \ + auto i3 = tmp2 / tmp1 + begin3; \ + tmp2 = tmp2 % tmp1; \ + tmp1 = dim1 * dim0; \ + auto i2 = tmp2 / tmp1 + begin2; \ + tmp2 = tmp2 % tmp1; \ + auto i1 = tmp2 / dim0 + begin1; \ + auto i0 = tmp2 % dim0 + begin0; \ + functor(i0, i1, i2, i3, i4, i5, val); \ + } \ + acc_wait(async_arg); \ + aval = val; \ + } \ + \ + template \ + void OpenACCParallelReduce##REDUCER(OpenACCIterateRight, ValueType& aval, \ + Functor const& afunctor, \ + OpenACCMDRangeBegin<6> const& begin, \ + OpenACCMDRangeEnd<6> const& end, \ + int async_arg) { \ + auto val = aval; \ + auto const functor(afunctor); \ + auto begin0 = begin[0]; \ + auto end0 = end[0]; \ + auto begin1 = begin[1]; \ + auto end1 = end[1]; \ + auto begin2 = begin[2]; \ + auto end2 = end[2]; \ + auto begin3 = begin[3]; \ + auto end3 = end[3]; \ + auto begin4 = begin[4]; \ + auto end4 = end[4]; \ + auto begin5 = begin[5]; \ + auto end5 = end[5]; \ + auto dim5 = end5 - begin5; \ + auto dim4 = end4 - begin4; \ + auto dim3 = end3 - begin3; \ + auto dim2 = end2 - begin2; \ + auto dim1 = end1 - begin1; \ + auto dim0 = end0 - begin0; \ + auto nIter = dim5 * dim4 * dim3 * dim2 * dim1 * dim0; \ + /* clang-format off */ \ + KOKKOS_IMPL_ACC_PRAGMA(parallel loop gang vector reduction(OPERATOR:val) copyin(functor) async(async_arg)) \ + /* clang-format on */ \ + for (decltype(nIter) m = 0; m < nIter; ++m) { \ + auto tmp1 = dim5 * dim4 * dim3 * dim2 * dim1; \ + auto i0 = m / tmp1 + begin0; \ + auto tmp2 = m % tmp1; \ + tmp1 = dim5 * dim4 * dim3 * dim2; \ + auto i1 = tmp2 / tmp1 + begin1; \ + tmp2 = tmp2 % tmp1; \ + tmp1 = dim5 * dim4 * dim3; \ + auto i2 = tmp2 / tmp1 + begin2; \ + tmp2 = tmp2 % tmp1; \ + tmp1 = dim5 * dim4; \ + auto i3 = tmp2 / tmp1 + begin3; \ + tmp2 = tmp2 % tmp1; \ + auto i4 = tmp2 / dim5 + begin4; \ + auto i5 = tmp2 % dim5 + begin5; \ + functor(i0, i1, i2, i3, i4, i5, val); \ + } \ + acc_wait(async_arg); \ + aval = val; \ + } \ + } // namespace Kokkos::Experimental::Impl + +#else + #define KOKKOS_IMPL_OPENACC_PARALLEL_REDUCE_DISPATCH_ITERATE(REDUCER, \ OPERATOR) \ namespace Kokkos::Experimental::Impl { \ @@ -124,10 +522,10 @@ class Kokkos::Impl::ParallelReduce \ diff --git a/lib/kokkos/core/src/OpenACC/Kokkos_OpenACC_ParallelReduce_Team.hpp b/lib/kokkos/core/src/OpenACC/Kokkos_OpenACC_ParallelReduce_Team.hpp index 430bdcb680..d4cb73164d 100644 --- a/lib/kokkos/core/src/OpenACC/Kokkos_OpenACC_ParallelReduce_Team.hpp +++ b/lib/kokkos/core/src/OpenACC/Kokkos_OpenACC_ParallelReduce_Team.hpp @@ -163,13 +163,24 @@ KOKKOS_INLINE_FUNCTION std::enable_if_t> parallel_reduce(const Impl::TeamThreadRangeBoundariesStruct< iType, Impl::OpenACCTeamMember>& loop_boundaries, const Lambda& lambda, ValueType& result) { - ValueType tmp = ValueType(); + using functor_analysis_type = typename Impl::FunctorAnalysis< + Impl::FunctorPatternInterface::REDUCE, + TeamPolicy, Lambda, + ValueType>; + using wrapped_reducer_type = typename functor_analysis_type::Reducer; + using value_type = typename wrapped_reducer_type::value_type; + + wrapped_reducer_type wrapped_reducer(lambda); + value_type tmp; + wrapped_reducer.init(&tmp); + iType j_start = loop_boundaries.team.team_rank() / loop_boundaries.team.vector_length(); if (j_start == 0) { #pragma acc loop seq for (iType i = loop_boundaries.start; i < loop_boundaries.end; i++) lambda(i, tmp); + wrapped_reducer.final(&tmp); result = tmp; } } @@ -180,15 +191,25 @@ KOKKOS_INLINE_FUNCTION std::enable_if_t> parallel_reduce(const Impl::TeamThreadRangeBoundariesStruct< iType, Impl::OpenACCTeamMember>& loop_boundaries, const Lambda& lambda, const ReducerType& reducer) { - using ValueType = typename ReducerType::value_type; - ValueType tmp; - reducer.init(tmp); + using value_type = typename ReducerType::value_type; + using functor_analysis_type = typename Impl::FunctorAnalysis< + Impl::FunctorPatternInterface::REDUCE, + TeamPolicy, + ReducerType, value_type>; + using wrapped_reducer_type = typename functor_analysis_type::Reducer; + + wrapped_reducer_type wrapped_reducer(reducer); + value_type tmp; + wrapped_reducer.init(&tmp); + iType j_start = loop_boundaries.team.team_rank() / loop_boundaries.team.vector_length(); if (j_start == 0) { #pragma acc loop seq for (iType i = loop_boundaries.start; i < loop_boundaries.end; i++) lambda(i, tmp); + + wrapped_reducer.final(&tmp); reducer.reference() = tmp; } } @@ -200,7 +221,17 @@ KOKKOS_INLINE_FUNCTION std::enable_if_t> parallel_reduce(const Impl::ThreadVectorRangeBoundariesStruct< iType, Impl::OpenACCTeamMember>& loop_boundaries, const Lambda& lambda, ValueType& result) { - ValueType tmp = ValueType(); + using functor_analysis_type = typename Impl::FunctorAnalysis< + Impl::FunctorPatternInterface::REDUCE, + TeamPolicy, Lambda, + ValueType>; + using wrapped_reducer_type = typename functor_analysis_type::Reducer; + using value_type = typename wrapped_reducer_type::value_type; + + wrapped_reducer_type wrapped_reducer(lambda); + value_type tmp; + wrapped_reducer.init(&tmp); + iType j_start = loop_boundaries.team.team_rank() % loop_boundaries.team.vector_length(); if (j_start == 0) { @@ -208,6 +239,7 @@ parallel_reduce(const Impl::ThreadVectorRangeBoundariesStruct< for (iType i = loop_boundaries.start; i < loop_boundaries.end; i++) { lambda(i, tmp); } + wrapped_reducer.final(&tmp); result = tmp; } } @@ -218,9 +250,17 @@ KOKKOS_INLINE_FUNCTION std::enable_if_t> parallel_reduce(const Impl::ThreadVectorRangeBoundariesStruct< iType, Impl::OpenACCTeamMember>& loop_boundaries, const Lambda& lambda, const ReducerType& reducer) { - using ValueType = typename ReducerType::value_type; - ValueType tmp; - reducer.init(tmp); + using value_type = typename ReducerType::value_type; + using functor_analysis_type = typename Impl::FunctorAnalysis< + Impl::FunctorPatternInterface::REDUCE, + TeamPolicy, + ReducerType, value_type>; + using wrapped_reducer_type = typename functor_analysis_type::Reducer; + + wrapped_reducer_type wrapped_reducer(reducer); + value_type tmp; + wrapped_reducer.init(&tmp); + iType j_start = loop_boundaries.team.team_rank() % loop_boundaries.team.vector_length(); if (j_start == 0) { @@ -228,6 +268,8 @@ parallel_reduce(const Impl::ThreadVectorRangeBoundariesStruct< for (iType i = loop_boundaries.start; i < loop_boundaries.end; i++) { lambda(i, tmp); } + + wrapped_reducer.final(&tmp); reducer.reference() = tmp; } } @@ -239,7 +281,17 @@ KOKKOS_INLINE_FUNCTION void parallel_reduce( const Impl::TeamVectorRangeBoundariesStruct& loop_boundaries, const Lambda& lambda, ValueType& result) { - ValueType tmp = ValueType(); + using functor_analysis_type = typename Impl::FunctorAnalysis< + Impl::FunctorPatternInterface::REDUCE, + TeamPolicy, Lambda, + ValueType>; + using wrapped_reducer_type = typename functor_analysis_type::Reducer; + using value_type = typename wrapped_reducer_type::value_type; + + wrapped_reducer_type wrapped_reducer(lambda); + value_type tmp; + wrapped_reducer.init(&tmp); + iType j_start = loop_boundaries.team.team_rank() % loop_boundaries.team.vector_length(); if (j_start == 0) { @@ -247,6 +299,7 @@ KOKKOS_INLINE_FUNCTION void parallel_reduce( for (iType i = loop_boundaries.start; i < loop_boundaries.end; i++) { lambda(i, tmp); } + wrapped_reducer.final(&tmp); result = tmp; } } @@ -273,10 +326,23 @@ KOKKOS_INLINE_FUNCTION std::enable_if_t> parallel_reduce(const Impl::TeamThreadRangeBoundariesStruct< iType, Impl::OpenACCTeamMember>& loop_boundaries, const Lambda& lambda, ValueType& result) { + using functor_analysis_type = typename Impl::FunctorAnalysis< + Impl::FunctorPatternInterface::REDUCE, + TeamPolicy, Lambda, + ValueType>; + using wrapped_reducer_type = typename functor_analysis_type::Reducer; + using value_type = typename wrapped_reducer_type::value_type; + + wrapped_reducer_type wrapped_reducer(lambda); + value_type tmp; + wrapped_reducer.init(&tmp); + ValueType tmp = ValueType(); #pragma acc loop worker reduction(+ : tmp) for (iType i = loop_boundaries.start; i < loop_boundaries.end; i++) lambda(i, tmp); + + wrapped_reducer.final(&tmp); result = tmp; } @@ -314,11 +380,22 @@ KOKKOS_INLINE_FUNCTION std::enable_if_t> parallel_reduce(const Impl::ThreadVectorRangeBoundariesStruct< iType, Impl::OpenACCTeamMember>& loop_boundaries, const Lambda& lambda, ValueType& result) { - ValueType tmp = ValueType(); + using functor_analysis_type = typename Impl::FunctorAnalysis< + Impl::FunctorPatternInterface::REDUCE, + TeamPolicy, Lambda, + ValueType>; + using wrapped_reducer_type = typename functor_analysis_type::Reducer; + using value_type = typename wrapped_reducer_type::value_type; + + wrapped_reducer_type wrapped_reducer(lambda); + value_type tmp; + wrapped_reducer.init(&tmp); + #pragma acc loop vector reduction(+ : tmp) for (iType i = loop_boundaries.start; i < loop_boundaries.end; i++) { lambda(i, tmp); } + wrapped_reducer.final(&tmp); result = tmp; } @@ -357,11 +434,23 @@ KOKKOS_INLINE_FUNCTION void parallel_reduce( const Impl::TeamVectorRangeBoundariesStruct& loop_boundaries, const Lambda& lambda, ValueType& result) { - ValueType tmp = ValueType(); + using functor_analysis_type = typename Impl::FunctorAnalysis< + Impl::FunctorPatternInterface::REDUCE, + TeamPolicy, Lambda, + ValueType>; + using wrapped_reducer_type = typename functor_analysis_type::Reducer; + using value_type = typename wrapped_reducer_type::value_type; + + wrapped_reducer_type wrapped_reducer(lambda); + value_type tmp; + wrapped_reducer.init(&tmp); + #pragma acc loop vector reduction(+ : tmp) for (iType i = loop_boundaries.start; i < loop_boundaries.end; i++) { lambda(i, tmp); } + + wrapped_reducer.final(&tmp); result = tmp; } diff --git a/lib/kokkos/core/src/OpenACC/Kokkos_OpenACC_ParallelScan_Range.hpp b/lib/kokkos/core/src/OpenACC/Kokkos_OpenACC_ParallelScan_Range.hpp index c6d3267bdb..b1c48baa1e 100644 --- a/lib/kokkos/core/src/OpenACC/Kokkos_OpenACC_ParallelScan_Range.hpp +++ b/lib/kokkos/core/src/OpenACC/Kokkos_OpenACC_ParallelScan_Range.hpp @@ -225,7 +225,7 @@ KOKKOS_IMPL_ACC_PRAGMA(parallel loop gang vector_length(chunk_size) KOKKOS_IMPL_ } #pragma acc exit data delete (functor, chunk_values, offset_values, \ - final_reducer)async(async_arg) + final_reducer)async(async_arg) acc_wait(async_arg); } diff --git a/lib/kokkos/core/src/OpenACC/Kokkos_OpenACC_Traits.hpp b/lib/kokkos/core/src/OpenACC/Kokkos_OpenACC_Traits.hpp index faa50aa7c3..95526aa784 100644 --- a/lib/kokkos/core/src/OpenACC/Kokkos_OpenACC_Traits.hpp +++ b/lib/kokkos/core/src/OpenACC/Kokkos_OpenACC_Traits.hpp @@ -28,8 +28,11 @@ struct OpenACC_Traits { #elif defined(KOKKOS_ARCH_AMD_GPU) static constexpr acc_device_t dev_type = acc_device_radeon; static constexpr bool may_fallback_to_host = false; +#elif defined(KOKKOS_ENABLE_OPENACC_FORCE_HOST_AS_DEVICE) + static constexpr acc_device_t dev_type = acc_device_host; + static constexpr bool may_fallback_to_host = true; #else - static constexpr acc_device_t dev_type = acc_device_not_host; + static constexpr acc_device_t dev_type = acc_device_default; static constexpr bool may_fallback_to_host = true; #endif }; diff --git a/lib/kokkos/core/src/OpenMP/Kokkos_OpenMP.hpp b/lib/kokkos/core/src/OpenMP/Kokkos_OpenMP.hpp index a403909f67..aa4be87ceb 100644 --- a/lib/kokkos/core/src/OpenMP/Kokkos_OpenMP.hpp +++ b/lib/kokkos/core/src/OpenMP/Kokkos_OpenMP.hpp @@ -30,7 +30,6 @@ static_assert(false, #include #include #include -#include #include #include #include @@ -93,11 +92,16 @@ class OpenMP { void fence(std::string const& name = "Kokkos::OpenMP::fence: Unnamed Instance Fence") const; +#ifdef KOKKOS_ENABLE_DEPRECATED_CODE_4 /// \brief Does the given instance return immediately after launching /// a parallel algorithm /// /// This always returns false on OpenMP - inline static bool is_asynchronous(OpenMP const& = OpenMP()) noexcept; + KOKKOS_DEPRECATED inline static bool is_asynchronous( + OpenMP const& = OpenMP()) noexcept { + return false; + } +#endif #ifdef KOKKOS_ENABLE_DEPRECATED_CODE_4 static int concurrency(OpenMP const& = OpenMP()); @@ -154,10 +158,6 @@ inline int OpenMP::impl_thread_pool_rank() noexcept { KOKKOS_IF_ON_DEVICE((return -1;)) } -inline bool OpenMP::is_asynchronous(OpenMP const& /*instance*/) noexcept { - return false; -} - inline int OpenMP::impl_thread_pool_size(int depth) const { return depth < 2 ? impl_thread_pool_size() : 1; } @@ -202,7 +202,9 @@ struct MemorySpaceAccess #include +#ifdef KOKKOS_ENABLE_DEPRECATED_CODE_4 #include +#endif #include /*--------------------------------------------------------------------------*/ diff --git a/lib/kokkos/core/src/OpenMP/Kokkos_OpenMP_Task.hpp b/lib/kokkos/core/src/OpenMP/Kokkos_OpenMP_Task.hpp index 2877d940fa..6edcbff0c2 100644 --- a/lib/kokkos/core/src/OpenMP/Kokkos_OpenMP_Task.hpp +++ b/lib/kokkos/core/src/OpenMP/Kokkos_OpenMP_Task.hpp @@ -26,12 +26,19 @@ #include #include +#include + #include #include //---------------------------------------------------------------------------- //---------------------------------------------------------------------------- +#ifdef KOKKOS_ENABLE_DEPRECATION_WARNINGS +// We allow using deprecated classes in this file +KOKKOS_IMPL_DISABLE_DEPRECATED_WARNINGS_PUSH() +#endif + namespace Kokkos { namespace Impl { @@ -360,6 +367,10 @@ extern template class TaskQueue #include #include -#include #include #include #include @@ -148,7 +147,6 @@ struct DeviceTypeTraits<::Kokkos::Experimental::OpenMPTarget> { #include #include #include -#include /*--------------------------------------------------------------------------*/ diff --git a/lib/kokkos/core/src/OpenMPTarget/Kokkos_OpenMPTargetSpace.hpp b/lib/kokkos/core/src/OpenMPTarget/Kokkos_OpenMPTargetSpace.hpp index ed625cfcc8..ec33d25b96 100644 --- a/lib/kokkos/core/src/OpenMPTarget/Kokkos_OpenMPTargetSpace.hpp +++ b/lib/kokkos/core/src/OpenMPTarget/Kokkos_OpenMPTargetSpace.hpp @@ -28,6 +28,7 @@ static_assert(false, #include #include +#include #ifdef KOKKOS_ENABLE_OPENMPTARGET @@ -91,9 +92,9 @@ class OpenMPTargetSpace { /**\brief Default memory space instance */ OpenMPTargetSpace(); - OpenMPTargetSpace(OpenMPTargetSpace&& rhs) = default; - OpenMPTargetSpace(const OpenMPTargetSpace& rhs) = default; - OpenMPTargetSpace& operator=(OpenMPTargetSpace&&) = default; + OpenMPTargetSpace(OpenMPTargetSpace&& rhs) = default; + OpenMPTargetSpace(const OpenMPTargetSpace& rhs) = default; + OpenMPTargetSpace& operator=(OpenMPTargetSpace&&) = default; OpenMPTargetSpace& operator=(const OpenMPTargetSpace&) = default; ~OpenMPTargetSpace() = default; @@ -141,79 +142,5 @@ class OpenMPTargetSpace { KOKKOS_IMPL_HOST_INACCESSIBLE_SHARED_ALLOCATION_SPECIALIZATION( Kokkos::Experimental::OpenMPTargetSpace); -//---------------------------------------------------------------------------- -//---------------------------------------------------------------------------- - -namespace Kokkos { -namespace Impl { - -// TODO: implement all possible deep_copies -template -struct DeepCopy { - DeepCopy(void* dst, const void* src, size_t n) { - // In the Release and RelWithDebInfo builds, the size of the memcpy should - // be greater than zero to avoid error. omp_target_memcpy returns zero on - // success. - if (n > 0) - KOKKOS_IMPL_OMPT_SAFE_CALL(omp_target_memcpy( - dst, const_cast(src), n, 0, 0, omp_get_default_device(), - omp_get_default_device())); - } - DeepCopy(const ExecutionSpace& exec, void* dst, const void* src, size_t n) { - exec.fence( - "Kokkos::Impl::DeepCopy: fence " - "before " - "copy"); - if (n > 0) - KOKKOS_IMPL_OMPT_SAFE_CALL(omp_target_memcpy( - dst, const_cast(src), n, 0, 0, omp_get_default_device(), - omp_get_default_device())); - } -}; - -template -struct DeepCopy { - DeepCopy(void* dst, const void* src, size_t n) { - if (n > 0) - KOKKOS_IMPL_OMPT_SAFE_CALL(omp_target_memcpy( - dst, const_cast(src), n, 0, 0, omp_get_default_device(), - omp_get_initial_device())); - } - DeepCopy(const ExecutionSpace& exec, void* dst, const void* src, size_t n) { - exec.fence( - "Kokkos::Impl::DeepCopy: fence before " - "copy"); - if (n > 0) - KOKKOS_IMPL_OMPT_SAFE_CALL(omp_target_memcpy( - dst, const_cast(src), n, 0, 0, omp_get_default_device(), - omp_get_initial_device())); - } -}; - -template -struct DeepCopy { - DeepCopy(void* dst, const void* src, size_t n) { - if (n > 0) - KOKKOS_IMPL_OMPT_SAFE_CALL(omp_target_memcpy( - dst, const_cast(src), n, 0, 0, omp_get_initial_device(), - omp_get_default_device())); - } - DeepCopy(const ExecutionSpace& exec, void* dst, const void* src, size_t n) { - exec.fence( - "Kokkos::Impl::DeepCopy: fence before " - "copy"); - if (n > 0) - KOKKOS_IMPL_OMPT_SAFE_CALL(omp_target_memcpy( - dst, const_cast(src), n, 0, 0, omp_get_initial_device(), - omp_get_default_device())); - } -}; - -} // namespace Impl -} // namespace Kokkos - #endif #endif /* #define KOKKOS_OPENMPTARGETSPACE_HPP */ diff --git a/lib/kokkos/core/src/OpenMPTarget/Kokkos_OpenMPTarget_DeepCopy.hpp b/lib/kokkos/core/src/OpenMPTarget/Kokkos_OpenMPTarget_DeepCopy.hpp new file mode 100644 index 0000000000..aace09e266 --- /dev/null +++ b/lib/kokkos/core/src/OpenMPTarget/Kokkos_OpenMPTarget_DeepCopy.hpp @@ -0,0 +1,101 @@ +//@HEADER +// ************************************************************************ +// +// Kokkos v. 4.0 +// Copyright (2022) National Technology & Engineering +// Solutions of Sandia, LLC (NTESS). +// +// Under the terms of Contract DE-NA0003525 with NTESS, +// the U.S. Government retains certain rights in this software. +// +// Part of Kokkos, under the Apache License v2.0 with LLVM Exceptions. +// See https://kokkos.org/LICENSE for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//@HEADER + +#ifndef KOKKOS_IMPL_PUBLIC_INCLUDE +#include +static_assert(false, + "Including non-public Kokkos header files is not allowed."); +#endif +#ifndef KOKKOS_OPENMPTARGET_DEEP_COPY_HPP +#define KOKKOS_OPENMPTARGET_DEEP_COPY_HPP + +#include + +//---------------------------------------------------------------------------- +//---------------------------------------------------------------------------- + +namespace Kokkos { +namespace Impl { + +// TODO: implement all possible deep_copies +template +struct DeepCopy { + DeepCopy(void* dst, const void* src, size_t n) { + // In the Release and RelWithDebInfo builds, the size of the memcpy should + // be greater than zero to avoid error. omp_target_memcpy returns zero on + // success. + if (n > 0) + KOKKOS_IMPL_OMPT_SAFE_CALL(omp_target_memcpy( + dst, const_cast(src), n, 0, 0, omp_get_default_device(), + omp_get_default_device())); + } + DeepCopy(const ExecutionSpace& exec, void* dst, const void* src, size_t n) { + exec.fence( + "Kokkos::Impl::DeepCopy: fence " + "before " + "copy"); + if (n > 0) + KOKKOS_IMPL_OMPT_SAFE_CALL(omp_target_memcpy( + dst, const_cast(src), n, 0, 0, omp_get_default_device(), + omp_get_default_device())); + } +}; + +template +struct DeepCopy { + DeepCopy(void* dst, const void* src, size_t n) { + if (n > 0) + KOKKOS_IMPL_OMPT_SAFE_CALL(omp_target_memcpy( + dst, const_cast(src), n, 0, 0, omp_get_default_device(), + omp_get_initial_device())); + } + DeepCopy(const ExecutionSpace& exec, void* dst, const void* src, size_t n) { + exec.fence( + "Kokkos::Impl::DeepCopy: fence before " + "copy"); + if (n > 0) + KOKKOS_IMPL_OMPT_SAFE_CALL(omp_target_memcpy( + dst, const_cast(src), n, 0, 0, omp_get_default_device(), + omp_get_initial_device())); + } +}; + +template +struct DeepCopy { + DeepCopy(void* dst, const void* src, size_t n) { + if (n > 0) + KOKKOS_IMPL_OMPT_SAFE_CALL(omp_target_memcpy( + dst, const_cast(src), n, 0, 0, omp_get_initial_device(), + omp_get_default_device())); + } + DeepCopy(const ExecutionSpace& exec, void* dst, const void* src, size_t n) { + exec.fence( + "Kokkos::Impl::DeepCopy: fence before " + "copy"); + if (n > 0) + KOKKOS_IMPL_OMPT_SAFE_CALL(omp_target_memcpy( + dst, const_cast(src), n, 0, 0, omp_get_initial_device(), + omp_get_default_device())); + } +}; + +} // namespace Impl +} // namespace Kokkos + +#endif // KOKKOS_OPENMPTARGET_DEEP_COPY_HPP diff --git a/lib/kokkos/core/src/OpenMPTarget/Kokkos_OpenMPTarget_Exec.cpp b/lib/kokkos/core/src/OpenMPTarget/Kokkos_OpenMPTarget_Exec.cpp deleted file mode 100644 index 6c5eb048e3..0000000000 --- a/lib/kokkos/core/src/OpenMPTarget/Kokkos_OpenMPTarget_Exec.cpp +++ /dev/null @@ -1,130 +0,0 @@ -//@HEADER -// ************************************************************************ -// -// Kokkos v. 4.0 -// Copyright (2022) National Technology & Engineering -// Solutions of Sandia, LLC (NTESS). -// -// Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. -// -// Part of Kokkos, under the Apache License v2.0 with LLVM Exceptions. -// See https://kokkos.org/LICENSE for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// -//@HEADER - -#ifndef KOKKOS_IMPL_PUBLIC_INCLUDE -#define KOKKOS_IMPL_PUBLIC_INCLUDE -#endif - -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#ifdef KOKKOS_ENABLE_OPENMPTARGET - -// FIXME_OPENMPTARGET currently unused -/* -namespace Kokkos { -namespace Impl { -namespace { - -KOKKOS_INLINE_FUNCTION -int kokkos_omp_in_parallel(); - -KOKKOS_INLINE_FUNCTION -int kokkos_omp_in_parallel() { return omp_in_parallel(); } - -bool s_using_hwloc = false; - -} // namespace -} // namespace Impl -} // namespace Kokkos -*/ - -namespace Kokkos { -namespace Impl { - -void OpenMPTargetExec::verify_is_process(const char* const label) { - // Fails if the current task is in a parallel region or is not on the host. - if (omp_in_parallel() && (!omp_is_initial_device())) { - std::string msg(label); - msg.append(" ERROR: in parallel or on device"); - Kokkos::Impl::throw_runtime_exception(msg); - } -} - -void OpenMPTargetExec::verify_initialized(const char* const label) { - if (0 == Kokkos::Experimental::OpenMPTarget().impl_is_initialized()) { - std::string msg(label); - msg.append(" ERROR: not initialized"); - Kokkos::Impl::throw_runtime_exception(msg); - } -} - -void* OpenMPTargetExec::m_scratch_ptr = nullptr; -int64_t OpenMPTargetExec::m_scratch_size = 0; -uint32_t* OpenMPTargetExec::m_uniquetoken_ptr = nullptr; -int OpenMPTargetExec::MAX_ACTIVE_THREADS = 0; -std::mutex OpenMPTargetExec::m_mutex_scratch_ptr; - -void OpenMPTargetExec::clear_scratch() { - Kokkos::Experimental::OpenMPTargetSpace space; - space.deallocate(m_scratch_ptr, m_scratch_size); - m_scratch_ptr = nullptr; - m_scratch_size = 0; -} - -void* OpenMPTargetExec::get_scratch_ptr() { return m_scratch_ptr; } - -void OpenMPTargetExec::resize_scratch(int64_t team_size, int64_t shmem_size_L0, - int64_t shmem_size_L1, - int64_t league_size) { - Kokkos::Experimental::OpenMPTargetSpace space; - // Level-0 scratch when using clang/17 and higher comes from their OpenMP - // extension, `ompx_dyn_cgroup_mem`. -#if defined(KOKKOS_IMPL_OPENMPTARGET_LLVM_EXTENSIONS) - shmem_size_L0 = 0; -#endif - const int64_t shmem_size = - shmem_size_L0 + shmem_size_L1; // L0 + L1 scratch memory per team. - const int64_t padding = shmem_size * 10 / 100; // Padding per team. - - // Maximum active teams possible. - // The number should not exceed the maximum in-flight teams possible or the - // league_size. - int max_active_teams = - std::min(OpenMPTargetExec::MAX_ACTIVE_THREADS / team_size, league_size); - - // max_active_teams is the number of active teams on the given hardware. - // We set the number of teams to be twice the number of max_active_teams for - // the compiler to pick the right number in its case. - // FIXME_OPENMPTARGET: Cray compiler did not yet implement omp_set_num_teams. -#if !defined(KOKKOS_COMPILER_CRAY_LLVM) - omp_set_num_teams(max_active_teams * 2); -#endif - - // Total amount of scratch memory allocated is depenedent - // on the maximum number of in-flight teams possible. - int64_t total_size = - (shmem_size + OpenMPTargetExecTeamMember::TEAM_REDUCE_SIZE + padding) * - max_active_teams * 2; - - if (total_size > m_scratch_size) { - space.deallocate(m_scratch_ptr, m_scratch_size); - m_scratch_size = total_size; - m_scratch_ptr = space.allocate(total_size); - } -} - -} // namespace Impl -} // namespace Kokkos - -#endif // KOKKOS_ENABLE_OPENMPTARGET diff --git a/lib/kokkos/core/src/OpenMPTarget/Kokkos_OpenMPTarget_FunctorAdapter.hpp b/lib/kokkos/core/src/OpenMPTarget/Kokkos_OpenMPTarget_FunctorAdapter.hpp new file mode 100644 index 0000000000..13b509c0ad --- /dev/null +++ b/lib/kokkos/core/src/OpenMPTarget/Kokkos_OpenMPTarget_FunctorAdapter.hpp @@ -0,0 +1,48 @@ +//@HEADER +// ************************************************************************ +// +// Kokkos v. 4.0 +// Copyright (2022) National Technology & Engineering +// Solutions of Sandia, LLC (NTESS). +// +// Under the terms of Contract DE-NA0003525 with NTESS, +// the U.S. Government retains certain rights in this software. +// +// Part of Kokkos, under the Apache License v2.0 with LLVM Exceptions. +// See https://kokkos.org/LICENSE for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//@HEADER + +#ifndef KOKKOS_OPENMPTARGET_FUNCTOR_ADAPTER_HPP +#define KOKKOS_OPENMPTARGET_FUNCTOR_ADAPTER_HPP + +#include +#include + +namespace Kokkos::Experimental::Impl { + +template +class FunctorAdapter { + Functor m_functor; + using WorkTag = typename Policy::work_tag; + + public: + FunctorAdapter() = default; + FunctorAdapter(Functor const &functor) : m_functor(functor) {} + + Functor get_functor() const { return m_functor; } + + template + KOKKOS_FUNCTION void operator()(Args &&...args) const { + if constexpr (std::is_void_v) { + m_functor(static_cast(args)...); + } else { + m_functor(WorkTag(), static_cast(args)...); + } + } +}; + +} // namespace Kokkos::Experimental::Impl + +#endif // KOKKOS_OPENMPTARGET_FUNCTOR_ADAPTER_HPP diff --git a/lib/kokkos/core/src/OpenMPTarget/Kokkos_OpenMPTarget_Instance.cpp b/lib/kokkos/core/src/OpenMPTarget/Kokkos_OpenMPTarget_Instance.cpp index 44e9119ea8..53e723882f 100644 --- a/lib/kokkos/core/src/OpenMPTarget/Kokkos_OpenMPTarget_Instance.cpp +++ b/lib/kokkos/core/src/OpenMPTarget/Kokkos_OpenMPTarget_Instance.cpp @@ -27,11 +27,11 @@ // constructor. undef'ed at the end #define KOKKOS_IMPL_OPENMPTARGET_WORKAROUND +#include #include #include #include #include -#include #include @@ -105,18 +105,15 @@ void OpenMPTargetInternal::print_configuration(std::ostream& os, void OpenMPTargetInternal::impl_finalize() { m_is_initialized = false; - Kokkos::Impl::OpenMPTargetExec space; - if (space.m_uniquetoken_ptr != nullptr) + if (m_uniquetoken_ptr != nullptr) Kokkos::kokkos_free( - space.m_uniquetoken_ptr); + m_uniquetoken_ptr); } void OpenMPTargetInternal::impl_initialize() { m_is_initialized = true; - Kokkos::Impl::OpenMPTargetExec::MAX_ACTIVE_THREADS = concurrency(); - // FIXME_OPENMPTARGET: Only fix the number of teams for NVIDIA architectures // from Pascal and upwards. // FIXME_OPENMPTARGTE: Cray compiler did not yet implement omp_set_num_teams. @@ -136,7 +133,75 @@ OpenMPTargetInternal* OpenMPTargetInternal::impl_singleton() { return &self; } -} // Namespace Impl +void OpenMPTargetInternal::verify_is_process(const char* const label) { + // Fails if the current task is in a parallel region or is not on the host. + if (omp_in_parallel() && (!omp_is_initial_device())) { + std::string msg(label); + msg.append(" ERROR: in parallel or on device"); + Kokkos::Impl::throw_runtime_exception(msg); + } +} + +void OpenMPTargetInternal::verify_initialized(const char* const label) { + if (0 == Kokkos::Experimental::OpenMPTarget().impl_is_initialized()) { + std::string msg(label); + msg.append(" ERROR: not initialized"); + Kokkos::Impl::throw_runtime_exception(msg); + } +} + +void OpenMPTargetInternal::clear_scratch() { + Kokkos::Experimental::OpenMPTargetSpace space; + space.deallocate(m_scratch_ptr, m_scratch_size); + m_scratch_ptr = nullptr; + m_scratch_size = 0; +} + +void* OpenMPTargetInternal::get_scratch_ptr() { return m_scratch_ptr; } + +void OpenMPTargetInternal::resize_scratch(int64_t team_size, + int64_t shmem_size_L0, + int64_t shmem_size_L1, + int64_t league_size) { + Kokkos::Experimental::OpenMPTargetSpace space; + // Level-0 scratch when using clang/17 and higher comes from their OpenMP + // extension, `ompx_dyn_cgroup_mem`. +#if defined(KOKKOS_IMPL_OPENMPTARGET_LLVM_EXTENSIONS) + shmem_size_L0 = 0; +#endif + const int64_t shmem_size = + shmem_size_L0 + shmem_size_L1; // L0 + L1 scratch memory per team. + const int64_t padding = shmem_size * 10 / 100; // Padding per team. + + // Maximum active teams possible. + // The number should not exceed the maximum in-flight teams possible or the + // league_size. + int max_active_teams = + std::min(OpenMPTargetInternal::concurrency() / team_size, league_size); + + // max_active_teams is the number of active teams on the given hardware. + // We set the number of teams to be twice the number of max_active_teams for + // the compiler to pick the right number in its case. + // FIXME_OPENMPTARGET: Cray compiler did not yet implement omp_set_num_teams. +#if !defined(KOKKOS_COMPILER_CRAY_LLVM) + omp_set_num_teams(max_active_teams * 2); +#endif + + // Total amount of scratch memory allocated is depenedent + // on the maximum number of in-flight teams possible. + int64_t total_size = + (shmem_size + + ::Kokkos::Impl::OpenMPTargetExecTeamMember::TEAM_REDUCE_SIZE + padding) * + max_active_teams * 2; + + if (total_size > m_scratch_size) { + space.deallocate(m_scratch_ptr, m_scratch_size); + m_scratch_size = total_size; + m_scratch_ptr = space.allocate(total_size); + } +} + +} // namespace Impl OpenMPTarget::OpenMPTarget() : m_space_instance(Impl::OpenMPTargetInternal::impl_singleton()) {} @@ -206,9 +271,9 @@ namespace Experimental { UniqueToken:: - UniqueToken(Kokkos::Experimental::OpenMPTarget const&) { + UniqueToken(Kokkos::Experimental::OpenMPTarget const& space) { #ifdef KOKKOS_IMPL_OPENMPTARGET_WORKAROUND - uint32_t* ptr = Kokkos::Impl::OpenMPTargetExec::m_uniquetoken_ptr; + uint32_t* ptr = space.impl_internal_space_instance()->m_uniquetoken_ptr; int count = Kokkos::Experimental::OpenMPTarget().concurrency(); if (ptr == nullptr) { int size = count * sizeof(uint32_t); @@ -221,7 +286,7 @@ UniqueTokenm_uniquetoken_ptr = ptr; } #else // FIXME_OPENMPTARGET - 2 versions of non-working implementations to fill `ptr` @@ -229,8 +294,7 @@ UniqueToken - namespace Kokkos { namespace Experimental { namespace Impl { @@ -27,9 +25,9 @@ enum class openmp_fence_is_static { yes, no }; class OpenMPTargetInternal { private: - OpenMPTargetInternal() = default; - OpenMPTargetInternal(const OpenMPTargetInternal&) = default; - OpenMPTargetInternal& operator=(const OpenMPTargetInternal&) = default; + OpenMPTargetInternal() = default; + OpenMPTargetInternal(const OpenMPTargetInternal&) = delete; + OpenMPTargetInternal& operator=(const OpenMPTargetInternal&) = delete; public: void fence(openmp_fence_is_static is_static = openmp_fence_is_static::no); @@ -55,6 +53,19 @@ class OpenMPTargetInternal { static OpenMPTargetInternal* impl_singleton(); + static void verify_is_process(const char* const); + static void verify_initialized(const char* const); + + void* get_scratch_ptr(); + void clear_scratch(); + void resize_scratch(int64_t team_reduce_bytes, int64_t team_shared_bytes, + int64_t thread_local_bytes, int64_t league_size); + + void* m_scratch_ptr = nullptr; + std::mutex m_mutex_scratch_ptr; + int64_t m_scratch_size = 0; + uint32_t* m_uniquetoken_ptr = nullptr; + private: bool m_is_initialized = false; uint32_t m_instance_id = Kokkos::Tools::Experimental::Impl::idForInstance< diff --git a/lib/kokkos/core/src/OpenMPTarget/Kokkos_OpenMPTarget_Parallel.hpp b/lib/kokkos/core/src/OpenMPTarget/Kokkos_OpenMPTarget_Parallel.hpp index e222d65250..f71f888713 100644 --- a/lib/kokkos/core/src/OpenMPTarget/Kokkos_OpenMPTarget_Parallel.hpp +++ b/lib/kokkos/core/src/OpenMPTarget/Kokkos_OpenMPTarget_Parallel.hpp @@ -116,8 +116,8 @@ class OpenMPTargetExecTeamMember { // FIXME_OPENMPTARGET this function currently ignores the reducer passed. template KOKKOS_INLINE_FUNCTION std::enable_if_t::value> - team_reduce(ReducerType const&, typename ReducerType::value_type& value) const - noexcept { + team_reduce(ReducerType const&, + typename ReducerType::value_type& value) const noexcept { #pragma omp barrier using value_type = typename ReducerType::value_type; @@ -741,43 +741,6 @@ struct TeamVectorRangeBoundariesStruct { } // namespace Impl -} // namespace Kokkos -//---------------------------------------------------------------------------- -//---------------------------------------------------------------------------- -namespace Kokkos { -namespace Impl { - -//---------------------------------------------------------------------------- -/** \brief Data for OpenMPTarget thread execution */ - -class OpenMPTargetExec { - public: - // FIXME_OPENMPTARGET - Currently the maximum number of - // teams possible is calculated based on NVIDIA's Volta GPU. In - // future this value should be based on the chosen architecture for the - // OpenMPTarget backend. - static int MAX_ACTIVE_THREADS; - - private: - static void* scratch_ptr; - - public: - static void verify_is_process(const char* const); - static void verify_initialized(const char* const); - - static void* get_scratch_ptr(); - static void clear_scratch(); - static void resize_scratch(int64_t team_reduce_bytes, - int64_t team_shared_bytes, - int64_t thread_local_bytes, int64_t league_size); - - static void* m_scratch_ptr; - static std::mutex m_mutex_scratch_ptr; - static int64_t m_scratch_size; - static uint32_t* m_uniquetoken_ptr; -}; - -} // namespace Impl } // namespace Kokkos #endif /* KOKKOS_OPENMPTARGET_PARALLEL_HPP */ diff --git a/lib/kokkos/core/src/OpenMPTarget/Kokkos_OpenMPTarget_ParallelFor_MDRange.hpp b/lib/kokkos/core/src/OpenMPTarget/Kokkos_OpenMPTarget_ParallelFor_MDRange.hpp index bd7d3eef5d..38ed7c5681 100644 --- a/lib/kokkos/core/src/OpenMPTarget/Kokkos_OpenMPTarget_ParallelFor_MDRange.hpp +++ b/lib/kokkos/core/src/OpenMPTarget/Kokkos_OpenMPTarget_ParallelFor_MDRange.hpp @@ -20,6 +20,8 @@ #include #include #include "Kokkos_OpenMPTarget_MDRangePolicy.hpp" +#include "Kokkos_OpenMPTarget_Instance.hpp" +#include "Kokkos_OpenMPTarget_FunctorAdapter.hpp" //---------------------------------------------------------------------------- //---------------------------------------------------------------------------- @@ -31,38 +33,38 @@ template class ParallelFor, Kokkos::Experimental::OpenMPTarget> { private: - using Policy = Kokkos::MDRangePolicy; - using WorkTag = typename Policy::work_tag; - using Member = typename Policy::member_type; - using Index = typename Policy::index_type; + using Policy = Kokkos::MDRangePolicy; + using Member = typename Policy::member_type; + using Index = typename Policy::index_type; + + using FunctorAdapter = + Kokkos::Experimental::Impl::FunctorAdapter; + const FunctorAdapter m_functor; - const FunctorType m_functor; const Policy m_policy; public: inline void execute() const { - OpenMPTargetExec::verify_is_process( + Experimental::Impl::OpenMPTargetInternal::verify_is_process( "Kokkos::Experimental::OpenMPTarget parallel_for"); - OpenMPTargetExec::verify_initialized( + Experimental::Impl::OpenMPTargetInternal::verify_initialized( "Kokkos::Experimental::OpenMPTarget parallel_for"); - FunctorType functor(m_functor); + Policy policy = m_policy; - typename Policy::point_type unused; static_assert(1 < Policy::rank && Policy::rank < 7); static_assert(Policy::inner_direction == Iterate::Left || Policy::inner_direction == Iterate::Right); execute_tile( - unused, functor, policy, + m_functor, policy, std::integral_constant()); } template inline std::enable_if_t execute_tile( - typename Policy::point_type offset, const FunctorType& functor, - const Policy& policy, OpenMPTargetIterateRight) const { - (void)offset; + const FunctorAdapter& functor, const Policy& policy, + OpenMPTargetIterateRight) const { const Index begin_0 = policy.m_lower[0]; const Index begin_1 = policy.m_lower[1]; @@ -72,18 +74,14 @@ class ParallelFor, #pragma omp target teams distribute parallel for collapse(2) map(to : functor) for (auto i0 = begin_0; i0 < end_0; ++i0) for (auto i1 = begin_1; i1 < end_1; ++i1) { - if constexpr (std::is_void::value) - functor(i0, i1); - else - functor(typename Policy::work_tag(), i0, i1); + functor(i0, i1); } } template inline std::enable_if_t execute_tile( - typename Policy::point_type offset, const FunctorType& functor, - const Policy& policy, OpenMPTargetIterateRight) const { - (void)offset; + const FunctorAdapter& functor, const Policy& policy, + OpenMPTargetIterateRight) const { const Index begin_0 = policy.m_lower[0]; const Index begin_1 = policy.m_lower[1]; const Index begin_2 = policy.m_lower[2]; @@ -96,10 +94,7 @@ class ParallelFor, for (auto i0 = begin_0; i0 < end_0; ++i0) { for (auto i1 = begin_1; i1 < end_1; ++i1) { for (auto i2 = begin_2; i2 < end_2; ++i2) { - if constexpr (std::is_void::value) - functor(i0, i1, i2); - else - functor(typename Policy::work_tag(), i0, i1, i2); + functor(i0, i1, i2); } } } @@ -107,9 +102,8 @@ class ParallelFor, template inline std::enable_if_t execute_tile( - typename Policy::point_type offset, const FunctorType& functor, - const Policy& policy, OpenMPTargetIterateRight) const { - (void)offset; + const FunctorAdapter& functor, const Policy& policy, + OpenMPTargetIterateRight) const { const Index begin_0 = policy.m_lower[0]; const Index begin_1 = policy.m_lower[1]; const Index begin_2 = policy.m_lower[2]; @@ -125,10 +119,7 @@ class ParallelFor, for (auto i1 = begin_1; i1 < end_1; ++i1) { for (auto i2 = begin_2; i2 < end_2; ++i2) { for (auto i3 = begin_3; i3 < end_3; ++i3) { - if constexpr (std::is_void::value) - functor(i0, i1, i2, i3); - else - functor(typename Policy::work_tag(), i0, i1, i2, i3); + functor(i0, i1, i2, i3); } } } @@ -137,9 +128,8 @@ class ParallelFor, template inline std::enable_if_t execute_tile( - typename Policy::point_type offset, const FunctorType& functor, - const Policy& policy, OpenMPTargetIterateRight) const { - (void)offset; + const FunctorAdapter& functor, const Policy& policy, + OpenMPTargetIterateRight) const { const Index begin_0 = policy.m_lower[0]; const Index begin_1 = policy.m_lower[1]; const Index begin_2 = policy.m_lower[2]; @@ -158,11 +148,7 @@ class ParallelFor, for (auto i2 = begin_2; i2 < end_2; ++i2) { for (auto i3 = begin_3; i3 < end_3; ++i3) { for (auto i4 = begin_4; i4 < end_4; ++i4) { - if constexpr (std::is_same::value) - functor(i0, i1, i2, i3, i4); - else - functor(typename Policy::work_tag(), i0, i1, i2, i3, i4); + functor(i0, i1, i2, i3, i4); } } } @@ -172,9 +158,8 @@ class ParallelFor, template inline std::enable_if_t execute_tile( - typename Policy::point_type offset, const FunctorType& functor, - const Policy& policy, OpenMPTargetIterateRight) const { - (void)offset; + const FunctorAdapter& functor, const Policy& policy, + OpenMPTargetIterateRight) const { const Index begin_0 = policy.m_lower[0]; const Index begin_1 = policy.m_lower[1]; const Index begin_2 = policy.m_lower[2]; @@ -197,12 +182,7 @@ class ParallelFor, for (auto i4 = begin_4; i4 < end_4; ++i4) { for (auto i5 = begin_5; i5 < end_5; ++i5) { { - if constexpr (std::is_same::value) - functor(i0, i1, i2, i3, i4, i5); - else - functor(typename Policy::work_tag(), i0, i1, i2, i3, i4, - i5); + functor(i0, i1, i2, i3, i4, i5); } } } @@ -214,9 +194,8 @@ class ParallelFor, template inline std::enable_if_t execute_tile( - typename Policy::point_type offset, const FunctorType& functor, - const Policy& policy, OpenMPTargetIterateLeft) const { - (void)offset; + const FunctorAdapter& functor, const Policy& policy, + OpenMPTargetIterateLeft) const { const Index begin_0 = policy.m_lower[0]; const Index begin_1 = policy.m_lower[1]; @@ -226,18 +205,14 @@ class ParallelFor, #pragma omp target teams distribute parallel for collapse(2) map(to : functor) for (auto i1 = begin_1; i1 < end_1; ++i1) for (auto i0 = begin_0; i0 < end_0; ++i0) { - if constexpr (std::is_void::value) - functor(i0, i1); - else - functor(typename Policy::work_tag(), i0, i1); + functor(i0, i1); } } template inline std::enable_if_t execute_tile( - typename Policy::point_type offset, const FunctorType& functor, - const Policy& policy, OpenMPTargetIterateLeft) const { - (void)offset; + const FunctorAdapter& functor, const Policy& policy, + OpenMPTargetIterateLeft) const { const Index begin_0 = policy.m_lower[0]; const Index begin_1 = policy.m_lower[1]; const Index begin_2 = policy.m_lower[2]; @@ -250,10 +225,7 @@ class ParallelFor, for (auto i2 = begin_2; i2 < end_2; ++i2) { for (auto i1 = begin_1; i1 < end_1; ++i1) { for (auto i0 = begin_0; i0 < end_0; ++i0) { - if constexpr (std::is_void::value) - functor(i0, i1, i2); - else - functor(typename Policy::work_tag(), i0, i1, i2); + functor(i0, i1, i2); } } } @@ -261,9 +233,8 @@ class ParallelFor, template inline std::enable_if_t execute_tile( - typename Policy::point_type offset, const FunctorType& functor, - const Policy& policy, OpenMPTargetIterateLeft) const { - (void)offset; + const FunctorAdapter& functor, const Policy& policy, + OpenMPTargetIterateLeft) const { const Index begin_0 = policy.m_lower[0]; const Index begin_1 = policy.m_lower[1]; const Index begin_2 = policy.m_lower[2]; @@ -279,10 +250,7 @@ class ParallelFor, for (auto i2 = begin_2; i2 < end_2; ++i2) { for (auto i1 = begin_1; i1 < end_1; ++i1) { for (auto i0 = begin_0; i0 < end_0; ++i0) { - if constexpr (std::is_void::value) - functor(i0, i1, i2, i3); - else - functor(typename Policy::work_tag(), i0, i1, i2, i3); + functor(i0, i1, i2, i3); } } } @@ -291,9 +259,8 @@ class ParallelFor, template inline std::enable_if_t execute_tile( - typename Policy::point_type offset, const FunctorType& functor, - const Policy& policy, OpenMPTargetIterateLeft) const { - (void)offset; + const FunctorAdapter& functor, const Policy& policy, + OpenMPTargetIterateLeft) const { const Index begin_0 = policy.m_lower[0]; const Index begin_1 = policy.m_lower[1]; const Index begin_2 = policy.m_lower[2]; @@ -312,11 +279,7 @@ class ParallelFor, for (auto i2 = begin_2; i2 < end_2; ++i2) { for (auto i1 = begin_1; i1 < end_1; ++i1) { for (auto i0 = begin_0; i0 < end_0; ++i0) { - if constexpr (std::is_same::value) - functor(i0, i1, i2, i3, i4); - else - functor(typename Policy::work_tag(), i0, i1, i2, i3, i4); + functor(i0, i1, i2, i3, i4); } } } @@ -326,9 +289,8 @@ class ParallelFor, template inline std::enable_if_t execute_tile( - typename Policy::point_type offset, const FunctorType& functor, - const Policy& policy, OpenMPTargetIterateLeft) const { - (void)offset; + const FunctorAdapter& functor, const Policy& policy, + OpenMPTargetIterateLeft) const { const Index begin_0 = policy.m_lower[0]; const Index begin_1 = policy.m_lower[1]; const Index begin_2 = policy.m_lower[2]; @@ -351,12 +313,7 @@ class ParallelFor, for (auto i1 = begin_1; i1 < end_1; ++i1) { for (auto i0 = begin_0; i0 < end_0; ++i0) { { - if constexpr (std::is_same::value) - functor(i0, i1, i2, i3, i4, i5); - else - functor(typename Policy::work_tag(), i0, i1, i2, i3, i4, - i5); + functor(i0, i1, i2, i3, i4, i5); } } } diff --git a/lib/kokkos/core/src/OpenMPTarget/Kokkos_OpenMPTarget_ParallelFor_Range.hpp b/lib/kokkos/core/src/OpenMPTarget/Kokkos_OpenMPTarget_ParallelFor_Range.hpp index a674637a3b..502461cc5e 100644 --- a/lib/kokkos/core/src/OpenMPTarget/Kokkos_OpenMPTarget_ParallelFor_Range.hpp +++ b/lib/kokkos/core/src/OpenMPTarget/Kokkos_OpenMPTarget_ParallelFor_Range.hpp @@ -20,6 +20,8 @@ #include #include #include +#include "Kokkos_OpenMPTarget_Instance.hpp" +#include "Kokkos_OpenMPTarget_FunctorAdapter.hpp" namespace Kokkos { namespace Impl { @@ -28,36 +30,30 @@ template class ParallelFor, Kokkos::Experimental::OpenMPTarget> { private: - using Policy = Kokkos::RangePolicy; - using WorkTag = typename Policy::work_tag; - using Member = typename Policy::member_type; + using Policy = Kokkos::RangePolicy; + using Member = typename Policy::member_type; - const FunctorType m_functor; + Kokkos::Experimental::Impl::FunctorAdapter m_functor; const Policy m_policy; public: - void execute() const { execute_impl(); } + void execute() const { execute_impl(); } - template void execute_impl() const { - OpenMPTargetExec::verify_is_process( + Experimental::Impl::OpenMPTargetInternal::verify_is_process( "Kokkos::Experimental::OpenMPTarget parallel_for"); - OpenMPTargetExec::verify_initialized( + Experimental::Impl::OpenMPTargetInternal::verify_initialized( "Kokkos::Experimental::OpenMPTarget parallel_for"); const auto begin = m_policy.begin(); const auto end = m_policy.end(); if (end <= begin) return; - FunctorType a_functor(m_functor); + auto const a_functor(m_functor); #pragma omp target teams distribute parallel for map(to : a_functor) for (auto i = begin; i < end; ++i) { - if constexpr (std::is_void::value) { - a_functor(i); - } else { - a_functor(TagType(), i); - } + a_functor(i); } } diff --git a/lib/kokkos/core/src/OpenMPTarget/Kokkos_OpenMPTarget_ParallelFor_Team.hpp b/lib/kokkos/core/src/OpenMPTarget/Kokkos_OpenMPTarget_ParallelFor_Team.hpp index 26085f1140..77dc71a87b 100644 --- a/lib/kokkos/core/src/OpenMPTarget/Kokkos_OpenMPTarget_ParallelFor_Team.hpp +++ b/lib/kokkos/core/src/OpenMPTarget/Kokkos_OpenMPTarget_ParallelFor_Team.hpp @@ -22,6 +22,7 @@ #include #include #include +#include namespace Kokkos { @@ -76,28 +77,27 @@ class ParallelFor, using Policy = Kokkos::Impl::TeamPolicyInternal; - using WorkTag = typename Policy::work_tag; - using Member = typename Policy::member_type; + using Member = typename Policy::member_type; + + Kokkos::Experimental::Impl::FunctorAdapter m_functor; - const FunctorType m_functor; const Policy m_policy; const size_t m_shmem_size; public: void execute() const { - OpenMPTargetExec::verify_is_process( + Experimental::Impl::OpenMPTargetInternal::verify_is_process( "Kokkos::Experimental::OpenMPTarget parallel_for"); - OpenMPTargetExec::verify_initialized( + Experimental::Impl::OpenMPTargetInternal::verify_initialized( "Kokkos::Experimental::OpenMPTarget parallel_for"); - execute_impl(); + execute_impl(); } private: - template void execute_impl() const { - OpenMPTargetExec::verify_is_process( + Experimental::Impl::OpenMPTargetInternal::verify_is_process( "Kokkos::Experimental::OpenMPTarget parallel_for"); - OpenMPTargetExec::verify_initialized( + Experimental::Impl::OpenMPTargetInternal::verify_initialized( "Kokkos::Experimental::OpenMPTarget parallel_for"); const auto league_size = m_policy.league_size(); const auto team_size = m_policy.team_size(); @@ -105,11 +105,12 @@ class ParallelFor, const size_t shmem_size_L0 = m_policy.scratch_size(0, team_size); const size_t shmem_size_L1 = m_policy.scratch_size(1, team_size); - OpenMPTargetExec::resize_scratch(team_size, shmem_size_L0, shmem_size_L1, - league_size); + m_policy.space().impl_internal_space_instance()->resize_scratch( + team_size, shmem_size_L0, shmem_size_L1, league_size); - void* scratch_ptr = OpenMPTargetExec::get_scratch_ptr(); - FunctorType a_functor(m_functor); + void* scratch_ptr = + m_policy.space().impl_internal_space_instance()->get_scratch_ptr(); + auto const a_functor(m_functor); // FIXME_OPENMPTARGET - If the team_size is not a multiple of 32, the // scratch implementation does not work in the Release or RelWithDebugInfo @@ -122,7 +123,7 @@ class ParallelFor, int max_active_teams = omp_get_max_teams(); #else int max_active_teams = - std::min(OpenMPTargetExec::MAX_ACTIVE_THREADS / team_size, league_size); + std::min(m_policy.space().concurrency() / team_size, league_size); #endif // FIXME_OPENMPTARGET: Although the maximum number of teams is set using the @@ -161,16 +162,13 @@ class ParallelFor, typename Policy::member_type team(league_id, league_size, team_size, vector_length, scratch_ptr, blockIdx, shmem_size_L0, shmem_size_L1); - if constexpr (std::is_void_v) - m_functor(team); - else - m_functor(TagType(), team); + a_functor(team); } } #else #pragma omp target teams distribute firstprivate(a_functor) \ is_device_ptr(scratch_ptr) num_teams(max_active_teams) \ - thread_limit(team_size) + thread_limit(team_size) for (int i = 0; i < league_size; i++) { #pragma omp parallel { @@ -180,10 +178,7 @@ class ParallelFor, typename Policy::member_type team(i, league_size, team_size, vector_length, scratch_ptr, i, shmem_size_L0, shmem_size_L1); - if constexpr (std::is_void_v) - m_functor(team); - else - m_functor(TagType(), team); + a_functor(team); } } #endif diff --git a/lib/kokkos/core/src/OpenMPTarget/Kokkos_OpenMPTarget_ParallelReduce_MDRange.hpp b/lib/kokkos/core/src/OpenMPTarget/Kokkos_OpenMPTarget_ParallelReduce_MDRange.hpp index e86a121974..bee604834c 100644 --- a/lib/kokkos/core/src/OpenMPTarget/Kokkos_OpenMPTarget_ParallelReduce_MDRange.hpp +++ b/lib/kokkos/core/src/OpenMPTarget/Kokkos_OpenMPTarget_ParallelReduce_MDRange.hpp @@ -37,9 +37,8 @@ class ParallelReduce; + public: inline void execute() const { // Only let one ParallelReduce instance at a time use the scratch memory. std::scoped_lock scratch_memory_lock( - OpenMPTargetExec::m_mutex_scratch_ptr); + m_policy.space().impl_internal_space_instance()->m_mutex_scratch_ptr); + + auto const functor = FunctorAdapter(m_functor_reducer.get_functor()); execute_tile( - m_functor_reducer.get_functor(), m_policy, m_result_ptr, + functor, m_policy, m_result_ptr, std::integral_constant()); } @@ -77,7 +81,7 @@ class ParallelReduce inline std::enable_if_t execute_tile( - const FunctorType& functor, const Policy& policy, pointer_type ptr, + const FunctorAdapter& functor, const Policy& policy, pointer_type ptr, OpenMPTargetIterateLeft) const { const Index begin_0 = policy.m_lower[0]; const Index begin_1 = policy.m_lower[1]; @@ -90,32 +94,23 @@ class ParallelReduce::join(omp_out, omp_in)) \ - initializer(OpenMPTargetReducerWrapper ::init(omp_priv)) +#pragma omp declare reduction(custom \ +:ValueType : OpenMPTargetReducerWrapper::join(omp_out, omp_in)) \ + initializer(OpenMPTargetReducerWrapper::init(omp_priv)) -#pragma omp target teams distribute parallel for collapse(2) map(to \ - : functor) \ - reduction(custom \ - : result) +#pragma omp target teams distribute parallel for collapse(2) map(to : functor) \ + reduction(custom : result) for (auto i1 = begin_1; i1 < end_1; ++i1) { for (auto i0 = begin_0; i0 < end_0; ++i0) { - if constexpr (std::is_void::value) - functor(i0, i1, result); - else - functor(typename Policy::work_tag(), i0, i1, result); + functor(i0, i1, result); } } } else { #pragma omp target teams distribute parallel for collapse(2) map(to : functor) \ -reduction(+:result) + reduction(+ : result) for (auto i1 = begin_1; i1 < end_1; ++i1) { for (auto i0 = begin_0; i0 < end_0; ++i0) { - if constexpr (std::is_void::value) - functor(i0, i1, result); - else - functor(typename Policy::work_tag(), i0, i1, result); + functor(i0, i1, result); } } } @@ -126,7 +121,7 @@ reduction(+:result) template inline std::enable_if_t execute_tile( - const FunctorType& functor, const Policy& policy, pointer_type ptr, + const FunctorAdapter& functor, const Policy& policy, pointer_type ptr, OpenMPTargetIterateLeft) const { const Index begin_0 = policy.m_lower[0]; const Index begin_1 = policy.m_lower[1]; @@ -141,38 +136,29 @@ reduction(+:result) // FIXME_OPENMPTARGET: Unable to separate directives and their companion // loops which leads to code duplication for different reduction types. if constexpr (UseReducer) { -#pragma omp declare reduction( \ - custom:ValueType \ - : OpenMPTargetReducerWrapper ::join( \ - omp_out, omp_in)) \ - initializer( \ - OpenMPTargetReducerWrapper ::init( \ - omp_priv)) +#pragma omp declare reduction( \ + custom \ +:ValueType : OpenMPTargetReducerWrapper< \ + typename ReducerType::functor_type>::join(omp_out, omp_in)) \ + initializer(OpenMPTargetReducerWrapper< \ + typename ReducerType::functor_type>::init(omp_priv)) -#pragma omp target teams distribute parallel for collapse(3) map(to \ - : functor) \ - reduction(custom \ - : result) +#pragma omp target teams distribute parallel for collapse(3) map(to : functor) \ + reduction(custom : result) for (auto i2 = begin_2; i2 < end_2; ++i2) { for (auto i1 = begin_1; i1 < end_1; ++i1) { for (auto i0 = begin_0; i0 < end_0; ++i0) { - if constexpr (std::is_void::value) - functor(i0, i1, i2, result); - else - functor(typename Policy::work_tag(), i0, i1, i2, result); + functor(i0, i1, i2, result); } } } } else { #pragma omp target teams distribute parallel for collapse(3) map(to : functor) \ -reduction(+:result) + reduction(+ : result) for (auto i2 = begin_2; i2 < end_2; ++i2) { for (auto i1 = begin_1; i1 < end_1; ++i1) { for (auto i0 = begin_0; i0 < end_0; ++i0) { - if constexpr (std::is_void::value) - functor(i0, i1, i2, result); - else - functor(typename Policy::work_tag(), i0, i1, i2, result); + functor(i0, i1, i2, result); } } } @@ -184,7 +170,7 @@ reduction(+:result) template inline std::enable_if_t execute_tile( - const FunctorType& functor, const Policy& policy, pointer_type ptr, + const FunctorAdapter& functor, const Policy& policy, pointer_type ptr, OpenMPTargetIterateLeft) const { const Index begin_0 = policy.m_lower[0]; const Index begin_1 = policy.m_lower[1]; @@ -201,40 +187,29 @@ reduction(+:result) // FIXME_OPENMPTARGET: Unable to separate directives and their companion // loops which leads to code duplication for different reduction types. if constexpr (UseReducer) { -#pragma omp declare reduction( \ - custom:ValueType \ - : OpenMPTargetReducerWrapper ::join(omp_out, omp_in)) \ - initializer(OpenMPTargetReducerWrapper ::init(omp_priv)) +#pragma omp declare reduction(custom \ +:ValueType : OpenMPTargetReducerWrapper::join(omp_out, omp_in)) \ + initializer(OpenMPTargetReducerWrapper::init(omp_priv)) -#pragma omp target teams distribute parallel for collapse(4) map(to \ - : functor) \ - reduction(custom \ - : result) +#pragma omp target teams distribute parallel for collapse(4) map(to : functor) \ + reduction(custom : result) for (auto i3 = begin_3; i3 < end_3; ++i3) { for (auto i2 = begin_2; i2 < end_2; ++i2) { for (auto i1 = begin_1; i1 < end_1; ++i1) { for (auto i0 = begin_0; i0 < end_0; ++i0) { - if constexpr (std::is_same::value) - functor(i0, i1, i2, i3, result); - else - functor(typename Policy::work_tag(), i0, i1, i2, i3, result); + functor(i0, i1, i2, i3, result); } } } } } else { #pragma omp target teams distribute parallel for collapse(4) map(to : functor) \ -reduction(+:result) + reduction(+ : result) for (auto i3 = begin_3; i3 < end_3; ++i3) { for (auto i2 = begin_2; i2 < end_2; ++i2) { for (auto i1 = begin_1; i1 < end_1; ++i1) { for (auto i0 = begin_0; i0 < end_0; ++i0) { - if constexpr (std::is_same::value) - functor(i0, i1, i2, i3, result); - else - functor(typename Policy::work_tag(), i0, i1, i2, i3, result); + functor(i0, i1, i2, i3, result); } } } @@ -247,7 +222,7 @@ reduction(+:result) template inline std::enable_if_t execute_tile( - const FunctorType& functor, const Policy& policy, pointer_type ptr, + const FunctorAdapter& functor, const Policy& policy, pointer_type ptr, OpenMPTargetIterateLeft) const { const Index begin_0 = policy.m_lower[0]; const Index begin_1 = policy.m_lower[1]; @@ -266,26 +241,18 @@ reduction(+:result) // FIXME_OPENMPTARGET: Unable to separate directives and their companion // loops which leads to code duplication for different reduction types. if constexpr (UseReducer) { -#pragma omp declare reduction( \ - custom:ValueType \ - : OpenMPTargetReducerWrapper ::join(omp_out, omp_in)) \ - initializer(OpenMPTargetReducerWrapper ::init(omp_priv)) +#pragma omp declare reduction(custom \ +:ValueType : OpenMPTargetReducerWrapper::join(omp_out, omp_in)) \ + initializer(OpenMPTargetReducerWrapper::init(omp_priv)) -#pragma omp target teams distribute parallel for collapse(5) map(to \ - : functor) \ - reduction(custom \ - : result) +#pragma omp target teams distribute parallel for collapse(5) map(to : functor) \ + reduction(custom : result) for (auto i4 = begin_4; i4 < end_4; ++i4) { for (auto i3 = begin_3; i3 < end_3; ++i3) { for (auto i2 = begin_2; i2 < end_2; ++i2) { for (auto i1 = begin_1; i1 < end_1; ++i1) { for (auto i0 = begin_0; i0 < end_0; ++i0) { - if constexpr (std::is_same::value) - functor(i0, i1, i2, i3, i4, result); - else - functor(typename Policy::work_tag(), i0, i1, i2, i3, i4, - result); + functor(i0, i1, i2, i3, i4, result); } } } @@ -293,18 +260,13 @@ reduction(+:result) } } else { #pragma omp target teams distribute parallel for collapse(5) map(to : functor) \ -reduction(+:result) + reduction(+ : result) for (auto i4 = begin_4; i4 < end_4; ++i4) { for (auto i3 = begin_3; i3 < end_3; ++i3) { for (auto i2 = begin_2; i2 < end_2; ++i2) { for (auto i1 = begin_1; i1 < end_1; ++i1) { for (auto i0 = begin_0; i0 < end_0; ++i0) { - if constexpr (std::is_same::value) - functor(i0, i1, i2, i3, i4, result); - else - functor(typename Policy::work_tag(), i0, i1, i2, i3, i4, - result); + functor(i0, i1, i2, i3, i4, result); } } } @@ -318,7 +280,7 @@ reduction(+:result) template inline std::enable_if_t execute_tile( - const FunctorType& functor, const Policy& policy, pointer_type ptr, + const FunctorAdapter& functor, const Policy& policy, pointer_type ptr, OpenMPTargetIterateLeft) const { const Index begin_0 = policy.m_lower[0]; const Index begin_1 = policy.m_lower[1]; @@ -339,27 +301,19 @@ reduction(+:result) // FIXME_OPENMPTARGET: Unable to separate directives and their companion // loops which leads to code duplication for different reduction types. if constexpr (UseReducer) { -#pragma omp declare reduction( \ - custom:ValueType \ - : OpenMPTargetReducerWrapper ::join(omp_out, omp_in)) \ - initializer(OpenMPTargetReducerWrapper ::init(omp_priv)) +#pragma omp declare reduction(custom \ +:ValueType : OpenMPTargetReducerWrapper::join(omp_out, omp_in)) \ + initializer(OpenMPTargetReducerWrapper::init(omp_priv)) -#pragma omp target teams distribute parallel for collapse(6) map(to \ - : functor) \ - reduction(custom \ - : result) +#pragma omp target teams distribute parallel for collapse(6) map(to : functor) \ + reduction(custom : result) for (auto i5 = begin_5; i5 < end_5; ++i5) { for (auto i4 = begin_4; i4 < end_4; ++i4) { for (auto i3 = begin_3; i3 < end_3; ++i3) { for (auto i2 = begin_2; i2 < end_2; ++i2) { for (auto i1 = begin_1; i1 < end_1; ++i1) { for (auto i0 = begin_0; i0 < end_0; ++i0) { - if constexpr (std::is_same::value) - functor(i0, i1, i2, i3, i4, i5, result); - else - functor(typename Policy::work_tag(), i0, i1, i2, i3, i4, i5, - result); + functor(i0, i1, i2, i3, i4, i5, result); } } } @@ -368,19 +322,14 @@ reduction(+:result) } } else { #pragma omp target teams distribute parallel for collapse(6) map(to : functor) \ -reduction(+:result) + reduction(+ : result) for (auto i5 = begin_5; i5 < end_5; ++i5) { for (auto i4 = begin_4; i4 < end_4; ++i4) { for (auto i3 = begin_3; i3 < end_3; ++i3) { for (auto i2 = begin_2; i2 < end_2; ++i2) { for (auto i1 = begin_1; i1 < end_1; ++i1) { for (auto i0 = begin_0; i0 < end_0; ++i0) { - if constexpr (std::is_same::value) - functor(i0, i1, i2, i3, i4, i5, result); - else - functor(typename Policy::work_tag(), i0, i1, i2, i3, i4, i5, - result); + functor(i0, i1, i2, i3, i4, i5, result); } } } @@ -395,7 +344,7 @@ reduction(+:result) template inline std::enable_if_t execute_tile( - const FunctorType& functor, const Policy& policy, pointer_type ptr, + const FunctorAdapter& functor, const Policy& policy, pointer_type ptr, OpenMPTargetIterateRight) const { const Index begin_0 = policy.m_lower[0]; const Index begin_1 = policy.m_lower[1]; @@ -408,32 +357,23 @@ reduction(+:result) // FIXME_OPENMPTARGET: Unable to separate directives and their companion // loops which leads to code duplication for different reduction types. if constexpr (UseReducer) { -#pragma omp declare reduction( \ - custom:ValueType \ - : OpenMPTargetReducerWrapper ::join(omp_out, omp_in)) \ - initializer(OpenMPTargetReducerWrapper ::init(omp_priv)) +#pragma omp declare reduction(custom \ +:ValueType : OpenMPTargetReducerWrapper::join(omp_out, omp_in)) \ + initializer(OpenMPTargetReducerWrapper::init(omp_priv)) -#pragma omp target teams distribute parallel for collapse(2) map(to \ - : functor) \ - reduction(custom \ - : result) +#pragma omp target teams distribute parallel for collapse(2) map(to : functor) \ + reduction(custom : result) for (auto i0 = begin_0; i0 < end_0; ++i0) { for (auto i1 = begin_1; i1 < end_1; ++i1) { - if constexpr (std::is_void::value) - functor(i0, i1, result); - else - functor(typename Policy::work_tag(), i0, i1, result); + functor(i0, i1, result); } } } else { #pragma omp target teams distribute parallel for collapse(2) map(to : functor) \ -reduction(+:result) + reduction(+ : result) for (auto i0 = begin_0; i0 < end_0; ++i0) { for (auto i1 = begin_1; i1 < end_1; ++i1) { - if constexpr (std::is_void::value) - functor(i0, i1, result); - else - functor(typename Policy::work_tag(), i0, i1, result); + functor(i0, i1, result); } } } @@ -444,7 +384,7 @@ reduction(+:result) template inline std::enable_if_t execute_tile( - const FunctorType& functor, const Policy& policy, pointer_type ptr, + const FunctorAdapter& functor, const Policy& policy, pointer_type ptr, OpenMPTargetIterateRight) const { const Index begin_0 = policy.m_lower[0]; const Index begin_1 = policy.m_lower[1]; @@ -459,38 +399,29 @@ reduction(+:result) // FIXME_OPENMPTARGET: Unable to separate directives and their companion // loops which leads to code duplication for different reduction types. if constexpr (UseReducer) { -#pragma omp declare reduction( \ - custom:ValueType \ - : OpenMPTargetReducerWrapper ::join( \ - omp_out, omp_in)) \ - initializer( \ - OpenMPTargetReducerWrapper ::init( \ - omp_priv)) +#pragma omp declare reduction( \ + custom \ +:ValueType : OpenMPTargetReducerWrapper< \ + typename ReducerType::functor_type>::join(omp_out, omp_in)) \ + initializer(OpenMPTargetReducerWrapper< \ + typename ReducerType::functor_type>::init(omp_priv)) -#pragma omp target teams distribute parallel for collapse(3) map(to \ - : functor) \ - reduction(custom \ - : result) +#pragma omp target teams distribute parallel for collapse(3) map(to : functor) \ + reduction(custom : result) for (auto i0 = begin_0; i0 < end_0; ++i0) { for (auto i1 = begin_1; i1 < end_1; ++i1) { for (auto i2 = begin_2; i2 < end_2; ++i2) { - if constexpr (std::is_void::value) - functor(i0, i1, i2, result); - else - functor(typename Policy::work_tag(), i0, i1, i2, result); + functor(i0, i1, i2, result); } } } } else { #pragma omp target teams distribute parallel for collapse(3) map(to : functor) \ -reduction(+:result) + reduction(+ : result) for (auto i0 = begin_0; i0 < end_0; ++i0) { for (auto i1 = begin_1; i1 < end_1; ++i1) { for (auto i2 = begin_2; i2 < end_2; ++i2) { - if constexpr (std::is_void::value) - functor(i0, i1, i2, result); - else - functor(typename Policy::work_tag(), i0, i1, i2, result); + functor(i0, i1, i2, result); } } } @@ -502,7 +433,7 @@ reduction(+:result) template inline std::enable_if_t execute_tile( - const FunctorType& functor, const Policy& policy, pointer_type ptr, + const FunctorAdapter& functor, const Policy& policy, pointer_type ptr, OpenMPTargetIterateRight) const { const Index begin_0 = policy.m_lower[0]; const Index begin_1 = policy.m_lower[1]; @@ -519,40 +450,29 @@ reduction(+:result) // FIXME_OPENMPTARGET: Unable to separate directives and their companion // loops which leads to code duplication for different reduction types. if constexpr (UseReducer) { -#pragma omp declare reduction( \ - custom:ValueType \ - : OpenMPTargetReducerWrapper ::join(omp_out, omp_in)) \ - initializer(OpenMPTargetReducerWrapper ::init(omp_priv)) +#pragma omp declare reduction(custom \ +:ValueType : OpenMPTargetReducerWrapper::join(omp_out, omp_in)) \ + initializer(OpenMPTargetReducerWrapper::init(omp_priv)) -#pragma omp target teams distribute parallel for collapse(4) map(to \ - : functor) \ - reduction(custom \ - : result) +#pragma omp target teams distribute parallel for collapse(4) map(to : functor) \ + reduction(custom : result) for (auto i0 = begin_0; i0 < end_0; ++i0) { for (auto i1 = begin_1; i1 < end_1; ++i1) { for (auto i2 = begin_2; i2 < end_2; ++i2) { for (auto i3 = begin_3; i3 < end_3; ++i3) { - if constexpr (std::is_same::value) - functor(i0, i1, i2, i3, result); - else - functor(typename Policy::work_tag(), i0, i1, i2, i3, result); + functor(i0, i1, i2, i3, result); } } } } } else { #pragma omp target teams distribute parallel for collapse(4) map(to : functor) \ -reduction(+:result) + reduction(+ : result) for (auto i0 = begin_0; i0 < end_0; ++i0) { for (auto i1 = begin_1; i1 < end_1; ++i1) { for (auto i2 = begin_2; i2 < end_2; ++i2) { for (auto i3 = begin_3; i3 < end_3; ++i3) { - if constexpr (std::is_same::value) - functor(i0, i1, i2, i3, result); - else - functor(typename Policy::work_tag(), i0, i1, i2, i3, result); + functor(i0, i1, i2, i3, result); } } } @@ -565,7 +485,7 @@ reduction(+:result) template inline std::enable_if_t execute_tile( - const FunctorType& functor, const Policy& policy, pointer_type ptr, + const FunctorAdapter& functor, const Policy& policy, pointer_type ptr, OpenMPTargetIterateRight) const { const Index begin_0 = policy.m_lower[0]; const Index begin_1 = policy.m_lower[1]; @@ -584,26 +504,18 @@ reduction(+:result) // FIXME_OPENMPTARGET: Unable to separate directives and their companion // loops which leads to code duplication for different reduction types. if constexpr (UseReducer) { -#pragma omp declare reduction( \ - custom:ValueType \ - : OpenMPTargetReducerWrapper ::join(omp_out, omp_in)) \ - initializer(OpenMPTargetReducerWrapper ::init(omp_priv)) +#pragma omp declare reduction(custom \ +:ValueType : OpenMPTargetReducerWrapper::join(omp_out, omp_in)) \ + initializer(OpenMPTargetReducerWrapper::init(omp_priv)) -#pragma omp target teams distribute parallel for collapse(5) map(to \ - : functor) \ - reduction(custom \ - : result) +#pragma omp target teams distribute parallel for collapse(5) map(to : functor) \ + reduction(custom : result) for (auto i0 = begin_0; i0 < end_0; ++i0) { for (auto i1 = begin_1; i1 < end_1; ++i1) { for (auto i2 = begin_2; i2 < end_2; ++i2) { for (auto i3 = begin_3; i3 < end_3; ++i3) { for (auto i4 = begin_4; i4 < end_4; ++i4) { - if constexpr (std::is_same::value) - functor(i0, i1, i2, i3, i4, result); - else - functor(typename Policy::work_tag(), i0, i1, i2, i3, i4, - result); + functor(i0, i1, i2, i3, i4, result); } } } @@ -611,18 +523,13 @@ reduction(+:result) } } else { #pragma omp target teams distribute parallel for collapse(5) map(to : functor) \ -reduction(+:result) + reduction(+ : result) for (auto i0 = begin_0; i0 < end_0; ++i0) { for (auto i1 = begin_1; i1 < end_1; ++i1) { for (auto i2 = begin_2; i2 < end_2; ++i2) { for (auto i3 = begin_3; i3 < end_3; ++i3) { for (auto i4 = begin_4; i4 < end_4; ++i4) { - if constexpr (std::is_same::value) - functor(i0, i1, i2, i3, i4, result); - else - functor(typename Policy::work_tag(), i0, i1, i2, i3, i4, - result); + functor(i0, i1, i2, i3, i4, result); } } } @@ -636,7 +543,7 @@ reduction(+:result) template inline std::enable_if_t execute_tile( - const FunctorType& functor, const Policy& policy, pointer_type ptr, + const FunctorAdapter& functor, const Policy& policy, pointer_type ptr, OpenMPTargetIterateRight) const { const Index begin_0 = policy.m_lower[0]; const Index begin_1 = policy.m_lower[1]; @@ -657,27 +564,19 @@ reduction(+:result) // FIXME_OPENMPTARGET: Unable to separate directives and their companion // loops which leads to code duplication for different reduction types. if constexpr (UseReducer) { -#pragma omp declare reduction( \ - custom:ValueType \ - : OpenMPTargetReducerWrapper ::join(omp_out, omp_in)) \ - initializer(OpenMPTargetReducerWrapper ::init(omp_priv)) +#pragma omp declare reduction(custom \ +:ValueType : OpenMPTargetReducerWrapper::join(omp_out, omp_in)) \ + initializer(OpenMPTargetReducerWrapper::init(omp_priv)) -#pragma omp target teams distribute parallel for collapse(6) map(to \ - : functor) \ - reduction(custom \ - : result) +#pragma omp target teams distribute parallel for collapse(6) map(to : functor) \ + reduction(custom : result) for (auto i0 = begin_0; i0 < end_0; ++i0) { for (auto i1 = begin_1; i1 < end_1; ++i1) { for (auto i2 = begin_2; i2 < end_2; ++i2) { for (auto i3 = begin_3; i3 < end_3; ++i3) { for (auto i4 = begin_4; i4 < end_4; ++i4) { for (auto i5 = begin_5; i5 < end_5; ++i5) { - if constexpr (std::is_same::value) - functor(i0, i1, i2, i3, i4, i5, result); - else - functor(typename Policy::work_tag(), i0, i1, i2, i3, i4, i5, - result); + functor(i0, i1, i2, i3, i4, i5, result); } } } @@ -686,19 +585,14 @@ reduction(+:result) } } else { #pragma omp target teams distribute parallel for collapse(6) map(to : functor) \ -reduction(+:result) + reduction(+ : result) for (auto i0 = begin_0; i0 < end_0; ++i0) { for (auto i1 = begin_1; i1 < end_1; ++i1) { for (auto i2 = begin_2; i2 < end_2; ++i2) { for (auto i3 = begin_3; i3 < end_3; ++i3) { for (auto i4 = begin_4; i4 < end_4; ++i4) { for (auto i5 = begin_5; i5 < end_5; ++i5) { - if constexpr (std::is_same::value) - functor(i0, i1, i2, i3, i4, i5, result); - else - functor(typename Policy::work_tag(), i0, i1, i2, i3, i4, i5, - result); + functor(i0, i1, i2, i3, i4, i5, result); } } } diff --git a/lib/kokkos/core/src/OpenMPTarget/Kokkos_OpenMPTarget_ParallelReduce_Range.hpp b/lib/kokkos/core/src/OpenMPTarget/Kokkos_OpenMPTarget_ParallelReduce_Range.hpp index 4a112ed11d..b7c8abcb44 100644 --- a/lib/kokkos/core/src/OpenMPTarget/Kokkos_OpenMPTarget_ParallelReduce_Range.hpp +++ b/lib/kokkos/core/src/OpenMPTarget/Kokkos_OpenMPTarget_ParallelReduce_Range.hpp @@ -21,6 +21,7 @@ #include #include #include +#include namespace Kokkos { namespace Impl { @@ -33,8 +34,6 @@ class ParallelReduce, using FunctorType = typename CombinedFunctorReducerType::functor_type; using ReducerType = typename CombinedFunctorReducerType::reducer_type; - using WorkTag = typename Policy::work_tag; - using pointer_type = typename ReducerType::pointer_type; using reference_type = typename ReducerType::reference_type; @@ -55,14 +54,17 @@ class ParallelReduce, const pointer_type m_result_ptr; bool m_result_ptr_on_device; const int m_result_ptr_num_elems; - using TagType = typename Policy::work_tag; public: void execute() const { // Only let one ParallelReduce instance at a time use the scratch memory. std::scoped_lock scratch_memory_lock( - OpenMPTargetExec::m_mutex_scratch_ptr); - const FunctorType& functor = m_functor_reducer.get_functor(); + m_policy.space().impl_internal_space_instance()->m_mutex_scratch_ptr); + + auto const functor = + Kokkos::Experimental::Impl::FunctorAdapter( + m_functor_reducer.get_functor()); + if constexpr (FunctorHasJoin) { // Enter this loop if the Functor has a init-join. ParReduceSpecialize::execute_init_join(functor, m_policy, m_result_ptr, @@ -75,26 +77,26 @@ class ParallelReduce, // Enter this loop if the reduction is on an array and the routine is // templated over the size of the array. if (m_result_ptr_num_elems <= 2) { - ParReduceSpecialize::template execute_array( + ParReduceSpecialize::template execute_array<2>( functor, m_policy, m_result_ptr, m_result_ptr_on_device); } else if (m_result_ptr_num_elems <= 4) { - ParReduceSpecialize::template execute_array( + ParReduceSpecialize::template execute_array<4>( functor, m_policy, m_result_ptr, m_result_ptr_on_device); } else if (m_result_ptr_num_elems <= 8) { - ParReduceSpecialize::template execute_array( + ParReduceSpecialize::template execute_array<8>( functor, m_policy, m_result_ptr, m_result_ptr_on_device); } else if (m_result_ptr_num_elems <= 16) { - ParReduceSpecialize::template execute_array( + ParReduceSpecialize::template execute_array<16>( functor, m_policy, m_result_ptr, m_result_ptr_on_device); } else if (m_result_ptr_num_elems <= 32) { - ParReduceSpecialize::template execute_array( + ParReduceSpecialize::template execute_array<32>( functor, m_policy, m_result_ptr, m_result_ptr_on_device); } else { Kokkos::abort("array reduction length must be <= 32"); } } else { // This loop handles the basic scalar reduction. - ParReduceSpecialize::template execute_array( + ParReduceSpecialize::template execute_array<1>( functor, m_policy, m_result_ptr, m_result_ptr_on_device); } } diff --git a/lib/kokkos/core/src/OpenMPTarget/Kokkos_OpenMPTarget_ParallelReduce_Team.hpp b/lib/kokkos/core/src/OpenMPTarget/Kokkos_OpenMPTarget_ParallelReduce_Team.hpp index 16c0eedb81..b81e3aa7ed 100644 --- a/lib/kokkos/core/src/OpenMPTarget/Kokkos_OpenMPTarget_ParallelReduce_Team.hpp +++ b/lib/kokkos/core/src/OpenMPTarget/Kokkos_OpenMPTarget_ParallelReduce_Team.hpp @@ -59,7 +59,7 @@ parallel_reduce(const Impl::TeamThreadRangeBoundariesStruct< #pragma omp barrier if constexpr (std::is_arithmetic::value) { -#pragma omp for reduction(+ : TeamThread_scratch[:1]) +#pragma omp for reduction(+ : TeamThread_scratch[ : 1]) for (iType i = loop_boundaries.start; i < loop_boundaries.end; i++) { ValueType tmp = ValueType(); lambda(i, tmp); @@ -68,7 +68,7 @@ parallel_reduce(const Impl::TeamThreadRangeBoundariesStruct< } else { #pragma omp declare reduction(custom:ValueType : omp_out += omp_in) -#pragma omp for reduction(custom : TeamThread_scratch[:1]) +#pragma omp for reduction(custom : TeamThread_scratch[ : 1]) for (iType i = loop_boundaries.start; i < loop_boundaries.end; i++) { ValueType tmp = ValueType(); lambda(i, tmp); @@ -90,11 +90,10 @@ parallel_reduce(const Impl::TeamThreadRangeBoundariesStruct< const Lambda& lambda, ReducerType result) { using ValueType = typename ReducerType::value_type; -#pragma omp declare reduction( \ - custominner:ValueType \ - : Impl::OpenMPTargetReducerWrapper ::join(omp_out, omp_in)) \ - initializer( \ - Impl::OpenMPTargetReducerWrapper ::init(omp_priv)) +#pragma omp declare reduction(custominner \ +:ValueType : Impl::OpenMPTargetReducerWrapper::join(omp_out, \ + omp_in)) \ + initializer(Impl::OpenMPTargetReducerWrapper::init(omp_priv)) // FIXME_OPENMPTARGET - Make sure that if its an array reduction, number of // elements in the array <= 32. For reduction we allocate, 16 bytes per @@ -109,7 +108,7 @@ parallel_reduce(const Impl::TeamThreadRangeBoundariesStruct< Impl::OpenMPTargetReducerWrapper::init(TeamThread_scratch[0]); #pragma omp barrier -#pragma omp for reduction(custominner : TeamThread_scratch[:1]) +#pragma omp for reduction(custominner : TeamThread_scratch[ : 1]) for (iType i = loop_boundaries.start; i < loop_boundaries.end; i++) { lambda(i, TeamThread_scratch[0]); } @@ -132,11 +131,10 @@ parallel_reduce(const Impl::TeamThreadRangeBoundariesStruct< ValueType* TeamThread_scratch = static_cast(loop_boundaries.team.impl_reduce_scratch()); -#pragma omp declare reduction( \ - omp_red_teamthread_reducer:ValueType \ - : Impl::OpenMPTargetReducerWrapper ::join(omp_out, omp_in)) \ - initializer( \ - Impl::OpenMPTargetReducerWrapper ::init(omp_priv)) +#pragma omp declare reduction(omp_red_teamthread_reducer \ +:ValueType : Impl::OpenMPTargetReducerWrapper::join(omp_out, \ + omp_in)) \ + initializer(Impl::OpenMPTargetReducerWrapper::init(omp_priv)) #pragma omp barrier ValueType tmp; @@ -145,8 +143,9 @@ parallel_reduce(const Impl::TeamThreadRangeBoundariesStruct< #pragma omp barrier iType team_size = iType(omp_get_num_threads()); -#pragma omp for reduction(omp_red_teamthread_reducer \ - : TeamThread_scratch[:1]) schedule(static, 1) +#pragma omp for reduction( \ + omp_red_teamthread_reducer : TeamThread_scratch[ : 1]) \ + schedule(static, 1) for (iType t = 0; t < team_size; t++) { ValueType tmp2; result.init(tmp2); @@ -259,11 +258,10 @@ parallel_reduce(const Impl::ThreadVectorRangeBoundariesStruct< const Lambda& lambda, ReducerType const& result) { using ValueType = typename ReducerType::value_type; -#pragma omp declare reduction( \ - custom:ValueType \ - : Impl::OpenMPTargetReducerWrapper ::join(omp_out, omp_in)) \ - initializer( \ - Impl::OpenMPTargetReducerWrapper ::init(omp_priv)) +#pragma omp declare reduction(custom \ +:ValueType : Impl::OpenMPTargetReducerWrapper::join(omp_out, \ + omp_in)) \ + initializer(Impl::OpenMPTargetReducerWrapper::init(omp_priv)) ValueType vector_reduce; Impl::OpenMPTargetReducerWrapper::init(vector_reduce); @@ -329,7 +327,7 @@ KOKKOS_INLINE_FUNCTION void parallel_reduce( #pragma omp barrier if constexpr (std::is_arithmetic::value) { -#pragma omp for simd reduction(+ : TeamVector_scratch[:1]) +#pragma omp for simd reduction(+ : TeamVector_scratch[ : 1]) for (iType i = loop_boundaries.start; i < loop_boundaries.end; i++) { ValueType tmp = ValueType(); lambda(i, tmp); @@ -338,7 +336,7 @@ KOKKOS_INLINE_FUNCTION void parallel_reduce( } else { #pragma omp declare reduction(custom:ValueType : omp_out += omp_in) -#pragma omp for simd reduction(custom : TeamVector_scratch[:1]) +#pragma omp for simd reduction(custom : TeamVector_scratch[ : 1]) for (iType i = loop_boundaries.start; i < loop_boundaries.end; i++) { ValueType tmp = ValueType(); lambda(i, tmp); @@ -363,11 +361,10 @@ parallel_reduce(const Impl::TeamVectorRangeBoundariesStruct< static_assert(sizeof(ValueType) <= Impl::OpenMPTargetExecTeamMember::TEAM_REDUCE_SIZE); -#pragma omp declare reduction( \ - custom:ValueType \ - : Impl::OpenMPTargetReducerWrapper ::join(omp_out, omp_in)) \ - initializer( \ - Impl::OpenMPTargetReducerWrapper ::init(omp_priv)) +#pragma omp declare reduction(custom \ +:ValueType : Impl::OpenMPTargetReducerWrapper::join(omp_out, \ + omp_in)) \ + initializer(Impl::OpenMPTargetReducerWrapper::init(omp_priv)) ValueType* TeamVector_scratch = static_cast(loop_boundaries.team.impl_reduce_scratch()); @@ -376,7 +373,7 @@ parallel_reduce(const Impl::TeamVectorRangeBoundariesStruct< Impl::OpenMPTargetReducerWrapper::init(TeamVector_scratch[0]); #pragma omp barrier -#pragma omp for simd reduction(custom : TeamVector_scratch[:1]) +#pragma omp for simd reduction(custom : TeamVector_scratch[ : 1]) for (iType i = loop_boundaries.start; i < loop_boundaries.end; i++) { lambda(i, TeamVector_scratch[0]); } @@ -400,11 +397,10 @@ parallel_reduce(const Impl::TeamVectorRangeBoundariesStruct< ValueType* TeamVector_scratch = static_cast(loop_boundaries.team.impl_reduce_scratch()); -#pragma omp declare reduction( \ - omp_red_teamthread_reducer:ValueType \ - : Impl::OpenMPTargetReducerWrapper ::join(omp_out, omp_in)) \ - initializer( \ - Impl::OpenMPTargetReducerWrapper ::init(omp_priv)) +#pragma omp declare reduction(omp_red_teamthread_reducer \ +:ValueType : Impl::OpenMPTargetReducerWrapper::join(omp_out, \ + omp_in)) \ + initializer(Impl::OpenMPTargetReducerWrapper::init(omp_priv)) #pragma omp barrier ValueType tmp; @@ -413,8 +409,9 @@ parallel_reduce(const Impl::TeamVectorRangeBoundariesStruct< #pragma omp barrier iType team_size = iType(omp_get_num_threads()); -#pragma omp for simd reduction(omp_red_teamthread_reducer \ - : TeamVector_scratch[:1]) schedule(static, 1) +#pragma omp for simd reduction( \ + omp_red_teamthread_reducer : TeamVector_scratch[ : 1]) \ + schedule(static, 1) for (iType t = 0; t < team_size; t++) { ValueType tmp2; result.init(tmp2); @@ -443,8 +440,7 @@ class ParallelReduce scratch_memory_lock( - OpenMPTargetExec::m_mutex_scratch_ptr); - const FunctorType& functor = m_functor_reducer.get_functor(); + m_policy.space().impl_internal_space_instance()->m_mutex_scratch_ptr); + auto const functor = + Kokkos::Experimental::Impl::FunctorAdapter( + m_functor_reducer.get_functor()); if constexpr (FunctorHasJoin) { ParReduceSpecialize::execute_init_join(functor, m_policy, m_result_ptr, m_result_ptr_on_device); diff --git a/lib/kokkos/core/src/OpenMPTarget/Kokkos_OpenMPTarget_ParallelScan_Range.hpp b/lib/kokkos/core/src/OpenMPTarget/Kokkos_OpenMPTarget_ParallelScan_Range.hpp index 29df0163c8..ec8a96cb2f 100644 --- a/lib/kokkos/core/src/OpenMPTarget/Kokkos_OpenMPTarget_ParallelScan_Range.hpp +++ b/lib/kokkos/core/src/OpenMPTarget/Kokkos_OpenMPTarget_ParallelScan_Range.hpp @@ -20,6 +20,7 @@ #include #include #include +#include namespace Kokkos { namespace Impl { @@ -30,7 +31,6 @@ class ParallelScan, protected: using Policy = Kokkos::RangePolicy; - using WorkTag = typename Policy::work_tag; using Member = typename Policy::member_type; using idx_type = typename Policy::index_type; @@ -48,18 +48,8 @@ class ParallelScan, value_type* m_result_ptr; const bool m_result_ptr_device_accessible; - template - std::enable_if_t::value> call_with_tag( - const FunctorType& f, const idx_type& idx, value_type& val, - const bool& is_final) const { - f(idx, val, is_final); - } - template - std::enable_if_t::value> call_with_tag( - const FunctorType& f, const idx_type& idx, value_type& val, - const bool& is_final) const { - f(WorkTag(), idx, val, is_final); - } + using FunctorAdapter = + Kokkos::Experimental::Impl::FunctorAdapter; public: void impl_execute( @@ -77,8 +67,10 @@ class ParallelScan, idx_type team_size = 128; auto a_functor_reducer = m_functor_reducer; -#pragma omp target teams distribute map(to \ - : a_functor_reducer) num_teams(nteams) + auto a_functor = FunctorAdapter(m_functor_reducer.get_functor()); + +#pragma omp target teams distribute map(to : a_functor_reducer, a_functor) \ + num_teams(nteams) for (idx_type team_id = 0; team_id < n_chunks; ++team_id) { const typename Analysis::Reducer& reducer = a_functor_reducer.get_reducer(); @@ -91,9 +83,8 @@ class ParallelScan, const idx_type idx = local_offset + i; value_type val; reducer.init(&val); - if (idx < N) - call_with_tag(a_functor_reducer.get_functor(), idx, val, - false); + if (idx < N) a_functor(idx, val, false); + element_values(team_id, i) = val; } #pragma omp barrier @@ -120,9 +111,8 @@ class ParallelScan, } } -#pragma omp target teams distribute map(to \ - : a_functor_reducer) num_teams(nteams) \ - thread_limit(team_size) +#pragma omp target teams distribute map(to : a_functor_reducer, a_functor) \ + num_teams(nteams) thread_limit(team_size) for (idx_type team_id = 0; team_id < n_chunks; ++team_id) { const typename Analysis::Reducer& reducer = a_functor_reducer.get_reducer(); @@ -145,12 +135,7 @@ class ParallelScan, #if defined(KOKKOS_ARCH_AMD_GPU) && !defined(KOKKOS_ARCH_AMD_GFX1030) && \ !defined(KOKKOS_ARCH_AMD_GFX1100) && !defined(KOKKOS_ARCH_AMD_GFX1103) if constexpr (Analysis::Reducer::has_join_member_function()) { - if constexpr (std::is_void_v) - a_functor_reducer.get_functor().join(local_offset_value, - offset_value); - else - a_functor_reducer.get_functor().join( - WorkTag{}, local_offset_value, offset_value); + a_functor.get_functor().join(local_offset_value, offset_value); } else local_offset_value += offset_value; #else @@ -158,9 +143,8 @@ class ParallelScan, #endif } else local_offset_value = offset_value; - if (idx < N) - call_with_tag(a_functor_reducer.get_functor(), idx, - local_offset_value, true); + if (idx < N) a_functor(idx, local_offset_value, true); + if (idx == N - 1 && m_result_ptr_device_accessible) *m_result_ptr = local_offset_value; } @@ -169,9 +153,9 @@ class ParallelScan, } void execute() const { - OpenMPTargetExec::verify_is_process( + Experimental::Impl::OpenMPTargetInternal::verify_is_process( "Kokkos::Experimental::OpenMPTarget parallel_for"); - OpenMPTargetExec::verify_initialized( + Experimental::Impl::OpenMPTargetInternal::verify_initialized( "Kokkos::Experimental::OpenMPTarget parallel_for"); const idx_type N = m_policy.end() - m_policy.begin(); const idx_type chunk_size = 128; @@ -179,7 +163,7 @@ class ParallelScan, // Only let one ParallelReduce instance at a time use the scratch memory. std::scoped_lock scratch_memory_lock( - OpenMPTargetExec::m_mutex_scratch_ptr); + m_policy.space().impl_internal_space_instance()->m_mutex_scratch_ptr); // This could be scratch memory per team Kokkos::View, public: void execute() const { - OpenMPTargetExec::verify_is_process( + Experimental::Impl::OpenMPTargetInternal::verify_is_process( "Kokkos::Experimental::OpenMPTarget parallel_for"); - OpenMPTargetExec::verify_initialized( + Experimental::Impl::OpenMPTargetInternal::verify_initialized( "Kokkos::Experimental::OpenMPTarget parallel_for"); const int64_t N = base_t::m_policy.end() - base_t::m_policy.begin(); const int chunk_size = 128; @@ -231,7 +215,9 @@ class ParallelScanWithTotal, if (N > 0) { // Only let one ParallelReduce instance at a time use the scratch memory. std::scoped_lock scratch_memory_lock( - OpenMPTargetExec::m_mutex_scratch_ptr); + base_t::m_policy.space() + .impl_internal_space_instance() + ->m_mutex_scratch_ptr); // This could be scratch memory per team Kokkos::View #include #include +#include namespace Kokkos { namespace Impl { @@ -72,7 +73,6 @@ template , ReducerType, PointerType, ValueType> { using PolicyType = Kokkos::RangePolicy; - using TagType = typename PolicyType::work_tag; using ReducerTypeFwd = std::conditional_t::value, FunctorType, ReducerType>; @@ -82,12 +82,15 @@ struct ParallelReduceSpecialize, using ParReduceCopy = ParallelReduceCopy; - static void execute_reducer(const FunctorType& f, const PolicyType& p, + using FunctorAdapter = + Kokkos::Experimental::Impl::FunctorAdapter; + + static void execute_reducer(const FunctorAdapter& f, const PolicyType& p, PointerType result_ptr, bool ptr_on_device) { - OpenMPTargetExec::verify_is_process( + Experimental::Impl::OpenMPTargetInternal::verify_is_process( "Kokkos::Experimental::OpenMPTarget RangePolicy " "parallel_reduce:reducer"); - OpenMPTargetExec::verify_initialized( + Experimental::Impl::OpenMPTargetInternal::verify_initialized( "Kokkos::Experimental::OpenMPTarget RangePolicy " "parallel_reduce:reducer"); const auto begin = p.begin(); @@ -104,33 +107,27 @@ struct ParallelReduceSpecialize, return; } -#pragma omp declare reduction( \ - custom:ValueType \ - : OpenMPTargetReducerWrapper ::join(omp_out, omp_in)) \ - initializer(OpenMPTargetReducerWrapper ::init(omp_priv)) +#pragma omp declare reduction(custom \ +:ValueType : OpenMPTargetReducerWrapper::join(omp_out, omp_in)) \ + initializer(OpenMPTargetReducerWrapper::init(omp_priv)) -#pragma omp target teams distribute parallel for map(to \ - : f) reduction(custom \ - : result) +#pragma omp target teams distribute parallel for map(to : f) \ + reduction(custom : result) for (auto i = begin; i < end; ++i) { - if constexpr (std::is_void_v) { - f(i, result); - } else { - f(TagType(), i, result); - } + f(i, result); } ParReduceCopy::memcpy_result(result_ptr, &result, sizeof(ValueType), ptr_on_device); } - template - static void execute_array(const FunctorType& f, const PolicyType& p, + template + static void execute_array(const FunctorAdapter& f, const PolicyType& p, PointerType result_ptr, bool ptr_on_device) { - OpenMPTargetExec::verify_is_process( + Experimental::Impl::OpenMPTargetInternal::verify_is_process( "Kokkos::Experimental::OpenMPTarget RangePolicy " "parallel_reduce:array_reduction"); - OpenMPTargetExec::verify_initialized( + Experimental::Impl::OpenMPTargetInternal::verify_initialized( "Kokkos::Experimental::OpenMPTarget RangePolicy " "parallel_reduce:array_reduction"); const auto begin = p.begin(); @@ -150,27 +147,14 @@ struct ParallelReduceSpecialize, // Case where reduction is on a native data type. if constexpr (std::is_arithmetic::value) { -#pragma omp target teams distribute parallel for \ - map(to:f) reduction(+: result) - for (auto i = begin; i < end; ++i) - - if constexpr (std::is_void_v) { - f(i, result); - } else { - f(TagType(), i, result); - } +#pragma omp target teams distribute parallel for map(to : f) \ + reduction(+ : result) + for (auto i = begin; i < end; ++i) f(i, result); } else { #pragma omp declare reduction(custom:ValueType : omp_out += omp_in) -#pragma omp target teams distribute parallel for map(to \ - : f) reduction(custom \ - : result) - for (auto i = begin; i < end; ++i) - - if constexpr (std::is_void_v) { - f(i, result); - } else { - f(TagType(), i, result); - } +#pragma omp target teams distribute parallel for map(to : f) \ + reduction(custom : result) + for (auto i = begin; i < end; ++i) f(i, result); } ParReduceCopy::memcpy_result(result_ptr, &result, sizeof(ValueType), @@ -186,13 +170,10 @@ struct ParallelReduceSpecialize, ptr_on_device); return; } -#pragma omp target teams distribute parallel for map(to:f) reduction(+:result[:NumReductions]) +#pragma omp target teams distribute parallel for map(to : f) \ + reduction(+ : result[ : NumReductions]) for (auto i = begin; i < end; ++i) { - if constexpr (std::is_void_v) { - f(i, result); - } else { - f(TagType(), i, result); - } + f(i, result); } ParReduceCopy::memcpy_result( @@ -200,12 +181,12 @@ struct ParallelReduceSpecialize, } } - static void execute_init_join(const FunctorType& f, const PolicyType& p, + static void execute_init_join(const FunctorAdapter& f, const PolicyType& p, PointerType ptr, const bool ptr_on_device) { - OpenMPTargetExec::verify_is_process( + Experimental::Impl::OpenMPTargetInternal::verify_is_process( "Kokkos::Experimental::OpenMPTarget RangePolicy " "parallel_reduce:init_join"); - OpenMPTargetExec::verify_initialized( + Experimental::Impl::OpenMPTargetInternal::verify_initialized( "Kokkos::Experimental::OpenMPTarget RangePolicy " "parallel_reduce:init_join"); const auto begin = p.begin(); @@ -219,23 +200,25 @@ struct ParallelReduceSpecialize, const auto size = end - begin; - // FIXME_OPENMPTARGET: The team size and MAX_ACTIVE_THREADS are currently + // FIXME_OPENMPTARGET: The team size and concurrency are currently // based on NVIDIA-V100 and should be modifid to be based on the // architecture in the future. const int max_team_threads = 32; const int max_teams = - OpenMPTargetExec::MAX_ACTIVE_THREADS / max_team_threads; + p.space().impl_internal_space_instance()->concurrency() / + max_team_threads; // Number of elements in the reduction - const auto value_count = FunctorAnalysis::value_count(f); + const auto value_count = FunctorAnalysis::value_count(f.get_functor()); // Allocate scratch per active thread. Achieved by setting the first // parameter of `resize_scratch=1`. - OpenMPTargetExec::resize_scratch(1, 0, value_count * sizeof(ValueType), - std::numeric_limits::max()); - ValueType* scratch_ptr = - static_cast(OpenMPTargetExec::get_scratch_ptr()); + p.space().impl_internal_space_instance()->resize_scratch( + 1, 0, value_count * sizeof(ValueType), + std::numeric_limits::max()); + ValueType* scratch_ptr = static_cast( + p.space().impl_internal_space_instance()->get_scratch_ptr()); - typename FunctorAnalysis::Reducer final_reducer(f); + typename FunctorAnalysis::Reducer final_reducer(f.get_functor()); if (end <= begin) { #pragma omp target map(to : final_reducer) is_device_ptr(scratch_ptr) @@ -260,8 +243,7 @@ struct ParallelReduceSpecialize, } #pragma omp target teams num_teams(max_teams) thread_limit(max_team_threads) \ - map(to \ - : final_reducer) is_device_ptr(scratch_ptr) + map(to : final_reducer) is_device_ptr(scratch_ptr) { #pragma omp parallel { @@ -279,11 +261,7 @@ struct ParallelReduceSpecialize, // Accumulate partial results in thread specific storage. #pragma omp for simd for (auto i = team_begin; i < team_end; ++i) { - if constexpr (std::is_void_v) { - f(i, result); - } else { - f(TagType(), i, result); - } + f(i, result); } // Reduce all paritial results within a team. @@ -304,8 +282,7 @@ struct ParallelReduceSpecialize, int tree_neighbor_offset = 1; do { -#pragma omp target teams distribute parallel for simd map(to \ - : f) \ +#pragma omp target teams distribute parallel for simd map(to : f) \ is_device_ptr(scratch_ptr) for (int i = 0; i < max_teams - tree_neighbor_offset; i += 2 * tree_neighbor_offset) { @@ -344,7 +321,6 @@ template , ReducerType, PointerType, ValueType> { using PolicyType = TeamPolicyInternal; - using TagType = typename PolicyType::work_tag; using ReducerTypeFwd = std::conditional_t::value, FunctorType, ReducerType>; @@ -355,12 +331,15 @@ struct ParallelReduceSpecialize, using ParReduceCopy = ParallelReduceCopy; - static void execute_reducer(const FunctorType& f, const PolicyType& p, + using FunctorAdapter = + Kokkos::Experimental::Impl::FunctorAdapter; + + static void execute_reducer(const FunctorAdapter& f, const PolicyType& p, PointerType result_ptr, bool ptr_on_device) { - OpenMPTargetExec::verify_is_process( + Experimental::Impl::OpenMPTargetInternal::verify_is_process( "Kokkos::Experimental::OpenMPTarget TeamPolicy " "parallel_reduce:reducer"); - OpenMPTargetExec::verify_initialized( + Experimental::Impl::OpenMPTargetInternal::verify_initialized( "Kokkos::Experimental::OpenMPTarget TeamPolicy " "parallel_reduce:reducer"); @@ -370,9 +349,11 @@ struct ParallelReduceSpecialize, const size_t shmem_size_L0 = p.scratch_size(0, team_size); const size_t shmem_size_L1 = p.scratch_size(1, team_size); - OpenMPTargetExec::resize_scratch(PolicyType::member_type::TEAM_REDUCE_SIZE, - shmem_size_L0, shmem_size_L1, league_size); - void* scratch_ptr = OpenMPTargetExec::get_scratch_ptr(); + p.space().impl_internal_space_instance()->resize_scratch( + PolicyType::member_type::TEAM_REDUCE_SIZE, shmem_size_L0, shmem_size_L1, + league_size); + void* scratch_ptr = + p.space().impl_internal_space_instance()->get_scratch_ptr(); ValueType result = ValueType(); @@ -383,16 +364,15 @@ struct ParallelReduceSpecialize, int max_active_teams = omp_get_max_teams(); #else int max_active_teams = - std::min(OpenMPTargetExec::MAX_ACTIVE_THREADS / team_size, league_size); + std::min(p.space().concurrency() / team_size, league_size); #endif // If the league size is <=0, do not launch the kernel. if (max_active_teams <= 0) return; -#pragma omp declare reduction( \ - custom:ValueType \ - : OpenMPTargetReducerWrapper ::join(omp_out, omp_in)) \ - initializer(OpenMPTargetReducerWrapper ::init(omp_priv)) +#pragma omp declare reduction(custom \ +:ValueType : OpenMPTargetReducerWrapper::join(omp_out, omp_in)) \ + initializer(OpenMPTargetReducerWrapper::init(omp_priv)) #if !defined(KOKKOS_IMPL_OPENMPTARGET_HIERARCHICAL_INTEL_GPU) KOKKOS_IMPL_OMPTARGET_PRAGMA( @@ -414,16 +394,13 @@ struct ParallelReduceSpecialize, typename PolicyType::member_type team( league_id, league_size, team_size, vector_length, scratch_ptr, blockIdx, shmem_size_L0, shmem_size_L1); - if constexpr (std::is_void_v) - f(team, result); - else - f(TagType(), team, result); + f(team, result); } } #else #pragma omp target teams distribute firstprivate(f) is_device_ptr(scratch_ptr) \ - num_teams(max_active_teams) thread_limit(team_size) reduction(custom \ - : result) + num_teams(max_active_teams) thread_limit(team_size) \ + reduction(custom : result) for (int i = 0; i < league_size; i++) { #pragma omp parallel reduction(custom : result) { @@ -433,10 +410,7 @@ struct ParallelReduceSpecialize, typename PolicyType::member_type team(i, league_size, team_size, vector_length, scratch_ptr, i, shmem_size_L0, shmem_size_L1); - if constexpr (std::is_void_v) - f(team, result); - else - f(TagType(), team, result); + f(team, result); } } #endif @@ -447,12 +421,12 @@ struct ParallelReduceSpecialize, } template - static void execute_array(const FunctorType& f, const PolicyType& p, + static void execute_array(const FunctorAdapter& f, const PolicyType& p, PointerType result_ptr, bool ptr_on_device) { - OpenMPTargetExec::verify_is_process( + Experimental::Impl::OpenMPTargetInternal::verify_is_process( "Kokkos::Experimental::OpenMPTarget TeamPolicy " "parallel_reduce:array_reduction"); - OpenMPTargetExec::verify_initialized( + Experimental::Impl::OpenMPTargetInternal::verify_initialized( "Kokkos::Experimental::OpenMPTarget TeamPolicy " "parallel_reduce:array_reduction"); @@ -462,9 +436,11 @@ struct ParallelReduceSpecialize, const size_t shmem_size_L0 = p.scratch_size(0, team_size); const size_t shmem_size_L1 = p.scratch_size(1, team_size); - OpenMPTargetExec::resize_scratch(PolicyType::member_type::TEAM_REDUCE_SIZE, - shmem_size_L0, shmem_size_L1, league_size); - void* scratch_ptr = OpenMPTargetExec::get_scratch_ptr(); + p.space().impl_internal_space_instance()->resize_scratch( + PolicyType::member_type::TEAM_REDUCE_SIZE, shmem_size_L0, shmem_size_L1, + league_size); + void* scratch_ptr = + p.space().impl_internal_space_instance()->get_scratch_ptr(); // Maximum active teams possible. // FIXME_OPENMPTARGET: Cray compiler did not yet implement @@ -473,7 +449,7 @@ struct ParallelReduceSpecialize, int max_active_teams = omp_get_max_teams(); #else int max_active_teams = - std::min(OpenMPTargetExec::MAX_ACTIVE_THREADS / team_size, league_size); + std::min(p.space().concurrency() / team_size, league_size); #endif // If the league size is <=0, do not launch the kernel. @@ -504,19 +480,14 @@ struct ParallelReduceSpecialize, typename PolicyType::member_type team( league_id, league_size, team_size, vector_length, scratch_ptr, blockIdx, shmem_size_L0, shmem_size_L1); - if constexpr (std::is_void_v) - f(team, result); - else - f(TagType(), team, result); + f(team, result); } } } else { // Case where the reduction is on a non-native data type. #pragma omp declare reduction(custom:ValueType : omp_out += omp_in) #pragma omp target teams num_teams(max_active_teams) thread_limit(team_size) \ - map(to \ - : f) is_device_ptr(scratch_ptr) reduction(custom \ - : result) + map(to : f) is_device_ptr(scratch_ptr) reduction(custom : result) #pragma omp parallel reduction(custom : result) { if (omp_get_num_teams() > max_active_teams) @@ -531,10 +502,7 @@ struct ParallelReduceSpecialize, typename PolicyType::member_type team( league_id, league_size, team_size, vector_length, scratch_ptr, blockIdx, shmem_size_L0, shmem_size_L1); - if constexpr (std::is_void_v) - f(team, result); - else - f(TagType(), team, result); + f(team, result); } } } @@ -545,10 +513,10 @@ struct ParallelReduceSpecialize, } else { ValueType result[NumReductions] = {}; // Case where the reduction is on an array. -#pragma omp target teams num_teams(max_active_teams) thread_limit(team_size) map(to \ - : f) \ - is_device_ptr(scratch_ptr) reduction(+ : result[:NumReductions]) -#pragma omp parallel reduction(+ : result[:NumReductions]) +#pragma omp target teams num_teams(max_active_teams) thread_limit(team_size) \ + map(to : f) is_device_ptr(scratch_ptr) \ + reduction(+ : result[ : NumReductions]) +#pragma omp parallel reduction(+ : result[ : NumReductions]) { if (omp_get_num_teams() > max_active_teams) Kokkos::abort("`omp_set_num_teams` call was not respected.\n"); @@ -562,10 +530,7 @@ struct ParallelReduceSpecialize, typename PolicyType::member_type team( league_id, league_size, team_size, vector_length, scratch_ptr, blockIdx, shmem_size_L0, shmem_size_L1); - if constexpr (std::is_void_v) - f(team, result); - else - f(TagType(), team, result); + f(team, result); } } @@ -577,12 +542,12 @@ struct ParallelReduceSpecialize, // FIXME_OPENMPTARGET : This routine is a copy from `parallel_reduce` over // RangePolicy. Need a new implementation. - static void execute_init_join(const FunctorType& f, const PolicyType& p, + static void execute_init_join(const FunctorAdapter& f, const PolicyType& p, PointerType ptr, const bool ptr_on_device) { - OpenMPTargetExec::verify_is_process( + Experimental::Impl::OpenMPTargetInternal::verify_is_process( "Kokkos::Experimental::OpenMPTarget TeamPolicy " "parallel_reduce:init_join "); - OpenMPTargetExec::verify_initialized( + Experimental::Impl::OpenMPTargetInternal::verify_initialized( "Kokkos::Experimental::OpenMPTarget TeamPolicy " "parallel_reduce:init_join"); using FunctorAnalysis = @@ -611,13 +576,14 @@ struct ParallelReduceSpecialize, const auto nteams = league_size; // Number of elements in the reduction - const auto value_count = FunctorAnalysis::value_count(f); + const auto value_count = FunctorAnalysis::value_count(f.get_functor()); // Allocate scratch per active thread. - OpenMPTargetExec::resize_scratch(1, 0, value_count * sizeof(ValueType), - league_size); - void* scratch_ptr = OpenMPTargetExec::get_scratch_ptr(); - typename FunctorAnalysis::Reducer final_reducer(f); + p.space().impl_internal_space_instance()->resize_scratch( + 1, 0, value_count * sizeof(ValueType), league_size); + void* scratch_ptr = + p.space().impl_internal_space_instance()->get_scratch_ptr(); + typename FunctorAnalysis::Reducer final_reducer(f.get_functor()); if (end <= begin) { // If there is no work to be done, copy back the initialized values and @@ -661,11 +627,7 @@ struct ParallelReduceSpecialize, typename PolicyType::member_type team( league_id, league_size, team_size, vector_length, scratch_ptr, team_num, shmem_size_L0, shmem_size_L1); - if constexpr (std::is_void_v) { - f(team, result); - } else { - f(TagType(), team, result); - } + f(team, result); } } // end parallel } // end target @@ -673,7 +635,7 @@ struct ParallelReduceSpecialize, int tree_neighbor_offset = 1; do { #pragma omp target teams distribute parallel for simd firstprivate( \ - final_reducer) is_device_ptr(scratch_ptr) + final_reducer) is_device_ptr(scratch_ptr) for (int i = 0; i < nteams - tree_neighbor_offset; i += 2 * tree_neighbor_offset) { ValueType* team_scratch = static_cast(scratch_ptr); diff --git a/lib/kokkos/core/src/OpenMPTarget/Kokkos_OpenMPTarget_Reducer.hpp b/lib/kokkos/core/src/OpenMPTarget/Kokkos_OpenMPTarget_Reducer.hpp index 9b578aca11..4308fb042a 100644 --- a/lib/kokkos/core/src/OpenMPTarget/Kokkos_OpenMPTarget_Reducer.hpp +++ b/lib/kokkos/core/src/OpenMPTarget/Kokkos_OpenMPTarget_Reducer.hpp @@ -34,9 +34,6 @@ struct OpenMPTargetReducerWrapper { KOKKOS_INLINE_FUNCTION static void join(value_type&, const value_type&) = delete; - KOKKOS_INLINE_FUNCTION - static void join(volatile value_type&, const volatile value_type&) = delete; - KOKKOS_INLINE_FUNCTION static void init(value_type&) = delete; }; @@ -51,11 +48,6 @@ struct OpenMPTargetReducerWrapper> { KOKKOS_INLINE_FUNCTION static void join(value_type& dest, const value_type& src) { dest += src; } - KOKKOS_INLINE_FUNCTION - static void join(volatile value_type& dest, const volatile value_type& src) { - dest += src; - } - KOKKOS_INLINE_FUNCTION static void init(value_type& val) { val = reduction_identity::sum(); @@ -72,11 +64,6 @@ struct OpenMPTargetReducerWrapper> { KOKKOS_INLINE_FUNCTION static void join(value_type& dest, const value_type& src) { dest *= src; } - KOKKOS_INLINE_FUNCTION - static void join(volatile value_type& dest, const volatile value_type& src) { - dest *= src; - } - KOKKOS_INLINE_FUNCTION static void init(value_type& val) { val = reduction_identity::prod(); @@ -95,11 +82,6 @@ struct OpenMPTargetReducerWrapper> { if (src < dest) dest = src; } - KOKKOS_INLINE_FUNCTION - static void join(volatile value_type& dest, const volatile value_type& src) { - if (src < dest) dest = src; - } - KOKKOS_INLINE_FUNCTION static void init(value_type& val) { val = reduction_identity::min(); @@ -118,11 +100,6 @@ struct OpenMPTargetReducerWrapper> { if (src > dest) dest = src; } - KOKKOS_INLINE_FUNCTION - static void join(volatile value_type& dest, const volatile value_type& src) { - if (src > dest) dest = src; - } - // Required KOKKOS_INLINE_FUNCTION static void init(value_type& val) { @@ -141,11 +118,6 @@ struct OpenMPTargetReducerWrapper> { dest = dest && src; } - KOKKOS_INLINE_FUNCTION - static void join(volatile value_type& dest, const volatile value_type& src) { - dest = dest && src; - } - KOKKOS_INLINE_FUNCTION static void init(value_type& val) { val = reduction_identity::land(); @@ -166,11 +138,6 @@ struct OpenMPTargetReducerWrapper> { dest = dest || src; } - KOKKOS_INLINE_FUNCTION - static void join(volatile value_type& dest, const volatile value_type& src) { - dest = dest || src; - } - KOKKOS_INLINE_FUNCTION static void init(value_type& val) { val = reduction_identity::lor(); @@ -189,11 +156,6 @@ struct OpenMPTargetReducerWrapper> { dest = dest & src; } - KOKKOS_INLINE_FUNCTION - static void join(volatile value_type& dest, const volatile value_type& src) { - dest = dest & src; - } - KOKKOS_INLINE_FUNCTION static void init(value_type& val) { val = reduction_identity::band(); @@ -212,11 +174,6 @@ struct OpenMPTargetReducerWrapper> { dest = dest | src; } - KOKKOS_INLINE_FUNCTION - static void join(volatile value_type& dest, const volatile value_type& src) { - dest = dest | src; - } - KOKKOS_INLINE_FUNCTION static void init(value_type& val) { val = reduction_identity::bor(); @@ -236,12 +193,12 @@ struct OpenMPTargetReducerWrapper> { // Required KOKKOS_INLINE_FUNCTION static void join(value_type& dest, const value_type& src) { - if (src.val < dest.val) dest = src; - } - - KOKKOS_INLINE_FUNCTION - static void join(volatile value_type& dest, const volatile value_type& src) { - if (src.val < dest.val) dest = src; + if (src.val < dest.val) + dest = src; + else if (src.val == dest.val && + dest.loc == reduction_identity::min()) { + dest.loc = src.loc; + } } KOKKOS_INLINE_FUNCTION @@ -263,12 +220,12 @@ struct OpenMPTargetReducerWrapper> { KOKKOS_INLINE_FUNCTION static void join(value_type& dest, const value_type& src) { - if (src.val > dest.val) dest = src; - } - - KOKKOS_INLINE_FUNCTION - static void join(volatile value_type& dest, const volatile value_type& src) { - if (src.val > dest.val) dest = src; + if (src.val > dest.val) + dest = src; + else if (src.val == dest.val && + dest.loc == reduction_identity::min()) { + dest.loc = src.loc; + } } KOKKOS_INLINE_FUNCTION @@ -298,16 +255,6 @@ struct OpenMPTargetReducerWrapper> { } } - KOKKOS_INLINE_FUNCTION - static void join(volatile value_type& dest, const volatile value_type& src) { - if (src.min_val < dest.min_val) { - dest.min_val = src.min_val; - } - if (src.max_val > dest.max_val) { - dest.max_val = src.max_val; - } - } - KOKKOS_INLINE_FUNCTION static void init(value_type& val) { val.max_val = reduction_identity::max(); @@ -331,22 +278,16 @@ struct OpenMPTargetReducerWrapper> { if (src.min_val < dest.min_val) { dest.min_val = src.min_val; dest.min_loc = src.min_loc; - } - if (src.max_val > dest.max_val) { - dest.max_val = src.max_val; - dest.max_loc = src.max_loc; - } - } - - KOKKOS_INLINE_FUNCTION - static void join(volatile value_type& dest, const volatile value_type& src) { - if (src.min_val < dest.min_val) { - dest.min_val = src.min_val; + } else if (dest.min_val == src.min_val && + dest.min_loc == reduction_identity::min()) { dest.min_loc = src.min_loc; } if (src.max_val > dest.max_val) { dest.max_val = src.max_val; dest.max_loc = src.max_loc; + } else if (dest.max_val == src.max_val && + dest.max_loc == reduction_identity::min()) { + dest.max_loc = src.max_loc; } } @@ -385,15 +326,6 @@ struct OpenMPTargetReducerWrapper> { } } - KOKKOS_INLINE_FUNCTION - static void join(volatile value_type& dest, const volatile value_type& src) { - if (dest.val < src.val) { - dest = src; - } else if (!(src.val < dest.val)) { - dest.loc = (src.loc < dest.loc) ? src.loc : dest.loc; - } - } - KOKKOS_INLINE_FUNCTION static void init(value_type& val) { val.val = reduction_identity::max(); @@ -428,15 +360,6 @@ struct OpenMPTargetReducerWrapper> { } } - KOKKOS_INLINE_FUNCTION - static void join(volatile value_type& dest, const volatile value_type& src) { - if (src.val < dest.val) { - dest = src; - } else if (!(dest.val < src.val)) { - dest.loc = (src.loc < dest.loc) ? src.loc : dest.loc; - } - } - KOKKOS_INLINE_FUNCTION static void init(value_type& val) { val.val = reduction_identity::min(); @@ -480,23 +403,6 @@ struct OpenMPTargetReducerWrapper> { } } - KOKKOS_INLINE_FUNCTION - static void join(volatile value_type& dest, const volatile value_type& src) { - if (src.min_val < dest.min_val) { - dest.min_val = src.min_val; - dest.min_loc = src.min_loc; - } else if (!(dest.min_val < src.min_val)) { - dest.min_loc = (src.min_loc < dest.min_loc) ? src.min_loc : dest.min_loc; - } - - if (dest.max_val < src.max_val) { - dest.max_val = src.max_val; - dest.max_loc = src.max_loc; - } else if (!(src.max_val < dest.max_val)) { - dest.max_loc = (src.max_loc > dest.max_loc) ? src.max_loc : dest.max_loc; - } - } - KOKKOS_INLINE_FUNCTION static void init(value_type& val) { val.max_val = reduction_identity::max(); @@ -531,13 +437,6 @@ struct OpenMPTargetReducerWrapper> { : dest.min_loc_true; } - KOKKOS_INLINE_FUNCTION - static void join(volatile value_type& dest, const volatile value_type& src) { - dest.min_loc_true = (src.min_loc_true < dest.min_loc_true) - ? src.min_loc_true - : dest.min_loc_true; - } - KOKKOS_INLINE_FUNCTION static void init(value_type& val) { val.min_loc_true = reduction_identity::min(); @@ -569,13 +468,6 @@ struct OpenMPTargetReducerWrapper> { : dest.max_loc_true; } - KOKKOS_INLINE_FUNCTION - static void join(volatile value_type& dest, const volatile value_type& src) { - dest.max_loc_true = (src.max_loc_true > dest.max_loc_true) - ? src.max_loc_true - : dest.max_loc_true; - } - KOKKOS_INLINE_FUNCTION static void init(value_type& val) { val.max_loc_true = reduction_identity::max(); @@ -611,17 +503,6 @@ struct OpenMPTargetReducerWrapper> { : src.min_loc_false; } - KOKKOS_INLINE_FUNCTION - static void join(volatile value_type& dest, const volatile value_type& src) { - dest.max_loc_true = (dest.max_loc_true < src.max_loc_true) - ? src.max_loc_true - : dest.max_loc_true; - - dest.min_loc_false = (dest.min_loc_false < src.min_loc_false) - ? dest.min_loc_false - : src.min_loc_false; - } - KOKKOS_INLINE_FUNCTION static void init(value_type& val) { val.max_loc_true = ::Kokkos::reduction_identity::max(); @@ -654,13 +535,6 @@ struct OpenMPTargetReducerWrapper> { : src.min_loc_false; } - KOKKOS_INLINE_FUNCTION - static void join(volatile value_type& dest, const volatile value_type& src) { - dest.min_loc_false = (dest.min_loc_false < src.min_loc_false) - ? dest.min_loc_false - : src.min_loc_false; - } - KOKKOS_INLINE_FUNCTION static void init(value_type& val) { val.min_loc_false = ::Kokkos::reduction_identity::min(); diff --git a/lib/kokkos/core/src/OpenMPTarget/Kokkos_OpenMPTarget_Task.cpp b/lib/kokkos/core/src/OpenMPTarget/Kokkos_OpenMPTarget_Task.cpp deleted file mode 100644 index 458c4c9a43..0000000000 --- a/lib/kokkos/core/src/OpenMPTarget/Kokkos_OpenMPTarget_Task.cpp +++ /dev/null @@ -1,251 +0,0 @@ -//@HEADER -// ************************************************************************ -// -// Kokkos v. 4.0 -// Copyright (2022) National Technology & Engineering -// Solutions of Sandia, LLC (NTESS). -// -// Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. -// -// Part of Kokkos, under the Apache License v2.0 with LLVM Exceptions. -// See https://kokkos.org/LICENSE for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// -//@HEADER - -#ifndef KOKKOS_IMPL_PUBLIC_INCLUDE -#define KOKKOS_IMPL_PUBLIC_INCLUDE -#endif - -#include - -#if defined(KOKKOS_ENABLE_OPENMPTARGET) && defined(KOKKOS_ENABLE_TASKPOLICY) - -#include - -//---------------------------------------------------------------------------- -//---------------------------------------------------------------------------- - -namespace Kokkos { -namespace Impl { - -template class TaskQueue; - -//---------------------------------------------------------------------------- - -TaskExec::TaskExec() - : m_self_exec(0), - m_team_exec(0), - m_sync_mask(0), - m_sync_value(0), - m_sync_step(0), - m_group_rank(0), - m_team_rank(0), - m_team_size(1) {} - -TaskExec::TaskExec( - Kokkos::Impl::OpenMPTargetExec &arg_exec, int const arg_team_size) - : m_self_exec(&arg_exec), - m_team_exec(arg_exec.pool_rev(arg_exec.pool_rank_rev() / arg_team_size)), - m_sync_mask(0), - m_sync_value(0), - m_sync_step(0), - m_group_rank(arg_exec.pool_rank_rev() / arg_team_size), - m_team_rank(arg_exec.pool_rank_rev() % arg_team_size), - m_team_size(arg_team_size) { - // This team spans - // m_self_exec->pool_rev( team_size * group_rank ) - // m_self_exec->pool_rev( team_size * ( group_rank + 1 ) - 1 ) - - int64_t volatile *const sync = (int64_t *)m_self_exec->scratch_reduce(); - - sync[0] = int64_t(0); - sync[1] = int64_t(0); - - for (int i = 0; i < m_team_size; ++i) { - m_sync_value |= int64_t(1) << (8 * i); - m_sync_mask |= int64_t(3) << (8 * i); - } - - Kokkos::memory_fence(); -} - -void TaskExec::team_barrier_impl() const { - if (m_team_exec->scratch_reduce_size() < int(2 * sizeof(int64_t))) { - Kokkos::abort("TaskQueue scratch_reduce memory too small"); - } - - // Use team shared memory to synchronize. - // Alternate memory locations between barriers to avoid a sequence - // of barriers overtaking one another. - - int64_t volatile *const sync = - ((int64_t *)m_team_exec->scratch_reduce()) + (m_sync_step & 0x01); - - // This team member sets one byte within the sync variable - int8_t volatile *const sync_self = ((int8_t *)sync) + m_team_rank; - - *sync_self = int8_t(m_sync_value & 0x03); // signal arrival - - while (m_sync_value != *sync) - ; // wait for team to arrive - - ++m_sync_step; - - if (0 == (0x01 & m_sync_step)) { // Every other step - m_sync_value ^= m_sync_mask; - if (1000 < m_sync_step) m_sync_step = 0; - } -} - -//---------------------------------------------------------------------------- - -void TaskQueueSpecialization::execute( - TaskQueue *const queue) { - using execution_space = Kokkos::Experimental::OpenMPTarget; - using queue_type = TaskQueue; - using task_root_type = TaskBase; - using PoolExec = Kokkos::Impl::OpenMPTargetExec; - using Member = TaskExec; - - task_root_type *const end = (task_root_type *)task_root_type::EndTag; - - // Required: team_size <= 8 - - const int team_size = PoolExec::pool_size(2); // Threads per core - // const int team_size = PoolExec::pool_size(1); // Threads per NUMA - - if (8 < team_size) { - Kokkos::abort("TaskQueue unsupported team size"); - } - -#pragma omp parallel - { - PoolExec &self = *PoolExec::get_thread_omp(); - - Member single_exec; - Member team_exec(self, team_size); - - // Team shared memory - task_root_type *volatile *const task_shared = - (task_root_type **)team_exec.m_team_exec->scratch_thread(); - -// Barrier across entire OpenMPTarget thread pool to insure initialization -#pragma omp barrier - - // Loop until all queues are empty and no tasks in flight - - do { - task_root_type *task = 0; - - // Each team lead attempts to acquire either a thread team task - // or a single thread task for the team. - - if (0 == team_exec.team_rank()) { - task = 0 < *((volatile int *)&queue->m_ready_count) ? end : 0; - - // Loop by priority and then type - for (int i = 0; i < queue_type::NumQueue && end == task; ++i) { - for (int j = 0; j < 2 && end == task; ++j) { - task = queue_type::pop_task(&queue->m_ready[i][j]); - } - } - } - - // Team lead broadcast acquired task to team members: - - if (1 < team_exec.team_size()) { - if (0 == team_exec.team_rank()) *task_shared = task; - - // Fence to be sure task_shared is stored before the barrier - Kokkos::memory_fence(); - - // Whole team waits for every team member to reach this statement - team_exec.team_barrier(); - - // Fence to be sure task_shared is stored - Kokkos::memory_fence(); - - task = *task_shared; - } - - if (0 == task) break; // 0 == m_ready_count - - if (end == task) { - // All team members wait for whole team to reach this statement. - // Is necessary to prevent task_shared from being updated - // before it is read by all threads. - team_exec.team_barrier(); - } else if (task_root_type::TaskTeam == task->m_task_type) { - // Thread Team Task - (*task->m_apply)(task, &team_exec); - - // The m_apply function performs a barrier - - if (0 == team_exec.team_rank()) { - // team member #0 completes the task, which may delete the task - queue->complete(task); - } - } else { - // Single Thread Task - - if (0 == team_exec.team_rank()) { - (*task->m_apply)(task, &single_exec); - - queue->complete(task); - } - - // All team members wait for whole team to reach this statement. - // Not necessary to complete the task. - // Is necessary to prevent task_shared from being updated - // before it is read by all threads. - team_exec.team_barrier(); - } - } while (1); - } - // END #pragma omp parallel -} - -void TaskQueueSpecialization:: - iff_single_thread_recursive_execute( - TaskQueue *const queue) { - using execution_space = Kokkos::Experimental::OpenMPTarget; - using queue_type = TaskQueue; - using task_root_type = TaskBase; - using Member = TaskExec; - - if (1 == omp_get_num_threads()) { - task_root_type *const end = (task_root_type *)task_root_type::EndTag; - - Member single_exec; - - task_root_type *task = end; - - do { - task = end; - - // Loop by priority and then type - for (int i = 0; i < queue_type::NumQueue && end == task; ++i) { - for (int j = 0; j < 2 && end == task; ++j) { - task = queue_type::pop_task(&queue->m_ready[i][j]); - } - } - - if (end == task) break; - - (*task->m_apply)(task, &single_exec); - - queue->complete(task); - - } while (1); - } -} - -} // namespace Impl -} // namespace Kokkos - -//---------------------------------------------------------------------------- - -#endif /* #if defined( KOKKOS_ENABLE_OPENMPTARGET ) && defined( \ - KOKKOS_ENABLE_TASKPOLICY ) */ diff --git a/lib/kokkos/core/src/OpenMPTarget/Kokkos_OpenMPTarget_Task.hpp b/lib/kokkos/core/src/OpenMPTarget/Kokkos_OpenMPTarget_Task.hpp deleted file mode 100644 index c9aa7b128f..0000000000 --- a/lib/kokkos/core/src/OpenMPTarget/Kokkos_OpenMPTarget_Task.hpp +++ /dev/null @@ -1,319 +0,0 @@ -//@HEADER -// ************************************************************************ -// -// Kokkos v. 4.0 -// Copyright (2022) National Technology & Engineering -// Solutions of Sandia, LLC (NTESS). -// -// Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. -// -// Part of Kokkos, under the Apache License v2.0 with LLVM Exceptions. -// See https://kokkos.org/LICENSE for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// -//@HEADER - -#ifndef KOKKOS_IMPL_OPENMP_TASK_HPP -#define KOKKOS_IMPL_OPENMP_TASK_HPP - -#if defined(KOKKOS_ENABLE_TASKPOLICY) - -//---------------------------------------------------------------------------- -//---------------------------------------------------------------------------- - -namespace Kokkos { -namespace Impl { - -template <> -class TaskQueueSpecialization { - public: - using execution_space = Kokkos::Experimental::OpenMPTarget; - using queue_type = Kokkos::Impl::TaskQueue; - using task_base_type = Kokkos::Impl::TaskBase; - - // Must specify memory space - using memory_space = Kokkos::HostSpace; - - static void iff_single_thread_recursive_execute(queue_type* const); - - // Must provide task queue execution function - static void execute(queue_type* const); - - // Must provide mechanism to set function pointer in - // execution space from the host process. - template - static void proc_set_apply(task_base_type::function_type* ptr) { - using TaskType = TaskBase; - *ptr = TaskType::apply; - } -}; - -extern template class TaskQueue; - -//---------------------------------------------------------------------------- - -template <> -class TaskExec { - private: - TaskExec(TaskExec&&) = delete; - TaskExec(TaskExec const&) = delete; - TaskExec& operator=(TaskExec&&) = delete; - TaskExec& operator=(TaskExec const&) = delete; - - using PoolExec = Kokkos::Impl::OpenMPTargetExec; - - friend class Kokkos::Impl::TaskQueue; - friend class Kokkos::Impl::TaskQueueSpecialization< - Kokkos::Experimental::OpenMPTarget>; - - PoolExec* const m_self_exec; ///< This thread's thread pool data structure - PoolExec* const m_team_exec; ///< Team thread's thread pool data structure - int64_t m_sync_mask; - int64_t mutable m_sync_value; - int mutable m_sync_step; - int m_group_rank; ///< Which "team" subset of thread pool - int m_team_rank; ///< Which thread within a team - int m_team_size; - - TaskExec(); - TaskExec(PoolExec& arg_exec, int arg_team_size); - - void team_barrier_impl() const; - - public: - KOKKOS_FUNCTION void* team_shared() const { - KOKKOS_IF_ON_HOST( - (return m_team_exec ? m_team_exec->scratch_thread() : nullptr;)) - - KOKKOS_IF_ON_DEVICE((return nullptr;)) - } - - KOKKOS_FUNCTION int team_shared_size() const { - KOKKOS_IF_ON_HOST( - (return m_team_exec ? m_team_exec->scratch_thread_size() : 0;)) - - KOKKOS_IF_ON_DEVICE((return 0;)) - } - - /**\brief Whole team enters this function call - * before any teeam member returns from - * this function call. - */ - KOKKOS_FUNCTION void team_barrier() const { - KOKKOS_IF_ON_HOST((if (1 < m_team_size) { team_barrier_impl(); })) - } - - KOKKOS_INLINE_FUNCTION - int team_rank() const { return m_team_rank; } - - KOKKOS_INLINE_FUNCTION - int team_size() const { return m_team_size; } -}; - -} // namespace Impl -} // namespace Kokkos - -//---------------------------------------------------------------------------- -//---------------------------------------------------------------------------- - -namespace Kokkos { - -template -KOKKOS_INLINE_FUNCTION Impl::TeamThreadRangeBoundariesStruct< - iType, Impl::TaskExec > -TeamThreadRange(Impl::TaskExec& thread, - const iType& count) { - return Impl::TeamThreadRangeBoundariesStruct< - iType, Impl::TaskExec >(thread, - count); -} - -template -KOKKOS_INLINE_FUNCTION Impl::TeamThreadRangeBoundariesStruct< - iType, Impl::TaskExec > -TeamThreadRange(Impl::TaskExec& thread, - const iType& start, const iType& end) { - return Impl::TeamThreadRangeBoundariesStruct< - iType, Impl::TaskExec >(thread, start, - end); -} - -/** \brief Inter-thread parallel_for. Executes lambda(iType i) for each - * i=0..N-1. - * - * The range i=0..N-1 is mapped to all threads of the the calling thread team. - */ -template -KOKKOS_INLINE_FUNCTION void parallel_for( - const Impl::TeamThreadRangeBoundariesStruct< - iType, Impl::TaskExec >& - loop_boundaries, - const Lambda& lambda) { - for (iType i = loop_boundaries.start; i < loop_boundaries.end; - i += loop_boundaries.increment) { - lambda(i); - } -} - -template -KOKKOS_INLINE_FUNCTION void parallel_reduce( - const Impl::TeamThreadRangeBoundariesStruct< - iType, Impl::TaskExec >& - loop_boundaries, - const Lambda& lambda, ValueType& initialized_result) { - int team_rank = - loop_boundaries.thread.team_rank(); // member num within the team - ValueType result = initialized_result; - - for (iType i = loop_boundaries.start; i < loop_boundaries.end; - i += loop_boundaries.increment) { - lambda(i, result); - } - - if (1 < loop_boundaries.thread.team_size()) { - ValueType* shared = (ValueType*)loop_boundaries.thread.team_shared(); - - loop_boundaries.thread.team_barrier(); - shared[team_rank] = result; - - loop_boundaries.thread.team_barrier(); - - // reduce across threads to thread 0 - if (team_rank == 0) { - for (int i = 1; i < loop_boundaries.thread.team_size(); i++) { - shared[0] += shared[i]; - } - } - - loop_boundaries.thread.team_barrier(); - - // broadcast result - initialized_result = shared[0]; - } else { - initialized_result = result; - } -} - -template -KOKKOS_INLINE_FUNCTION void parallel_reduce( - const Impl::TeamThreadRangeBoundariesStruct< - iType, Impl::TaskExec >& - loop_boundaries, - const Lambda& lambda, const JoinType& join, ValueType& initialized_result) { - int team_rank = - loop_boundaries.thread.team_rank(); // member num within the team - ValueType result = initialized_result; - - for (iType i = loop_boundaries.start; i < loop_boundaries.end; - i += loop_boundaries.increment) { - lambda(i, result); - } - - if (1 < loop_boundaries.thread.team_size()) { - ValueType* shared = (ValueType*)loop_boundaries.thread.team_shared(); - - loop_boundaries.thread.team_barrier(); - shared[team_rank] = result; - - loop_boundaries.thread.team_barrier(); - - // reduce across threads to thread 0 - if (team_rank == 0) { - for (int i = 1; i < loop_boundaries.thread.team_size(); i++) { - join(shared[0], shared[i]); - } - } - - loop_boundaries.thread.team_barrier(); - - // broadcast result - initialized_result = shared[0]; - } else { - initialized_result = result; - } -} - -// placeholder for future function -template -KOKKOS_INLINE_FUNCTION void parallel_reduce( - const Impl::ThreadVectorRangeBoundariesStruct< - iType, Impl::TaskExec >& - loop_boundaries, - const Lambda& lambda, ValueType& initialized_result) {} - -// placeholder for future function -template -KOKKOS_INLINE_FUNCTION void parallel_reduce( - const Impl::ThreadVectorRangeBoundariesStruct< - iType, Impl::TaskExec >& - loop_boundaries, - const Lambda& lambda, const JoinType& join, ValueType& initialized_result) { -} - -template -KOKKOS_INLINE_FUNCTION void parallel_scan( - const Impl::TeamThreadRangeBoundariesStruct< - iType, Impl::TaskExec >& - loop_boundaries, - const Lambda& lambda) { - ValueType accum = 0; - ValueType val, local_total; - ValueType* shared = (ValueType*)loop_boundaries.thread.team_shared(); - int team_size = loop_boundaries.thread.team_size(); - int team_rank = - loop_boundaries.thread.team_rank(); // member num within the team - - // Intra-member scan - for (iType i = loop_boundaries.start; i < loop_boundaries.end; - i += loop_boundaries.increment) { - local_total = 0; - lambda(i, local_total, false); - val = accum; - lambda(i, val, true); - accum += local_total; - } - - shared[team_rank] = accum; - loop_boundaries.thread.team_barrier(); - - // Member 0 do scan on accumulated totals - if (team_rank == 0) { - for (iType i = 1; i < team_size; i += 1) { - shared[i] += shared[i - 1]; - } - accum = 0; // Member 0 set accum to 0 in preparation for inter-member scan - } - - loop_boundaries.thread.team_barrier(); - - // Inter-member scan adding in accumulated totals - if (team_rank != 0) { - accum = shared[team_rank - 1]; - } - for (iType i = loop_boundaries.start; i < loop_boundaries.end; - i += loop_boundaries.increment) { - local_total = 0; - lambda(i, local_total, false); - val = accum; - lambda(i, val, true); - accum += local_total; - } -} - -// placeholder for future function -template -KOKKOS_INLINE_FUNCTION void parallel_scan( - const Impl::ThreadVectorRangeBoundariesStruct< - iType, Impl::TaskExec >& - loop_boundaries, - const Lambda& lambda) {} - -} /* namespace Kokkos */ - -//---------------------------------------------------------------------------- -//---------------------------------------------------------------------------- - -#endif /* #if defined( KOKKOS_ENABLE_TASKPOLICY ) */ -#endif /* #ifndef KOKKOS_IMPL_OPENMP_TASK_HPP */ diff --git a/lib/kokkos/core/src/SYCL/Kokkos_SYCL.cpp b/lib/kokkos/core/src/SYCL/Kokkos_SYCL.cpp index 4de6931918..2583a1cdc0 100644 --- a/lib/kokkos/core/src/SYCL/Kokkos_SYCL.cpp +++ b/lib/kokkos/core/src/SYCL/Kokkos_SYCL.cpp @@ -46,7 +46,6 @@ struct Container { } // namespace namespace Kokkos { -namespace Experimental { SYCL::SYCL() : m_space_instance(&Impl::SYCLInternal::singleton(), [](Impl::SYCLInternal*) {}) { @@ -100,6 +99,11 @@ void SYCL::print_configuration(std::ostream& os, bool verbose) const { #else os << "macro KOKKOS_IMPL_SYCL_DEVICE_GLOBAL_SUPPORTED : undefined\n"; #endif +#ifdef KOKKOS_ENABLE_SYCL_RELOCATABLE_DEVICE_CODE + os << "macro KOKKOS_ENABLE_SYCL_RELOCATABLE_DEVICE_CODE : defined\n"; +#else + os << "macro KOKKOS_ENABLE_SYCL_RELOCATABLE_DEVICE_CODE : undefined\n"; +#endif #ifdef SYCL_EXT_ONEAPI_DEVICE_GLOBAL os << "macro SYCL_EXT_ONEAPI_DEVICE_GLOBAL : defined\n"; #else @@ -172,8 +176,7 @@ void SYCL::fence(const std::string& name) const { } void SYCL::impl_static_fence(const std::string& name) { - Kokkos::Tools::Experimental::Impl::profile_fence_event< - Kokkos::Experimental::SYCL>( + Kokkos::Tools::Experimental::Impl::profile_fence_event( name, Kokkos::Tools::Experimental::SpecialSynchronizationCases:: GlobalDeviceSynchronization, @@ -261,8 +264,6 @@ std::ostream& SYCL::impl_sycl_info(std::ostream& os, << device.get_info() << "\nImage Max Buffer Size: " << device.get_info() - << "\nImage Max Array Size: " - << device.get_info() << "\nMax Samplers: " << device.get_info() << "\nMax Parameter Size: " << device.get_info() @@ -317,5 +318,4 @@ int g_sycl_space_factory_initialized = Kokkos::Impl::initialize_space_factory("170_SYCL"); } // namespace Impl -} // namespace Experimental } // namespace Kokkos diff --git a/lib/kokkos/core/src/SYCL/Kokkos_SYCL.hpp b/lib/kokkos/core/src/SYCL/Kokkos_SYCL.hpp index 0f3d1f0994..937dcceab4 100644 --- a/lib/kokkos/core/src/SYCL/Kokkos_SYCL.hpp +++ b/lib/kokkos/core/src/SYCL/Kokkos_SYCL.hpp @@ -39,7 +39,6 @@ static_assert(false, #include namespace Kokkos { -namespace Experimental { namespace Impl { class SYCLInternal; } @@ -91,9 +90,8 @@ class SYCL { /** \brief Wait until all dispatched functors complete. A noop for OpenMP. */ static void impl_static_fence(const std::string& name); - void fence( - const std::string& name = - "Kokkos::Experimental::SYCL::fence: Unnamed Instance Fence") const; + void fence(const std::string& name = + "Kokkos::SYCL::fence: Unnamed Instance Fence") const; /// \brief Print configuration information to the given output stream. void print_configuration(std::ostream& os, bool verbose = false) const; @@ -131,15 +129,13 @@ class SYCL { Kokkos::Impl::HostSharedPtr m_space_instance; }; -} // namespace Experimental - namespace Tools { namespace Experimental { template <> -struct DeviceTypeTraits { +struct DeviceTypeTraits { /// \brief An ID to differentiate (for example) Serial from OpenMP in Tooling static constexpr DeviceType id = DeviceType::SYCL; - static int device_id(const Kokkos::Experimental::SYCL& exec) { + static int device_id(const Kokkos::SYCL& exec) { return exec.impl_internal_space_instance()->m_syclDev; } }; @@ -185,10 +181,11 @@ std::vector partition_space(const SYCL& sycl_space, return instances; } +} // namespace Experimental + namespace Impl { std::vector get_sycl_devices(); } // namespace Impl -} // namespace Experimental } // namespace Kokkos diff --git a/lib/kokkos/core/src/SYCL/Kokkos_SYCL_DeepCopy.hpp b/lib/kokkos/core/src/SYCL/Kokkos_SYCL_DeepCopy.hpp index afc7eebd38..a9e2eca4fb 100644 --- a/lib/kokkos/core/src/SYCL/Kokkos_SYCL_DeepCopy.hpp +++ b/lib/kokkos/core/src/SYCL/Kokkos_SYCL_DeepCopy.hpp @@ -28,37 +28,34 @@ namespace Kokkos { namespace Impl { void DeepCopySYCL(void* dst, const void* src, size_t n); -void DeepCopyAsyncSYCL(const Kokkos::Experimental::SYCL& instance, void* dst, - const void* src, size_t n); +void DeepCopyAsyncSYCL(const Kokkos::SYCL& instance, void* dst, const void* src, + size_t n); void DeepCopyAsyncSYCL(void* dst, const void* src, size_t n); template -struct DeepCopy::value>> { DeepCopy(void* dst, const void* src, size_t n) { DeepCopySYCL(dst, src, n); } - DeepCopy(const Kokkos::Experimental::SYCL& instance, void* dst, - const void* src, size_t n) { + DeepCopy(const Kokkos::SYCL& instance, void* dst, const void* src, size_t n) { DeepCopyAsyncSYCL(instance, dst, src, n); } }; template -struct DeepCopy::value>> { DeepCopy(void* dst, const void* src, size_t n) { DeepCopySYCL(dst, src, n); } - DeepCopy(const Kokkos::Experimental::SYCL& instance, void* dst, - const void* src, size_t n) { + DeepCopy(const Kokkos::SYCL& instance, void* dst, const void* src, size_t n) { DeepCopyAsyncSYCL(instance, dst, src, n); } }; template -struct DeepCopy::value && is_sycl_type_space::value>> { DeepCopy(void* dst, const void* src, size_t n) { DeepCopySYCL(dst, src, n); } - DeepCopy(const Kokkos::Experimental::SYCL& instance, void* dst, - const void* src, size_t n) { + DeepCopy(const Kokkos::SYCL& instance, void* dst, const void* src, size_t n) { DeepCopyAsyncSYCL(instance, dst, src, n); } }; @@ -66,10 +63,9 @@ struct DeepCopy struct DeepCopy< MemSpace1, MemSpace2, ExecutionSpace, - std::enable_if_t< - is_sycl_type_space::value && - is_sycl_type_space::value && - !std::is_same::value>> { + std::enable_if_t::value && + is_sycl_type_space::value && + !std::is_same::value>> { inline DeepCopy(void* dst, const void* src, size_t n) { DeepCopySYCL(dst, src, n); } @@ -93,9 +89,8 @@ struct DeepCopy< template struct DeepCopy< MemSpace, HostSpace, ExecutionSpace, - std::enable_if_t< - is_sycl_type_space::value && - !std::is_same::value>> { + std::enable_if_t::value && + !std::is_same::value>> { inline DeepCopy(void* dst, const void* src, size_t n) { DeepCopySYCL(dst, src, n); } @@ -118,9 +113,8 @@ struct DeepCopy< template struct DeepCopy< HostSpace, MemSpace, ExecutionSpace, - std::enable_if_t< - is_sycl_type_space::value && - !std::is_same::value>> { + std::enable_if_t::value && + !std::is_same::value>> { inline DeepCopy(void* dst, const void* src, size_t n) { DeepCopySYCL(dst, src, n); } diff --git a/lib/kokkos/core/src/SYCL/Kokkos_SYCL_GraphNodeKernel.hpp b/lib/kokkos/core/src/SYCL/Kokkos_SYCL_GraphNodeKernel.hpp index 9c39df9415..54ca645995 100644 --- a/lib/kokkos/core/src/SYCL/Kokkos_SYCL_GraphNodeKernel.hpp +++ b/lib/kokkos/core/src/SYCL/Kokkos_SYCL_GraphNodeKernel.hpp @@ -32,30 +32,29 @@ namespace Impl { template -class GraphNodeKernelImpl - : public PatternImplSpecializationFromTag< - PatternTag, Functor, PolicyType, Args..., - Kokkos::Experimental::SYCL>::type { +class GraphNodeKernelImpl + : public PatternImplSpecializationFromTag::type { public: using Policy = PolicyType; using graph_kernel = GraphNodeKernelImpl; - using base_t = typename PatternImplSpecializationFromTag< - PatternTag, Functor, Policy, Args..., Kokkos::Experimental::SYCL>::type; + using base_t = + typename PatternImplSpecializationFromTag::type; // TODO use the name and executionspace template - GraphNodeKernelImpl(std::string, Kokkos::Experimental::SYCL const&, - Functor arg_functor, PolicyDeduced&& arg_policy, - ArgsDeduced&&... args) - : base_t(std::move(arg_functor), (PolicyDeduced &&) arg_policy, - (ArgsDeduced &&) args...) {} + GraphNodeKernelImpl(std::string, Kokkos::SYCL const&, Functor arg_functor, + PolicyDeduced&& arg_policy, ArgsDeduced&&... args) + : base_t(std::move(arg_functor), (PolicyDeduced&&)arg_policy, + (ArgsDeduced&&)args...) {} template - GraphNodeKernelImpl(Kokkos::Experimental::SYCL const& exec_space, - Functor arg_functor, PolicyDeduced&& arg_policy) + GraphNodeKernelImpl(Kokkos::SYCL const& exec_space, Functor arg_functor, + PolicyDeduced&& arg_policy) : GraphNodeKernelImpl("", exec_space, std::move(arg_functor), - (PolicyDeduced &&) arg_policy) {} + (PolicyDeduced&&)arg_policy) {} void set_sycl_graph_ptr( sycl::ext::oneapi::experimental::command_graph< @@ -102,14 +101,14 @@ template ::type> struct get_graph_node_kernel_type - : type_identity> {}; + : type_identity< + GraphNodeKernelImpl> {}; template struct get_graph_node_kernel_type : type_identity, Kokkos::ParallelReduceTag>> {}; diff --git a/lib/kokkos/core/src/SYCL/Kokkos_SYCL_GraphNode_Impl.hpp b/lib/kokkos/core/src/SYCL/Kokkos_SYCL_GraphNode_Impl.hpp index 6bbe6711a2..828f1cacb4 100644 --- a/lib/kokkos/core/src/SYCL/Kokkos_SYCL_GraphNode_Impl.hpp +++ b/lib/kokkos/core/src/SYCL/Kokkos_SYCL_GraphNode_Impl.hpp @@ -28,7 +28,7 @@ namespace Kokkos { namespace Impl { template <> -struct GraphNodeBackendSpecificDetails { +struct GraphNodeBackendSpecificDetails { std::optional node; explicit GraphNodeBackendSpecificDetails() = default; @@ -38,16 +38,16 @@ struct GraphNodeBackendSpecificDetails { }; template -struct GraphNodeBackendDetailsBeforeTypeErasure { +struct GraphNodeBackendDetailsBeforeTypeErasure { protected: GraphNodeBackendDetailsBeforeTypeErasure( - Kokkos::Experimental::SYCL const &, Kernel &, PredecessorRef const &, - GraphNodeBackendSpecificDetails &) noexcept {} + Kokkos::SYCL const &, Kernel &, PredecessorRef const &, + GraphNodeBackendSpecificDetails &) noexcept {} GraphNodeBackendDetailsBeforeTypeErasure( - Kokkos::Experimental::SYCL const &, _graph_node_is_root_ctor_tag, - GraphNodeBackendSpecificDetails &) noexcept {} + Kokkos::SYCL const &, _graph_node_is_root_ctor_tag, + GraphNodeBackendSpecificDetails &) noexcept {} }; } // namespace Impl diff --git a/lib/kokkos/core/src/SYCL/Kokkos_SYCL_Graph_Impl.hpp b/lib/kokkos/core/src/SYCL/Kokkos_SYCL_Graph_Impl.hpp index 1dc4a9c997..dc63052dd7 100644 --- a/lib/kokkos/core/src/SYCL/Kokkos_SYCL_Graph_Impl.hpp +++ b/lib/kokkos/core/src/SYCL/Kokkos_SYCL_Graph_Impl.hpp @@ -31,29 +31,28 @@ namespace Kokkos { namespace Impl { template <> -class GraphImpl { +class GraphImpl { public: - using node_details_t = - GraphNodeBackendSpecificDetails; - using root_node_impl_t = GraphNodeImpl; + using node_details_t = GraphNodeBackendSpecificDetails; + using root_node_impl_t = + GraphNodeImpl; using aggregate_kernel_impl_t = SYCLGraphNodeAggregateKernel; using aggregate_node_impl_t = - GraphNodeImpl; // Not movable or copyable; it spends its whole life as a shared_ptr in the // Graph object. - GraphImpl() = delete; - GraphImpl(GraphImpl const&) = delete; - GraphImpl(GraphImpl&&) = delete; + GraphImpl() = delete; + GraphImpl(GraphImpl const&) = delete; + GraphImpl(GraphImpl&&) = delete; GraphImpl& operator=(GraphImpl const&) = delete; - GraphImpl& operator=(GraphImpl&&) = delete; + GraphImpl& operator=(GraphImpl&&) = delete; ~GraphImpl(); - explicit GraphImpl(Kokkos::Experimental::SYCL instance); + explicit GraphImpl(Kokkos::SYCL instance); void add_node(std::shared_ptr const& arg_node_ptr); @@ -63,19 +62,25 @@ class GraphImpl { template void add_predecessor(NodeImplPtr arg_node_ptr, PredecessorRef arg_pred_ref); - void submit(); + void submit(const Kokkos::SYCL& exec); - Kokkos::Experimental::SYCL const& get_execution_space() const noexcept; + Kokkos::SYCL const& get_execution_space() const noexcept; auto create_root_node_ptr(); template auto create_aggregate_ptr(PredecessorRefs&&...); - private: - void instantiate_graph() { m_graph_exec = m_graph.finalize(); } + void instantiate() { + KOKKOS_EXPECTS(!m_graph_exec.has_value()); + m_graph_exec = m_graph.finalize(); + } - Kokkos::Experimental::SYCL m_execution_space; + auto& sycl_graph() { return m_graph; } + auto& sycl_graph_exec() { return m_graph_exec; } + + private: + Kokkos::SYCL m_execution_space; sycl::ext::oneapi::experimental::command_graph< sycl::ext::oneapi::experimental::graph_state::modifiable> m_graph; @@ -84,17 +89,16 @@ class GraphImpl { m_graph_exec; }; -inline GraphImpl::~GraphImpl() { +inline GraphImpl::~GraphImpl() { m_execution_space.fence("Kokkos::GraphImpl::~GraphImpl: Graph Destruction"); } -inline GraphImpl::GraphImpl( - Kokkos::Experimental::SYCL instance) +inline GraphImpl::GraphImpl(Kokkos::SYCL instance) : m_execution_space(std::move(instance)), m_graph(m_execution_space.sycl_queue().get_context(), m_execution_space.sycl_queue().get_device()) {} -inline void GraphImpl::add_node( +inline void GraphImpl::add_node( std::shared_ptr const& arg_node_ptr) { // add an empty node that needs to be set up before finalizing the graph arg_node_ptr->node_details_t::node = m_graph.add(); @@ -103,7 +107,7 @@ inline void GraphImpl::add_node( // Requires NodeImplPtr is a shared_ptr to specialization of GraphNodeImpl // Also requires that the kernel has the graph node tag in its policy template -inline void GraphImpl::add_node( +inline void GraphImpl::add_node( std::shared_ptr const& arg_node_ptr) { static_assert(NodeImpl::kernel_type::Policy::is_graph_kernel::value); KOKKOS_EXPECTS(arg_node_ptr); @@ -122,7 +126,7 @@ inline void GraphImpl::add_node( // already been added to this graph and NodeImpl is a specialization of // GraphNodeImpl that has already been added to this graph. template -inline void GraphImpl::add_predecessor( +inline void GraphImpl::add_predecessor( NodeImplPtr arg_node_ptr, PredecessorRef arg_pred_ref) { KOKKOS_EXPECTS(arg_node_ptr); auto pred_ptr = GraphAccess::get_node_ptr(arg_pred_ref); @@ -137,19 +141,19 @@ inline void GraphImpl::add_predecessor( m_graph.make_edge(*pred_node, *node); } -inline void GraphImpl::submit() { +inline void GraphImpl::submit(const Kokkos::SYCL& exec) { if (!m_graph_exec) { - instantiate_graph(); + instantiate(); } - m_execution_space.sycl_queue().ext_oneapi_graph(*m_graph_exec); + exec.sycl_queue().ext_oneapi_graph(*m_graph_exec); } -inline Kokkos::Experimental::SYCL const& -GraphImpl::get_execution_space() const noexcept { +inline Kokkos::SYCL const& GraphImpl::get_execution_space() + const noexcept { return m_execution_space; } -inline auto GraphImpl::create_root_node_ptr() { +inline auto GraphImpl::create_root_node_ptr() { KOKKOS_EXPECTS(!m_graph_exec); auto rv = std::make_shared(get_execution_space(), _graph_node_is_root_ctor_tag{}); @@ -158,7 +162,7 @@ inline auto GraphImpl::create_root_node_ptr() { } template -inline auto GraphImpl::create_aggregate_ptr( +inline auto GraphImpl::create_aggregate_ptr( PredecessorRefs&&...) { // The attachment to predecessors, which is all we really need, happens // in the generic layer, which calls through to add_predecessor for diff --git a/lib/kokkos/core/src/SYCL/Kokkos_SYCL_Instance.cpp b/lib/kokkos/core/src/SYCL/Kokkos_SYCL_Instance.cpp index 5843dca812..5af1330d93 100644 --- a/lib/kokkos/core/src/SYCL/Kokkos_SYCL_Instance.cpp +++ b/lib/kokkos/core/src/SYCL/Kokkos_SYCL_Instance.cpp @@ -24,14 +24,12 @@ #include namespace Kokkos { -namespace Experimental { namespace Impl { namespace { // FIXME_SYCL Should be a multiple of the maximum subgroup size. -static constexpr auto sizeScratchGrain = - sizeof(Kokkos::Experimental::SYCL::size_type[32]); +static constexpr auto sizeScratchGrain = sizeof(Kokkos::SYCL::size_type[32]); std::size_t scratch_count(const std::size_t size) { return (size + sizeScratchGrain - 1) / sizeScratchGrain; @@ -55,8 +53,8 @@ Kokkos::View sycl_global_unique_token_locks( SYCLInternal::~SYCLInternal() { if (!was_finalized || m_scratchSpace || m_scratchHost || m_scratchFlags) { - std::cerr << "Kokkos::Experimental::SYCL ERROR: Failed to call " - "Kokkos::Experimental::SYCL::finalize()" + std::cerr << "Kokkos::SYCL ERROR: Failed to call " + "Kokkos::SYCL::finalize()" << std::endl; std::cerr.flush(); } @@ -64,7 +62,7 @@ SYCLInternal::~SYCLInternal() { int SYCLInternal::verify_is_initialized(const char* const label) const { if (!is_initialized()) { - Kokkos::abort((std::string("Kokkos::Experimental::SYCL::") + label + + Kokkos::abort((std::string("Kokkos::SYCL::") + label + " : ERROR device not initialized\n") .c_str()); } @@ -171,12 +169,12 @@ Kokkos::Impl::sycl_device_ptr SYCLInternal::resize_team_scratch_space( // Multiple ParallelFor/Reduce Teams can call this function at the same time // and invalidate the m_team_scratch_ptr. We use a pool to avoid any race // condition. - auto mem_space = Kokkos::Experimental::SYCLDeviceUSMSpace(*m_queue); + auto mem_space = Kokkos::SYCLDeviceUSMSpace(*m_queue); if (m_team_scratch_current_size[scratch_pool_id] == 0 && bytes > 0) { m_team_scratch_current_size[scratch_pool_id] = bytes; - m_team_scratch_ptr[scratch_pool_id] = mem_space.allocate( - "Kokkos::Experimental::SYCL::InternalTeamScratchMemory", - m_team_scratch_current_size[scratch_pool_id]); + m_team_scratch_ptr[scratch_pool_id] = + mem_space.allocate("Kokkos::SYCL::InternalTeamScratchMemory", + m_team_scratch_current_size[scratch_pool_id]); } if ((bytes > m_team_scratch_current_size[scratch_pool_id]) || ((bytes < m_team_scratch_current_size[scratch_pool_id]) && @@ -184,9 +182,9 @@ Kokkos::Impl::sycl_device_ptr SYCLInternal::resize_team_scratch_space( mem_space.deallocate(m_team_scratch_ptr[scratch_pool_id], m_team_scratch_current_size[scratch_pool_id]); m_team_scratch_current_size[scratch_pool_id] = bytes; - m_team_scratch_ptr[scratch_pool_id] = mem_space.allocate( - "Kokkos::Experimental::SYCL::InternalTeamScratchMemory", - m_team_scratch_current_size[scratch_pool_id]); + m_team_scratch_ptr[scratch_pool_id] = + mem_space.allocate("Kokkos::SYCL::InternalTeamScratchMemory", + m_team_scratch_current_size[scratch_pool_id]); } return m_team_scratch_ptr[scratch_pool_id]; } @@ -255,7 +253,7 @@ Kokkos::Impl::sycl_device_ptr SYCLInternal::scratch_space( const std::size_t size) { if (verify_is_initialized("scratch_space") && m_scratchSpaceCount < scratch_count(size)) { - auto mem_space = Kokkos::Experimental::SYCLDeviceUSMSpace(*m_queue); + auto mem_space = Kokkos::SYCLDeviceUSMSpace(*m_queue); if (nullptr != m_scratchSpace) mem_space.deallocate(m_scratchSpace, @@ -265,8 +263,8 @@ Kokkos::Impl::sycl_device_ptr SYCLInternal::scratch_space( std::size_t alloc_size = Kokkos::Impl::multiply_overflow_abort( m_scratchSpaceCount, sizeScratchGrain); - m_scratchSpace = static_cast(mem_space.allocate( - "Kokkos::Experimental::SYCL::InternalScratchSpace", alloc_size)); + m_scratchSpace = static_cast( + mem_space.allocate("Kokkos::SYCL::InternalScratchSpace", alloc_size)); } return m_scratchSpace; @@ -276,7 +274,7 @@ Kokkos::Impl::sycl_host_ptr SYCLInternal::scratch_host( const std::size_t size) { if (verify_is_initialized("scratch_unified") && m_scratchHostCount < scratch_count(size)) { - auto mem_space = Kokkos::Experimental::SYCLHostUSMSpace(*m_queue); + auto mem_space = Kokkos::SYCLHostUSMSpace(*m_queue); if (nullptr != m_scratchHost) mem_space.deallocate(m_scratchHost, @@ -286,8 +284,8 @@ Kokkos::Impl::sycl_host_ptr SYCLInternal::scratch_host( std::size_t alloc_size = Kokkos::Impl::multiply_overflow_abort( m_scratchHostCount, sizeScratchGrain); - m_scratchHost = static_cast(mem_space.allocate( - "Kokkos::Experimental::SYCL::InternalScratchHost", alloc_size)); + m_scratchHost = static_cast( + mem_space.allocate("Kokkos::SYCL::InternalScratchHost", alloc_size)); } return m_scratchHost; @@ -297,7 +295,7 @@ Kokkos::Impl::sycl_device_ptr SYCLInternal::scratch_flags( const std::size_t size) { if (verify_is_initialized("scratch_flags") && m_scratchFlagsCount < scratch_count(size)) { - auto mem_space = Kokkos::Experimental::SYCLDeviceUSMSpace(*m_queue); + auto mem_space = Kokkos::SYCLDeviceUSMSpace(*m_queue); if (nullptr != m_scratchFlags) mem_space.deallocate(m_scratchFlags, @@ -307,8 +305,8 @@ Kokkos::Impl::sycl_device_ptr SYCLInternal::scratch_flags( std::size_t alloc_size = Kokkos::Impl::multiply_overflow_abort( m_scratchFlagsCount, sizeScratchGrain); - m_scratchFlags = static_cast(mem_space.allocate( - "Kokkos::Experimental::SYCL::InternalScratchFlags", alloc_size)); + m_scratchFlags = static_cast( + mem_space.allocate("Kokkos::SYCL::InternalScratchFlags", alloc_size)); // We only zero-initialize the allocation when we actually allocate. // It's the responsibility of the features using scratch_flags, @@ -326,8 +324,7 @@ Kokkos::Impl::sycl_device_ptr SYCLInternal::scratch_flags( template void SYCLInternal::fence_helper(WAT& wat, const std::string& name, uint32_t instance_id) { - Kokkos::Tools::Experimental::Impl::profile_fence_event< - Kokkos::Experimental::SYCL>( + Kokkos::Tools::Experimental::Impl::profile_fence_event( name, Kokkos::Tools::Experimental::Impl::DirectFenceIDHandle{instance_id}, [&]() { try { @@ -364,8 +361,7 @@ size_t SYCLInternal::USMObjectMem::reserve(size_t n) { AllocationSpace alloc_space(*m_q); if (m_data) alloc_space.deallocate(m_data, m_capacity); - m_data = - alloc_space.allocate("Kokkos::Experimental::SYCL::USMObjectMem", n); + m_data = alloc_space.allocate("Kokkos::SYCL::USMObjectMem", n); if constexpr (sycl::usm::alloc::device == Kind) m_staging.reset(new char[n]); @@ -396,5 +392,4 @@ template class SYCLInternal::USMObjectMem; template class SYCLInternal::USMObjectMem; } // namespace Impl -} // namespace Experimental } // namespace Kokkos diff --git a/lib/kokkos/core/src/SYCL/Kokkos_SYCL_Instance.hpp b/lib/kokkos/core/src/SYCL/Kokkos_SYCL_Instance.hpp index 2d784ef8a5..c982154a9a 100644 --- a/lib/kokkos/core/src/SYCL/Kokkos_SYCL_Instance.hpp +++ b/lib/kokkos/core/src/SYCL/Kokkos_SYCL_Instance.hpp @@ -28,7 +28,6 @@ #include #include namespace Kokkos { -namespace Experimental { namespace Impl { class SYCLInternal { @@ -38,10 +37,10 @@ class SYCLInternal { SYCLInternal() = default; ~SYCLInternal(); - SYCLInternal(const SYCLInternal&) = delete; + SYCLInternal(const SYCLInternal&) = delete; SYCLInternal& operator=(const SYCLInternal&) = delete; - SYCLInternal& operator=(SYCLInternal&&) = delete; - SYCLInternal(SYCLInternal&&) = delete; + SYCLInternal& operator=(SYCLInternal&&) = delete; + SYCLInternal(SYCLInternal&&) = delete; Kokkos::Impl::sycl_device_ptr scratch_space(const std::size_t size); Kokkos::Impl::sycl_device_ptr scratch_flags(const std::size_t size); @@ -76,8 +75,9 @@ class SYCLInternal { mutable sycl::event m_team_scratch_event[m_n_team_scratch] = {}; mutable std::mutex m_team_scratch_mutex; - uint32_t m_instance_id = Kokkos::Tools::Experimental::Impl::idForInstance< - Kokkos::Experimental::SYCL>(reinterpret_cast(this)); + uint32_t m_instance_id = + Kokkos::Tools::Experimental::Impl::idForInstance( + reinterpret_cast(this)); std::optional m_queue; // Using std::vector> reveals a compiler bug when @@ -102,9 +102,9 @@ class SYCLInternal { explicit USMObjectMem(sycl::queue q, uint32_t instance_id) noexcept : m_q(std::move(q)), m_instance_id(instance_id) {} - USMObjectMem(USMObjectMem const&) = delete; - USMObjectMem(USMObjectMem&&) = delete; - USMObjectMem& operator=(USMObjectMem&&) = delete; + USMObjectMem(USMObjectMem const&) = delete; + USMObjectMem(USMObjectMem&&) = delete; + USMObjectMem& operator=(USMObjectMem&&) = delete; USMObjectMem& operator=(USMObjectMem const&) = delete; ~USMObjectMem() { reset(); }; @@ -119,12 +119,12 @@ class SYCLInternal { size_t reserve(size_t n); private: - using AllocationSpace = std::conditional_t< - Kind == sycl::usm::alloc::device, - Kokkos::Experimental::SYCLDeviceUSMSpace, - std::conditional_t>; + using AllocationSpace = + std::conditional_t>; public: // Performs either sycl::memcpy (for USM device memory) or std::memcpy @@ -144,11 +144,10 @@ class SYCLInternal { } void fence() { - SYCLInternal::fence( - m_last_event, - "Kokkos::Experimental::SYCLInternal::USMObject fence to wait for " - "last event to finish", - m_instance_id); + SYCLInternal::fence(m_last_event, + "Kokkos::SYCLInternal::USMObject fence to wait for " + "last event to finish", + m_instance_id); } void register_event(sycl::event event) { @@ -324,13 +323,12 @@ auto make_sycl_function_wrapper(const Functor& functor, Storage& storage) { return SYCLFunctionWrapper(functor, storage); } } // namespace Impl -} // namespace Experimental } // namespace Kokkos #if defined(SYCL_DEVICE_COPYABLE) && defined(KOKKOS_ARCH_INTEL_GPU) template struct sycl::is_device_copyable< - Kokkos::Experimental::Impl::SYCLFunctionWrapper> + Kokkos::Impl::SYCLFunctionWrapper> : std::true_type {}; #if (defined(__INTEL_LLVM_COMPILER) && __INTEL_LLVM_COMPILER < 20240000) || \ @@ -352,8 +350,7 @@ static_assert( template struct sycl::is_device_copyable< - const Kokkos::Experimental::Impl::SYCLFunctionWrapper, + const Kokkos::Impl::SYCLFunctionWrapper, std::enable_if_t>>> : std::true_type {}; diff --git a/lib/kokkos/core/src/SYCL/Kokkos_SYCL_MDRangePolicy.hpp b/lib/kokkos/core/src/SYCL/Kokkos_SYCL_MDRangePolicy.hpp index d212e2dacc..9498513a3e 100644 --- a/lib/kokkos/core/src/SYCL/Kokkos_SYCL_MDRangePolicy.hpp +++ b/lib/kokkos/core/src/SYCL/Kokkos_SYCL_MDRangePolicy.hpp @@ -22,13 +22,13 @@ namespace Kokkos { template <> -struct default_outer_direction { +struct default_outer_direction { using type = Iterate; static constexpr Iterate value = Iterate::Left; }; template <> -struct default_inner_direction { +struct default_inner_direction { using type = Iterate; static constexpr Iterate value = Iterate::Left; }; @@ -37,8 +37,8 @@ namespace Impl { // Settings for MDRangePolicy template <> -inline TileSizeProperties get_tile_size_properties( - const Kokkos::Experimental::SYCL& space) { +inline TileSizeProperties get_tile_size_properties( + const Kokkos::SYCL& space) { TileSizeProperties properties; properties.max_threads = space.impl_internal_space_instance()->m_maxWorkgroupSize; @@ -50,8 +50,7 @@ inline TileSizeProperties get_tile_size_properties( // Settings for TeamMDRangePolicy template -struct ThreadAndVectorNestLevel +struct ThreadAndVectorNestLevel : AcceleratorBasedNestLevel {}; } // namespace Impl diff --git a/lib/kokkos/core/src/SYCL/Kokkos_SYCL_ParallelFor_MDRange.hpp b/lib/kokkos/core/src/SYCL/Kokkos_SYCL_ParallelFor_MDRange.hpp index cb7b1048da..3dbd63d81a 100644 --- a/lib/kokkos/core/src/SYCL/Kokkos_SYCL_ParallelFor_MDRange.hpp +++ b/lib/kokkos/core/src/SYCL/Kokkos_SYCL_ParallelFor_MDRange.hpp @@ -25,7 +25,7 @@ template class Kokkos::Impl::ParallelFor, - Kokkos::Experimental::SYCL> { + Kokkos::SYCL> { public: using Policy = Kokkos::MDRangePolicy; @@ -54,7 +54,7 @@ class Kokkos::Impl::ParallelFor, const typename Policy::index_type m_num_tiles; static constexpr Iterate inner_direction = Policy::inner_direction; } m_policy; - const Kokkos::Experimental::SYCL& m_space; + const Kokkos::SYCL& m_space; sycl::nd_range<3> compute_ranges() const { const auto& m_tile = m_policy.m_tile; @@ -180,12 +180,11 @@ class Kokkos::Impl::ParallelFor, } void execute() const { - Kokkos::Experimental::Impl::SYCLInternal::IndirectKernelMem& - indirectKernelMem = - m_space.impl_internal_space_instance()->get_indirect_kernel_mem(); + Kokkos::Impl::SYCLInternal::IndirectKernelMem& indirectKernelMem = + m_space.impl_internal_space_instance()->get_indirect_kernel_mem(); - auto functor_wrapper = Experimental::Impl::make_sycl_function_wrapper( - m_functor, indirectKernelMem); + auto functor_wrapper = + Impl::make_sycl_function_wrapper(m_functor, indirectKernelMem); sycl::event event = sycl_direct_launch(functor_wrapper, functor_wrapper.get_copy_event()); functor_wrapper.register_event(event); diff --git a/lib/kokkos/core/src/SYCL/Kokkos_SYCL_ParallelFor_Range.hpp b/lib/kokkos/core/src/SYCL/Kokkos_SYCL_ParallelFor_Range.hpp index 8ef43d392c..da75f3e901 100644 --- a/lib/kokkos/core/src/SYCL/Kokkos_SYCL_ParallelFor_Range.hpp +++ b/lib/kokkos/core/src/SYCL/Kokkos_SYCL_ParallelFor_Range.hpp @@ -67,7 +67,7 @@ struct FunctorWrapperRangePolicyParallelForCustom { template class Kokkos::Impl::ParallelFor, - Kokkos::Experimental::SYCL> { + Kokkos::SYCL> { public: using Policy = Kokkos::RangePolicy; @@ -82,8 +82,8 @@ class Kokkos::Impl::ParallelFor, sycl::event sycl_direct_launch(const Policy& policy, const Functor& functor, const sycl::event& memcpy_event) const { // Convenience references - const Kokkos::Experimental::SYCL& space = policy.space(); - sycl::queue& q = space.sycl_queue(); + const Kokkos::SYCL& space = policy.space(); + sycl::queue& q = space.sycl_queue(); desul::ensure_sycl_lock_arrays_on_device(q); @@ -160,13 +160,13 @@ class Kokkos::Impl::ParallelFor, void execute() const { if (m_policy.begin() == m_policy.end()) return; - Kokkos::Experimental::Impl::SYCLInternal::IndirectKernelMem& - indirectKernelMem = m_policy.space() - .impl_internal_space_instance() - ->get_indirect_kernel_mem(); + Kokkos::Impl::SYCLInternal::IndirectKernelMem& indirectKernelMem = + m_policy.space() + .impl_internal_space_instance() + ->get_indirect_kernel_mem(); - auto functor_wrapper = Experimental::Impl::make_sycl_function_wrapper( - m_functor, indirectKernelMem); + auto functor_wrapper = + Impl::make_sycl_function_wrapper(m_functor, indirectKernelMem); sycl::event event = sycl_direct_launch(m_policy, functor_wrapper, functor_wrapper.get_copy_event()); functor_wrapper.register_event(event); diff --git a/lib/kokkos/core/src/SYCL/Kokkos_SYCL_ParallelFor_Team.hpp b/lib/kokkos/core/src/SYCL/Kokkos_SYCL_ParallelFor_Team.hpp index cf7f582bc7..d8859cda9f 100644 --- a/lib/kokkos/core/src/SYCL/Kokkos_SYCL_ParallelFor_Team.hpp +++ b/lib/kokkos/core/src/SYCL/Kokkos_SYCL_ParallelFor_Team.hpp @@ -27,11 +27,11 @@ template class Kokkos::Impl::ParallelFor, - Kokkos::Experimental::SYCL> { + Kokkos::SYCL> { public: using Policy = TeamPolicy; using functor_type = FunctorType; - using size_type = ::Kokkos::Experimental::SYCL::size_type; + using size_type = ::Kokkos::SYCL::size_type; private: using member_type = typename Policy::member_type; @@ -52,8 +52,8 @@ class Kokkos::Impl::ParallelFor, const FunctorWrapper& functor_wrapper, const sycl::event& memcpy_event) const { // Convenience references - const Kokkos::Experimental::SYCL& space = m_policy.space(); - sycl::queue& q = space.sycl_queue(); + const Kokkos::SYCL& space = m_policy.space(); + sycl::queue& q = space.sycl_queue(); desul::ensure_sycl_lock_arrays_on_device(q); @@ -146,11 +146,11 @@ class Kokkos::Impl::ParallelFor, scratch_pool_id, static_cast(m_scratch_size[1]) * m_league_size)); - Kokkos::Experimental::Impl::SYCLInternal::IndirectKernelMem& - indirectKernelMem = instance.get_indirect_kernel_mem(); + Kokkos::Impl::SYCLInternal::IndirectKernelMem& indirectKernelMem = + instance.get_indirect_kernel_mem(); - auto functor_wrapper = Experimental::Impl::make_sycl_function_wrapper( - m_functor, indirectKernelMem); + auto functor_wrapper = + Impl::make_sycl_function_wrapper(m_functor, indirectKernelMem); sycl::event event = sycl_direct_launch(global_scratch_ptr, functor_wrapper, functor_wrapper.get_copy_event()); @@ -164,10 +164,14 @@ class Kokkos::Impl::ParallelFor, m_league_size(arg_policy.league_size()), m_team_size(arg_policy.team_size()), m_vector_size(arg_policy.impl_vector_length()) { - // FIXME_SYCL optimize - if (m_team_size < 0) + if (m_team_size < 0) { m_team_size = m_policy.team_size_recommended(arg_functor, ParallelForTag{}); + if (m_team_size <= 0) + Kokkos::Impl::throw_runtime_exception( + "Kokkos::Impl::ParallelFor could not find a " + "valid execution configuration."); + } m_shmem_begin = (sizeof(double) * (m_team_size + 2)); m_shmem_size = diff --git a/lib/kokkos/core/src/SYCL/Kokkos_SYCL_ParallelReduce_MDRange.hpp b/lib/kokkos/core/src/SYCL/Kokkos_SYCL_ParallelReduce_MDRange.hpp index 0774b24bca..1e31354975 100644 --- a/lib/kokkos/core/src/SYCL/Kokkos_SYCL_ParallelReduce_MDRange.hpp +++ b/lib/kokkos/core/src/SYCL/Kokkos_SYCL_ParallelReduce_MDRange.hpp @@ -30,7 +30,7 @@ template class Kokkos::Impl::ParallelReduce, - Kokkos::Experimental::SYCL> { + Kokkos::SYCL> { public: using Policy = Kokkos::MDRangePolicy; using FunctorType = typename CombinedFunctorReducerType::functor_type; @@ -76,7 +76,7 @@ class Kokkos::Impl::ParallelReduce::accessible) {} private: @@ -85,7 +85,7 @@ class Kokkos::Impl::ParallelReduce scratch_buffers_lock( instance.m_mutexScratchSpace); - using IndirectKernelMem = - Kokkos::Experimental::Impl::SYCLInternal::IndirectKernelMem; + using IndirectKernelMem = Kokkos::Impl::SYCLInternal::IndirectKernelMem; IndirectKernelMem& indirectKernelMem = instance.get_indirect_kernel_mem(); auto functor_reducer_wrapper = - Experimental::Impl::make_sycl_function_wrapper(m_functor_reducer, - indirectKernelMem); + Impl::make_sycl_function_wrapper(m_functor_reducer, indirectKernelMem); sycl::event event = sycl_direct_launch( functor_reducer_wrapper, functor_reducer_wrapper.get_copy_event()); @@ -370,7 +368,7 @@ class Kokkos::Impl::ParallelReduce template -class Kokkos::Impl::ParallelReduce, - Kokkos::Experimental::SYCL> { +class Kokkos::Impl::ParallelReduce< + CombinedFunctorReducerType, Kokkos::RangePolicy, Kokkos::SYCL> { public: using Policy = Kokkos::RangePolicy; using FunctorType = typename CombinedFunctorReducerType::functor_type; @@ -49,7 +48,7 @@ class Kokkos::Impl::ParallelReduce::accessible) {} private: @@ -59,8 +58,8 @@ class Kokkos::Impl::ParallelReduce scratch_buffers_lock( instance.m_mutexScratchSpace); - using IndirectKernelMem = - Kokkos::Experimental::Impl::SYCLInternal::IndirectKernelMem; + using IndirectKernelMem = Kokkos::Impl::SYCLInternal::IndirectKernelMem; IndirectKernelMem& indirectKernelMem = instance.get_indirect_kernel_mem(); auto functor_reducer_wrapper = - Experimental::Impl::make_sycl_function_wrapper(m_functor_reducer, - indirectKernelMem); + Impl::make_sycl_function_wrapper(m_functor_reducer, indirectKernelMem); sycl::event event = sycl_direct_launch(m_policy, functor_reducer_wrapper, diff --git a/lib/kokkos/core/src/SYCL/Kokkos_SYCL_ParallelReduce_Team.hpp b/lib/kokkos/core/src/SYCL/Kokkos_SYCL_ParallelReduce_Team.hpp index b443bcbf90..8f5310cbb2 100644 --- a/lib/kokkos/core/src/SYCL/Kokkos_SYCL_ParallelReduce_Team.hpp +++ b/lib/kokkos/core/src/SYCL/Kokkos_SYCL_ParallelReduce_Team.hpp @@ -29,7 +29,7 @@ template class Kokkos::Impl::ParallelReduce, - Kokkos::Experimental::SYCL> { + Kokkos::SYCL> { public: using Policy = TeamPolicy; using FunctorType = typename CombinedFunctorReducerType::functor_type; @@ -46,7 +46,7 @@ class Kokkos::Impl::ParallelReduce(m_scratch_size[1]) * m_league_size)); - using IndirectKernelMem = - Kokkos::Experimental::Impl::SYCLInternal::IndirectKernelMem; + using IndirectKernelMem = Kokkos::Impl::SYCLInternal::IndirectKernelMem; IndirectKernelMem& indirectKernelMem = instance.get_indirect_kernel_mem(); auto functor_reducer_wrapper = - Experimental::Impl::make_sycl_function_wrapper(m_functor_reducer, - indirectKernelMem); + Impl::make_sycl_function_wrapper(m_functor_reducer, indirectKernelMem); sycl::event event = sycl_direct_launch(global_scratch_ptr, functor_reducer_wrapper, @@ -436,16 +434,21 @@ class Kokkos::Impl::ParallelReduce::accessible), m_league_size(arg_policy.league_size()), m_team_size(arg_policy.team_size()), m_vector_size(arg_policy.impl_vector_length()) { - // FIXME_SYCL optimize - if (m_team_size < 0) + if (m_team_size < 0) { m_team_size = m_policy.team_size_recommended( m_functor_reducer.get_functor(), m_functor_reducer.get_reducer(), ParallelReduceTag{}); + if (m_team_size <= 0) + Kokkos::Impl::throw_runtime_exception( + "Kokkos::Impl::ParallelReduce could not find a " + "valid execution configuration."); + } + // Must be a power of two greater than two, get the one not bigger than the // requested one. if ((m_team_size & m_team_size - 1) || m_team_size < 2) { @@ -461,7 +464,7 @@ class Kokkos::Impl::ParallelReduce(instance.m_maxShmemPerBlock) < m_shmem_size - m_shmem_begin) { diff --git a/lib/kokkos/core/src/SYCL/Kokkos_SYCL_ParallelScan_Range.hpp b/lib/kokkos/core/src/SYCL/Kokkos_SYCL_ParallelScan_Range.hpp index bdb5b88377..ed7cee2805 100644 --- a/lib/kokkos/core/src/SYCL/Kokkos_SYCL_ParallelScan_Range.hpp +++ b/lib/kokkos/core/src/SYCL/Kokkos_SYCL_ParallelScan_Range.hpp @@ -145,7 +145,7 @@ class ParallelScanSYCLBase { using value_type = typename Analysis::value_type; using reference_type = typename Analysis::reference_type; using functor_type = FunctorType; - using size_type = Kokkos::Experimental::SYCL::size_type; + using size_type = Kokkos::SYCL::size_type; using index_type = typename Policy::index_type; protected: @@ -161,8 +161,8 @@ class ParallelScanSYCLBase { sycl::event sycl_direct_launch(const FunctorWrapper& functor_wrapper, sycl::event memcpy_event) { // Convenience references - const Kokkos::Experimental::SYCL& space = m_policy.space(); - Kokkos::Experimental::Impl::SYCLInternal& instance = + const Kokkos::SYCL& space = m_policy.space(); + Kokkos::Impl::SYCLInternal& instance = *space.impl_internal_space_instance(); sycl::queue& q = space.sycl_queue(); @@ -374,11 +374,11 @@ class ParallelScanSYCLBase { std::scoped_lock scratch_buffers_lock( instance.m_mutexScratchSpace); - Kokkos::Experimental::Impl::SYCLInternal::IndirectKernelMem& - indirectKernelMem = instance.get_indirect_kernel_mem(); + Kokkos::Impl::SYCLInternal::IndirectKernelMem& indirectKernelMem = + instance.get_indirect_kernel_mem(); - auto functor_wrapper = Experimental::Impl::make_sycl_function_wrapper( - m_functor_reducer, indirectKernelMem); + auto functor_wrapper = + Impl::make_sycl_function_wrapper(m_functor_reducer, indirectKernelMem); sycl::event event = sycl_direct_launch(functor_wrapper, functor_wrapper.get_copy_event()); @@ -399,7 +399,7 @@ class ParallelScanSYCLBase { template class Kokkos::Impl::ParallelScan, - Kokkos::Experimental::SYCL> + Kokkos::SYCL> : private ParallelScanSYCLBase { public: using Base = ParallelScanSYCLBase; @@ -417,13 +417,12 @@ class Kokkos::Impl::ParallelScan, template class Kokkos::Impl::ParallelScanWithTotal< - FunctorType, Kokkos::RangePolicy, ReturnType, - Kokkos::Experimental::SYCL> + FunctorType, Kokkos::RangePolicy, ReturnType, Kokkos::SYCL> : public ParallelScanSYCLBase { public: using Base = ParallelScanSYCLBase; - const Kokkos::Experimental::SYCL& m_exec; + const Kokkos::SYCL& m_exec; inline void execute() { Base::impl_execute([&]() { @@ -445,7 +444,7 @@ class Kokkos::Impl::ParallelScanWithTotal< const typename Base::Policy& arg_policy, const ViewType& arg_result_view) : Base(arg_functor, arg_policy, arg_result_view.data(), - MemorySpaceAccess::accessible), m_exec(arg_policy.space()) {} }; diff --git a/lib/kokkos/core/src/SYCL/Kokkos_SYCL_Space.cpp b/lib/kokkos/core/src/SYCL/Kokkos_SYCL_Space.cpp index 19fad29150..022f88e0a8 100644 --- a/lib/kokkos/core/src/SYCL/Kokkos_SYCL_Space.cpp +++ b/lib/kokkos/core/src/SYCL/Kokkos_SYCL_Space.cpp @@ -33,11 +33,11 @@ namespace Kokkos { namespace Impl { void DeepCopySYCL(void* dst, const void* src, size_t n) { - Experimental::Impl::SYCLInternal::singleton().m_queue->memcpy(dst, src, n); + Impl::SYCLInternal::singleton().m_queue->memcpy(dst, src, n); } -void DeepCopyAsyncSYCL(const Kokkos::Experimental::SYCL& instance, void* dst, - const void* src, size_t n) { +void DeepCopyAsyncSYCL(const Kokkos::SYCL& instance, void* dst, const void* src, + size_t n) { sycl::queue& q = *instance.impl_internal_space_instance()->m_queue; auto event = q.memcpy(dst, src, n); #ifndef KOKKOS_IMPL_SYCL_USE_IN_ORDER_QUEUES @@ -46,9 +46,8 @@ void DeepCopyAsyncSYCL(const Kokkos::Experimental::SYCL& instance, void* dst, } void DeepCopyAsyncSYCL(void* dst, const void* src, size_t n) { - Experimental::Impl::SYCLInternal::singleton().m_queue->memcpy(dst, src, n); - Experimental::SYCL().fence( - "Kokkos::Impl::DeepCopyAsyncSYCL: fence after memcpy"); + Impl::SYCLInternal::singleton().m_queue->memcpy(dst, src, n); + SYCL().fence("Kokkos::Impl::DeepCopyAsyncSYCL: fence after memcpy"); } } // namespace Impl @@ -60,12 +59,9 @@ namespace { std::string_view get_memory_space_name(sycl::usm::alloc allocation_kind) { switch (allocation_kind) { - case sycl::usm::alloc::host: - return Kokkos::Experimental::SYCLHostUSMSpace::name(); - case sycl::usm::alloc::device: - return Kokkos::Experimental::SYCLDeviceUSMSpace::name(); - case sycl::usm::alloc::shared: - return Kokkos::Experimental::SYCLSharedUSMSpace::name(); + case sycl::usm::alloc::host: return Kokkos::SYCLHostUSMSpace::name(); + case sycl::usm::alloc::device: return Kokkos::SYCLDeviceUSMSpace::name(); + case sycl::usm::alloc::shared: return Kokkos::SYCLSharedUSMSpace::name(); default: Kokkos::abort("bug: unknown sycl allocation type"); return "unreachable"; @@ -75,7 +71,6 @@ std::string_view get_memory_space_name(sycl::usm::alloc allocation_kind) { } // namespace namespace Kokkos { -namespace Experimental { SYCLDeviceUSMSpace::SYCLDeviceUSMSpace() : m_queue(*SYCL().impl_internal_space_instance()->m_queue) {} @@ -114,12 +109,12 @@ void* allocate_sycl(const char* arg_label, const size_t arg_alloc_size, return hostPtr; } -void* SYCLDeviceUSMSpace::allocate(const Kokkos::Experimental::SYCL& exec_space, +void* SYCLDeviceUSMSpace::allocate(const Kokkos::SYCL& exec_space, const size_t arg_alloc_size) const { return allocate(exec_space, "[unlabeled]", arg_alloc_size); } -void* SYCLDeviceUSMSpace::allocate(const Kokkos::Experimental::SYCL& exec_space, +void* SYCLDeviceUSMSpace::allocate(const Kokkos::SYCL& exec_space, const char* arg_label, const size_t arg_alloc_size, const size_t arg_logical_size) const { @@ -244,7 +239,6 @@ void SYCLHostUSMSpace::deallocate(const char* arg_label, Kokkos::Tools::make_space_handle(name()), m_queue); } -} // namespace Experimental } // namespace Kokkos //============================================================================== @@ -253,11 +247,11 @@ void SYCLHostUSMSpace::deallocate(const char* arg_label, #include KOKKOS_IMPL_HOST_INACCESSIBLE_SHARED_ALLOCATION_RECORD_EXPLICIT_INSTANTIATION( - Kokkos::Experimental::SYCLDeviceUSMSpace); + Kokkos::SYCLDeviceUSMSpace); KOKKOS_IMPL_SHARED_ALLOCATION_RECORD_EXPLICIT_INSTANTIATION( - Kokkos::Experimental::SYCLSharedUSMSpace); + Kokkos::SYCLSharedUSMSpace); KOKKOS_IMPL_SHARED_ALLOCATION_RECORD_EXPLICIT_INSTANTIATION( - Kokkos::Experimental::SYCLHostUSMSpace); + Kokkos::SYCLHostUSMSpace); // end Explicit instantiations of CRTP Base classes }}}1 //============================================================================== diff --git a/lib/kokkos/core/src/SYCL/Kokkos_SYCL_Space.hpp b/lib/kokkos/core/src/SYCL/Kokkos_SYCL_Space.hpp index b86cfca413..5a37da130c 100644 --- a/lib/kokkos/core/src/SYCL/Kokkos_SYCL_Space.hpp +++ b/lib/kokkos/core/src/SYCL/Kokkos_SYCL_Space.hpp @@ -39,8 +39,6 @@ template struct is_sycl_type_space : public std::false_type {}; } // namespace Impl -namespace Experimental { - class SYCLDeviceUSMSpace { public: using execution_space = SYCL; @@ -154,45 +152,40 @@ class SYCLHostUSMSpace { sycl::queue m_queue; }; -} // namespace Experimental - namespace Impl { template <> -struct is_sycl_type_space - : public std::true_type {}; +struct is_sycl_type_space : public std::true_type { +}; template <> -struct is_sycl_type_space - : public std::true_type {}; +struct is_sycl_type_space : public std::true_type { +}; template <> -struct is_sycl_type_space - : public std::true_type {}; +struct is_sycl_type_space : public std::true_type {}; -static_assert(Kokkos::Impl::MemorySpaceAccess< - Kokkos::Experimental::SYCLDeviceUSMSpace, - Kokkos::Experimental::SYCLDeviceUSMSpace>::assignable); +static_assert( + Kokkos::Impl::MemorySpaceAccess::assignable); -static_assert(Kokkos::Impl::MemorySpaceAccess< - Kokkos::Experimental::SYCLSharedUSMSpace, - Kokkos::Experimental::SYCLSharedUSMSpace>::assignable); +static_assert( + Kokkos::Impl::MemorySpaceAccess::assignable); -static_assert(Kokkos::Impl::MemorySpaceAccess< - Kokkos::Experimental::SYCLDeviceUSMSpace, - Kokkos::Experimental::SYCLDeviceUSMSpace>::assignable); +static_assert( + Kokkos::Impl::MemorySpaceAccess::assignable); template <> -struct MemorySpaceAccess { +struct MemorySpaceAccess { enum : bool { assignable = false }; enum : bool { accessible = false }; enum : bool { deepcopy = true }; }; template <> -struct MemorySpaceAccess { +struct MemorySpaceAccess { // HostSpace::execution_space != SYCLSharedUSMSpace::execution_space enum : bool { assignable = false }; enum : bool { accessible = true }; @@ -200,26 +193,24 @@ struct MemorySpaceAccess -struct MemorySpaceAccess { +struct MemorySpaceAccess { // HostSpace::execution_space == - // Experimental::SYCLHostUSMSpace::execution_space + // SYCLHostUSMSpace::execution_space enum : bool { assignable = true }; enum : bool { accessible = true }; enum : bool { deepcopy = true }; }; template <> -struct MemorySpaceAccess { +struct MemorySpaceAccess { enum : bool { assignable = false }; enum : bool { accessible = false }; enum : bool { deepcopy = true }; }; template <> -struct MemorySpaceAccess { +struct MemorySpaceAccess { // SYCLDeviceUSMSpace::execution_space == SYCLSharedUSMSpace::execution_space enum : bool { assignable = true }; enum : bool { accessible = true }; @@ -227,14 +218,11 @@ struct MemorySpaceAccess -struct MemorySpaceAccess { - // Experimental::SYCLDeviceUSMSpace::execution_space != - // Experimental::SYCLHostUSMSpace::execution_space +struct MemorySpaceAccess { + // SYCLDeviceUSMSpace::execution_space != + // SYCLHostUSMSpace::execution_space enum : bool { assignable = false }; - enum : bool { - accessible = true - }; // Experimental::SYCLDeviceUSMSpace::execution_space + enum : bool { accessible = true }; // SYCLDeviceUSMSpace::execution_space enum : bool { deepcopy = true }; }; @@ -243,16 +231,15 @@ struct MemorySpaceAccess -struct MemorySpaceAccess { +struct MemorySpaceAccess { enum : bool { assignable = false }; enum : bool { accessible = false }; // SYCL cannot access HostSpace enum : bool { deepcopy = true }; }; template <> -struct MemorySpaceAccess { +struct MemorySpaceAccess { // SYCLSharedUSMSpace::execution_space == SYCLDeviceUSMSpace::execution_space // Can access SYCLSharedUSMSpace from Host but cannot access // SYCLDeviceUSMSpace from Host @@ -264,47 +251,38 @@ struct MemorySpaceAccess -struct MemorySpaceAccess { - // Experimental::SYCLSharedUSMSpace::execution_space != - // Experimental::SYCLHostUSMSpace::execution_space +struct MemorySpaceAccess { + // SYCLSharedUSMSpace::execution_space != + // SYCLHostUSMSpace::execution_space enum : bool { assignable = false }; - enum : bool { - accessible = true - }; // Experimental::SYCLSharedUSMSpace::execution_space + enum : bool { accessible = true }; // SYCLSharedUSMSpace::execution_space enum : bool { deepcopy = true }; }; template <> -struct MemorySpaceAccess { +struct MemorySpaceAccess { enum : bool { assignable = false }; // Cannot access from SYCL - enum : bool { - accessible = true - }; // Experimental::SYCLHostUSMSpace::execution_space + enum : bool { accessible = true }; // SYCLHostUSMSpace::execution_space enum : bool { deepcopy = true }; }; template <> -struct MemorySpaceAccess { +struct MemorySpaceAccess { enum : bool { assignable = false }; // Cannot access from Host enum : bool { accessible = false }; enum : bool { deepcopy = true }; }; template <> -struct MemorySpaceAccess { +struct MemorySpaceAccess { enum : bool { assignable = false }; // different execution_space enum : bool { accessible = true }; // same accessibility enum : bool { deepcopy = true }; }; template <> -struct MemorySpaceAccess< - Kokkos::Experimental::SYCLDeviceUSMSpace, - Kokkos::ScratchMemorySpace> { +struct MemorySpaceAccess> { enum : bool { assignable = false }; enum : bool { accessible = true }; enum : bool { deepcopy = false }; @@ -315,11 +293,9 @@ struct MemorySpaceAccess< } // namespace Kokkos KOKKOS_IMPL_HOST_INACCESSIBLE_SHARED_ALLOCATION_SPECIALIZATION( - Kokkos::Experimental::SYCLDeviceUSMSpace); -KOKKOS_IMPL_SHARED_ALLOCATION_SPECIALIZATION( - Kokkos::Experimental::SYCLSharedUSMSpace); -KOKKOS_IMPL_SHARED_ALLOCATION_SPECIALIZATION( - Kokkos::Experimental::SYCLHostUSMSpace); + Kokkos::SYCLDeviceUSMSpace); +KOKKOS_IMPL_SHARED_ALLOCATION_SPECIALIZATION(Kokkos::SYCLSharedUSMSpace); +KOKKOS_IMPL_SHARED_ALLOCATION_SPECIALIZATION(Kokkos::SYCLHostUSMSpace); #endif #endif diff --git a/lib/kokkos/core/src/SYCL/Kokkos_SYCL_Team.hpp b/lib/kokkos/core/src/SYCL/Kokkos_SYCL_Team.hpp index 1e42faa5a8..6359e4a2d9 100644 --- a/lib/kokkos/core/src/SYCL/Kokkos_SYCL_Team.hpp +++ b/lib/kokkos/core/src/SYCL/Kokkos_SYCL_Team.hpp @@ -34,7 +34,7 @@ namespace Impl { */ class SYCLTeamMember { public: - using execution_space = Kokkos::Experimental::SYCL; + using execution_space = Kokkos::SYCL; using scratch_memory_space = execution_space::scratch_memory_space; using team_handle = SYCLTeamMember; @@ -126,6 +126,20 @@ class SYCLTeamMember { team_reduce(ReducerType const& reducer, typename ReducerType::value_type& value) const noexcept { using value_type = typename ReducerType::value_type; + using wrapped_reducer_type = + typename Impl::FunctorAnalysis, ReducerType, + value_type>::Reducer; + impl_team_reduce(wrapped_reducer_type(reducer), value); + reducer.reference() = value; + } + + template + KOKKOS_INLINE_FUNCTION std::enable_if_t::value> + impl_team_reduce( + WrappedReducerType const& wrapped_reducer, + typename WrappedReducerType::value_type& value) const noexcept { + using value_type = typename WrappedReducerType::value_type; auto sg = m_item.get_sub_group(); const auto sub_group_range = sg.get_local_range()[0]; @@ -139,7 +153,7 @@ class SYCLTeamMember { if (vector_range * shift < sub_group_range) { const value_type tmp = Kokkos::Impl::SYCLReduction::shift_group_left( sg, value, vector_range * shift); - if (team_rank_ + shift < team_size_) reducer.join(value, tmp); + if (team_rank_ + shift < team_size_) wrapped_reducer.join(&value, &tmp); } }; shuffle_combine(1); @@ -153,14 +167,13 @@ class SYCLTeamMember { shift <<= 1) { auto tmp = Kokkos::Impl::SYCLReduction::shift_group_left( sg, value, vector_range * shift); - if (team_rank_ + shift < team_size_) reducer.join(value, tmp); + if (team_rank_ + shift < team_size_) wrapped_reducer.join(&value, &tmp); } #endif value = Kokkos::Impl::SYCLReduction::select_from_group(sg, value, 0); const int n_subgroups = sg.get_group_range()[0]; if (n_subgroups == 1) { - reducer.reference() = value; return; } @@ -187,16 +200,15 @@ class SYCLTeamMember { for (int start = step_width; start < n_subgroups; start += step_width) { if (id_in_sg == 0 && group_id >= start && group_id < std::min(start + step_width, n_subgroups)) - reducer.join(reduction_array[group_id - start], value); + wrapped_reducer.join(&reduction_array[group_id - start], &value); sycl::group_barrier(m_item.get_group()); } // Do the final reduction for all threads redundantly value = reduction_array[0]; for (int i = 1; i < std::min(step_width, n_subgroups); ++i) - reducer.join(value, reduction_array[i]); + wrapped_reducer.join(&value, &reduction_array[i]); - reducer.reference() = value; // Make sure that every thread is done using the reduction array. sycl::group_barrier(m_item.get_group()); } @@ -271,8 +283,8 @@ class SYCLTeamMember { const auto update = Kokkos::Impl::SYCLReduction::shift_group_right(sg, value, vector_range); - Type intermediate = (group_id > 0 ? base_data[group_id - 1] : 0) + - (id_in_sg >= vector_range ? update : 0); + Type intermediate = (group_id > 0 ? base_data[group_id - 1] : Type{0}) + + (id_in_sg >= vector_range ? update : Type{0}); if (global_accum) { if (id_in_sg == sub_group_range - 1 && @@ -311,6 +323,19 @@ class SYCLTeamMember { KOKKOS_INLINE_FUNCTION std::enable_if_t::value> vector_reduce(ReducerType const& reducer, typename ReducerType::value_type& value) const { + using value_type = typename ReducerType::value_type; + using wrapped_reducer_type = + typename Impl::FunctorAnalysis, ReducerType, + value_type>::Reducer; + impl_vector_reduce(wrapped_reducer_type(reducer), value); + reducer.reference() = value; + } + + template + KOKKOS_INLINE_FUNCTION std::enable_if_t::value> + impl_vector_reduce(WrappedReducerType const& wrapped_reducer, + typename WrappedReducerType::value_type& value) const { const auto tidx1 = m_item.get_local_id(1); const auto grange1 = m_item.get_local_range(1); @@ -319,13 +344,13 @@ class SYCLTeamMember { if (grange1 == 1) return; // Intra vector lane shuffle reduction: - typename ReducerType::value_type tmp(value); - typename ReducerType::value_type tmp2 = tmp; + typename WrappedReducerType::value_type tmp(value); + typename WrappedReducerType::value_type tmp2 = tmp; for (int i = grange1; (i >>= 1);) { tmp2 = Kokkos::Impl::SYCLReduction::shift_group_left(sg, tmp, i); if (static_cast(tidx1) < i) { - reducer.join(tmp, tmp2); + wrapped_reducer.join(&tmp, &tmp2); } } @@ -336,8 +361,7 @@ class SYCLTeamMember { tmp2 = Kokkos::Impl::SYCLReduction::select_from_group( sg, tmp, (sg.get_local_id() / grange1) * grange1); - value = tmp2; - reducer.reference() = tmp2; + value = tmp2; } //---------------------------------------- @@ -531,8 +555,16 @@ KOKKOS_INLINE_FUNCTION std::enable_if_t::value> parallel_reduce(const Impl::TeamThreadRangeBoundariesStruct< iType, Impl::SYCLTeamMember>& loop_boundaries, const Closure& closure, const ReducerType& reducer) { - typename ReducerType::value_type value; - reducer.init(value); + using value_type = typename ReducerType::value_type; + using functor_analysis_type = typename Impl::FunctorAnalysis< + Impl::FunctorPatternInterface::REDUCE, + TeamPolicy, ReducerType, + value_type>; + using wrapped_reducer_type = typename functor_analysis_type::Reducer; + + wrapped_reducer_type wrapped_reducer(reducer); + value_type value; + wrapped_reducer.init(&value); for (iType i = loop_boundaries.start + loop_boundaries.member.item().get_local_id(0); @@ -541,7 +573,9 @@ parallel_reduce(const Impl::TeamThreadRangeBoundariesStruct< closure(i, value); } - loop_boundaries.member.team_reduce(reducer, value); + loop_boundaries.member.impl_team_reduce(wrapped_reducer, value); + wrapped_reducer.final(&value); + reducer.reference() = value; } /** \brief Inter-thread parallel_reduce assuming summation. @@ -557,20 +591,28 @@ KOKKOS_INLINE_FUNCTION std::enable_if_t::value> parallel_reduce(const Impl::TeamThreadRangeBoundariesStruct< iType, Impl::SYCLTeamMember>& loop_boundaries, const Closure& closure, ValueType& result) { - ValueType val; - Kokkos::Sum reducer(val); + using functor_analysis_type = typename Impl::FunctorAnalysis< + Impl::FunctorPatternInterface::REDUCE, + TeamPolicy, Closure, + ValueType>; + using wrapped_reducer_type = typename functor_analysis_type::Reducer; + using value_type = typename wrapped_reducer_type::value_type; - reducer.init(reducer.reference()); + wrapped_reducer_type wrapped_reducer(closure); + value_type value; + wrapped_reducer.init(&value); for (iType i = loop_boundaries.start + loop_boundaries.member.item().get_local_id(0); i < loop_boundaries.end; i += loop_boundaries.member.item().get_local_range(0)) { - closure(i, val); + closure(i, value); } - loop_boundaries.member.team_reduce(reducer, val); - result = reducer.reference(); + loop_boundaries.member.impl_team_reduce(wrapped_reducer, value); + + wrapped_reducer.final(&value); + result = value; } /** \brief Inter-thread parallel exclusive prefix sum. @@ -657,8 +699,16 @@ KOKKOS_INLINE_FUNCTION std::enable_if_t::value> parallel_reduce(const Impl::TeamVectorRangeBoundariesStruct< iType, Impl::SYCLTeamMember>& loop_boundaries, const Closure& closure, const ReducerType& reducer) { - typename ReducerType::value_type value; - reducer.init(value); + using value_type = typename ReducerType::value_type; + using functor_analysis_type = typename Impl::FunctorAnalysis< + Impl::FunctorPatternInterface::REDUCE, + TeamPolicy, ReducerType, + value_type>; + using wrapped_reducer_type = typename functor_analysis_type::Reducer; + + wrapped_reducer_type wrapped_reducer(reducer); + value_type value; + wrapped_reducer.init(&value); const iType tidx0 = loop_boundaries.member.item().get_local_id(0); const iType tidx1 = loop_boundaries.member.item().get_local_id(1); @@ -670,8 +720,11 @@ parallel_reduce(const Impl::TeamVectorRangeBoundariesStruct< i < loop_boundaries.end; i += grange0 * grange1) closure(i, value); - loop_boundaries.member.vector_reduce(reducer, value); - loop_boundaries.member.team_reduce(reducer, value); + loop_boundaries.member.impl_vector_reduce(wrapped_reducer, value); + loop_boundaries.member.impl_team_reduce(wrapped_reducer, value); + + wrapped_reducer.final(&value); + reducer.reference() = value; } template @@ -679,10 +732,16 @@ KOKKOS_INLINE_FUNCTION std::enable_if_t::value> parallel_reduce(const Impl::TeamVectorRangeBoundariesStruct< iType, Impl::SYCLTeamMember>& loop_boundaries, const Closure& closure, ValueType& result) { - ValueType val; - Kokkos::Sum reducer(val); + using functor_analysis_type = typename Impl::FunctorAnalysis< + Impl::FunctorPatternInterface::REDUCE, + TeamPolicy, Closure, + ValueType>; + using wrapped_reducer_type = typename functor_analysis_type::Reducer; + using value_type = typename wrapped_reducer_type::value_type; - reducer.init(reducer.reference()); + wrapped_reducer_type wrapped_reducer(closure); + value_type value; + wrapped_reducer.init(&value); const iType tidx0 = loop_boundaries.member.item().get_local_id(0); const iType tidx1 = loop_boundaries.member.item().get_local_id(1); @@ -692,11 +751,13 @@ parallel_reduce(const Impl::TeamVectorRangeBoundariesStruct< for (iType i = loop_boundaries.start + tidx0 * grange1 + tidx1; i < loop_boundaries.end; i += grange0 * grange1) - closure(i, val); + closure(i, value); - loop_boundaries.member.vector_reduce(reducer); - loop_boundaries.member.team_reduce(reducer); - result = reducer.reference(); + loop_boundaries.member.impl_vector_reduce(wrapped_reducer, value); + loop_boundaries.member.impl_team_reduce(wrapped_reducer, value); + + wrapped_reducer.final(&value); + result = value; } //---------------------------------------------------------------------------- @@ -746,16 +807,27 @@ KOKKOS_INLINE_FUNCTION std::enable_if_t::value> parallel_reduce(Impl::ThreadVectorRangeBoundariesStruct< iType, Impl::SYCLTeamMember> const& loop_boundaries, Closure const& closure, ReducerType const& reducer) { - reducer.init(reducer.reference()); + using value_type = typename ReducerType::value_type; + using functor_analysis_type = typename Impl::FunctorAnalysis< + Impl::FunctorPatternInterface::REDUCE, + TeamPolicy, ReducerType, + value_type>; + using wrapped_reducer_type = typename functor_analysis_type::Reducer; + + wrapped_reducer_type wrapped_reducer(reducer); + value_type value; + wrapped_reducer.init(&value); const iType tidx1 = loop_boundaries.member.item().get_local_id(1); const iType grange1 = loop_boundaries.member.item().get_local_range(1); for (iType i = loop_boundaries.start + tidx1; i < loop_boundaries.end; i += grange1) - closure(i, reducer.reference()); + closure(i, value); - loop_boundaries.member.vector_reduce(reducer); + loop_boundaries.member.impl_vector_reduce(wrapped_reducer, value); + wrapped_reducer.final(&value); + reducer.reference() = value; } /** \brief Intra-thread vector parallel_reduce. @@ -774,16 +846,27 @@ KOKKOS_INLINE_FUNCTION std::enable_if_t::value> parallel_reduce(Impl::ThreadVectorRangeBoundariesStruct< iType, Impl::SYCLTeamMember> const& loop_boundaries, Closure const& closure, ValueType& result) { - result = ValueType(); + using functor_analysis_type = typename Impl::FunctorAnalysis< + Impl::FunctorPatternInterface::REDUCE, + TeamPolicy, Closure, + ValueType>; + using wrapped_reducer_type = typename functor_analysis_type::Reducer; + using value_type = typename wrapped_reducer_type::value_type; + + wrapped_reducer_type wrapped_reducer(closure); + value_type value; + wrapped_reducer.init(&value); const iType tidx1 = loop_boundaries.member.item().get_local_id(1); const int grange1 = loop_boundaries.member.item().get_local_range(1); for (iType i = loop_boundaries.start + tidx1; i < loop_boundaries.end; i += grange1) - closure(i, result); + closure(i, value); - loop_boundaries.member.vector_reduce(Kokkos::Sum(result)); + loop_boundaries.member.impl_vector_reduce(wrapped_reducer, value); + wrapped_reducer.final(&value); + result = value; } //---------------------------------------------------------------------------- diff --git a/lib/kokkos/core/src/SYCL/Kokkos_SYCL_TeamPolicy.hpp b/lib/kokkos/core/src/SYCL/Kokkos_SYCL_TeamPolicy.hpp index 17ce59058b..556ca0d281 100644 --- a/lib/kokkos/core/src/SYCL/Kokkos_SYCL_TeamPolicy.hpp +++ b/lib/kokkos/core/src/SYCL/Kokkos_SYCL_TeamPolicy.hpp @@ -22,8 +22,7 @@ #include template -class Kokkos::Impl::TeamPolicyInternal +class Kokkos::Impl::TeamPolicyInternal : public PolicyTraits { public: using execution_policy = TeamPolicyInternal; @@ -45,7 +44,7 @@ class Kokkos::Impl::TeamPolicyInternal TeamPolicyInternal(TeamPolicyInternal const& p) { diff --git a/lib/kokkos/core/src/SYCL/Kokkos_SYCL_UniqueToken.hpp b/lib/kokkos/core/src/SYCL/Kokkos_SYCL_UniqueToken.hpp index d55fc6a84b..79d9e8a8d4 100644 --- a/lib/kokkos/core/src/SYCL/Kokkos_SYCL_UniqueToken.hpp +++ b/lib/kokkos/core/src/SYCL/Kokkos_SYCL_UniqueToken.hpp @@ -22,13 +22,14 @@ #include namespace Kokkos { -namespace Experimental { namespace Impl { Kokkos::View sycl_global_unique_token_locks( bool deallocate = false); } +namespace Experimental { + // both global and instance Unique Tokens are implemented in the same way // the global version has one shared static lock array underneath // but it can't be a static member variable since we need to acces it on device @@ -42,7 +43,7 @@ class UniqueToken { using size_type = int32_t; explicit UniqueToken(execution_space const& = execution_space()) - : m_locks(Impl::sycl_global_unique_token_locks()) {} + : m_locks(Kokkos::Impl::sycl_global_unique_token_locks()) {} KOKKOS_DEFAULTED_FUNCTION UniqueToken(const UniqueToken&) = default; @@ -75,11 +76,15 @@ class UniqueToken { /// \brief acquire value such that 0 <= value < size() KOKKOS_INLINE_FUNCTION size_type impl_acquire() const { +#if defined(__INTEL_LLVM_COMPILER) && __INTEL_LLVM_COMPILER >= 20250000 + auto item = sycl::ext::oneapi::this_work_item::get_nd_item<3>(); +#else auto item = sycl::ext::oneapi::experimental::this_nd_item<3>(); +#endif std::size_t threadIdx[3] = {item.get_local_id(2), item.get_local_id(1), item.get_local_id(0)}; std::size_t blockIdx[3] = {item.get_group(2), item.get_group(1), - item.get_group(0)}; + item.get_group(0)}; std::size_t blockDim[3] = {item.get_local_range(2), item.get_local_range(1), item.get_local_range(0)}; @@ -122,11 +127,11 @@ class UniqueToken public: UniqueToken() : UniqueToken( - Kokkos::Experimental::SYCL().concurrency()) {} + Kokkos::SYCL().concurrency()) {} explicit UniqueToken(execution_space const& arg) : UniqueToken( - Kokkos::Experimental::SYCL().concurrency(), arg) {} + Kokkos::SYCL().concurrency(), arg) {} explicit UniqueToken(size_type max_size) : UniqueToken(max_size) {} diff --git a/lib/kokkos/core/src/SYCL/Kokkos_SYCL_ZeroMemset.hpp b/lib/kokkos/core/src/SYCL/Kokkos_SYCL_ZeroMemset.hpp index 61db6b34aa..2905733a4d 100644 --- a/lib/kokkos/core/src/SYCL/Kokkos_SYCL_ZeroMemset.hpp +++ b/lib/kokkos/core/src/SYCL/Kokkos_SYCL_ZeroMemset.hpp @@ -23,12 +23,11 @@ namespace Kokkos { namespace Impl { -template -struct ZeroMemset> { - ZeroMemset(const Kokkos::Experimental::SYCL& exec_space, - const View& dst) { - auto event = exec_space.impl_internal_space_instance()->m_queue->memset( - dst.data(), 0, dst.size() * sizeof(typename View::value_type)); +template <> +struct ZeroMemset { + ZeroMemset(const Kokkos::SYCL& exec_space, void* dst, size_t cnt) { + auto event = + exec_space.impl_internal_space_instance()->m_queue->memset(dst, 0, cnt); #ifndef KOKKOS_IMPL_SYCL_USE_IN_ORDER_QUEUES exec_space.impl_internal_space_instance() ->m_queue->ext_oneapi_submit_barrier(std::vector{event}); diff --git a/lib/kokkos/core/src/Serial/Kokkos_Serial.hpp b/lib/kokkos/core/src/Serial/Kokkos_Serial.hpp index 81d43b31b3..a1fa9e43e0 100644 --- a/lib/kokkos/core/src/Serial/Kokkos_Serial.hpp +++ b/lib/kokkos/core/src/Serial/Kokkos_Serial.hpp @@ -34,7 +34,6 @@ static_assert(false, #include #include #include -#include #include #include #include @@ -267,7 +266,7 @@ template std::vector partition_space(const Serial&, std::vector const& weights) { static_assert( - std::is_arithmetic::value, + std::is_arithmetic_v, "Kokkos Error: partitioning arguments must be integers or floats"); // We only care about the number of instances to create and ignore weights @@ -284,7 +283,9 @@ std::vector partition_space(const Serial&, #include #include #include +#ifdef KOKKOS_ENABLE_DEPRECATED_CODE_4 #include +#endif #include #endif // defined( KOKKOS_ENABLE_SERIAL ) diff --git a/lib/kokkos/core/src/Serial/Kokkos_Serial_Parallel_MDRange.hpp b/lib/kokkos/core/src/Serial/Kokkos_Serial_Parallel_MDRange.hpp index 34e115eca9..addcaba009 100644 --- a/lib/kokkos/core/src/Serial/Kokkos_Serial_Parallel_MDRange.hpp +++ b/lib/kokkos/core/src/Serial/Kokkos_Serial_Parallel_MDRange.hpp @@ -44,11 +44,16 @@ class ParallelFor, public: inline void execute() const { + // caused a possibly codegen-related slowdown, especially in GCC 9-11 + // with KOKKOS_ARCH_NATIVE + // https://github.com/kokkos/kokkos/issues/7268 +#ifndef KOKKOS_ENABLE_ATOMICS_BYPASS // Make sure kernels are running sequentially even when using multiple // threads auto* internal_instance = m_iter.m_rp.space().impl_internal_space_instance(); std::lock_guard lock(internal_instance->m_instance_mutex); +#endif this->exec(); } template @@ -112,10 +117,15 @@ class ParallelReduce instance_lock( internal_instance->m_instance_mutex); +#endif internal_instance->resize_thread_team_data( pool_reduce_size, team_reduce_size, team_shared_size, thread_local_size); diff --git a/lib/kokkos/core/src/Serial/Kokkos_Serial_Parallel_Range.hpp b/lib/kokkos/core/src/Serial/Kokkos_Serial_Parallel_Range.hpp index 80faec9041..2ab7b7f803 100644 --- a/lib/kokkos/core/src/Serial/Kokkos_Serial_Parallel_Range.hpp +++ b/lib/kokkos/core/src/Serial/Kokkos_Serial_Parallel_Range.hpp @@ -31,7 +31,7 @@ class ParallelFor, Kokkos::Serial> { const Policy m_policy; template - std::enable_if_t::value> exec() const { + std::enable_if_t> exec() const { const typename Policy::member_type e = m_policy.end(); for (typename Policy::member_type i = m_policy.begin(); i < e; ++i) { m_functor(i); @@ -39,7 +39,7 @@ class ParallelFor, Kokkos::Serial> { } template - std::enable_if_t::value> exec() const { + std::enable_if_t> exec() const { const TagType t{}; const typename Policy::member_type e = m_policy.end(); for (typename Policy::member_type i = m_policy.begin(); i < e; ++i) { @@ -49,10 +49,15 @@ class ParallelFor, Kokkos::Serial> { public: inline void execute() const { + // caused a possibly codegen-related slowdown, especially in GCC 9-11 + // with KOKKOS_ARCH_NATIVE + // https://github.com/kokkos/kokkos/issues/7268 +#ifndef KOKKOS_ENABLE_ATOMICS_BYPASS // Make sure kernels are running sequentially even when using multiple // threads auto* internal_instance = m_policy.space().impl_internal_space_instance(); std::lock_guard lock(internal_instance->m_instance_mutex); +#endif this->template exec(); } @@ -79,7 +84,7 @@ class ParallelReduce, const pointer_type m_result_ptr; template - inline std::enable_if_t::value> exec( + inline std::enable_if_t> exec( reference_type update) const { const typename Policy::member_type e = m_policy.end(); for (typename Policy::member_type i = m_policy.begin(); i < e; ++i) { @@ -88,7 +93,7 @@ class ParallelReduce, } template - inline std::enable_if_t::value> exec( + inline std::enable_if_t> exec( reference_type update) const { const TagType t{}; @@ -108,10 +113,15 @@ class ParallelReduce, auto* internal_instance = m_policy.space().impl_internal_space_instance(); + // caused a possibly codegen-related slowdown, especially in GCC 9-11 + // with KOKKOS_ARCH_NATIVE + // https://github.com/kokkos/kokkos/issues/7268 +#ifndef KOKKOS_ENABLE_ATOMICS_BYPASS // Make sure kernels are running sequentially even when using multiple // threads, lock resize_thread_team_data std::lock_guard instance_lock( internal_instance->m_instance_mutex); +#endif internal_instance->resize_thread_team_data( pool_reduce_size, team_reduce_size, team_shared_size, thread_local_size); @@ -166,7 +176,7 @@ class ParallelScan, const Policy m_policy; template - inline std::enable_if_t::value> exec( + inline std::enable_if_t> exec( reference_type update) const { const typename Policy::member_type e = m_policy.end(); for (typename Policy::member_type i = m_policy.begin(); i < e; ++i) { @@ -175,7 +185,7 @@ class ParallelScan, } template - inline std::enable_if_t::value> exec( + inline std::enable_if_t> exec( reference_type update) const { const TagType t{}; const typename Policy::member_type e = m_policy.end(); @@ -194,10 +204,16 @@ class ParallelScan, const size_t thread_local_size = 0; // Never shrinks auto* internal_instance = m_policy.space().impl_internal_space_instance(); + + // caused a possibly codegen-related slowdown, especially in GCC 9-11 + // with KOKKOS_ARCH_NATIVE + // https://github.com/kokkos/kokkos/issues/7268 +#ifndef KOKKOS_ENABLE_ATOMICS_BYPASS // Make sure kernels are running sequentially even when using multiple // threads, lock resize_thread_team_data std::lock_guard instance_lock( internal_instance->m_instance_mutex); +#endif internal_instance->resize_thread_team_data( pool_reduce_size, team_reduce_size, team_shared_size, @@ -235,7 +251,7 @@ class ParallelScanWithTotal, const pointer_type m_result_ptr; template - inline std::enable_if_t::value> exec( + inline std::enable_if_t> exec( reference_type update) const { const typename Policy::member_type e = m_policy.end(); for (typename Policy::member_type i = m_policy.begin(); i < e; ++i) { @@ -244,7 +260,7 @@ class ParallelScanWithTotal, } template - inline std::enable_if_t::value> exec( + inline std::enable_if_t> exec( reference_type update) const { const TagType t{}; const typename Policy::member_type e = m_policy.end(); @@ -262,10 +278,16 @@ class ParallelScanWithTotal, const size_t thread_local_size = 0; // Never shrinks auto* internal_instance = m_policy.space().impl_internal_space_instance(); + + // caused a possibly codegen-related slowdown, especially in GCC 9-11 + // with KOKKOS_ARCH_NATIVE + // https://github.com/kokkos/kokkos/issues/7268 +#ifndef KOKKOS_ENABLE_ATOMICS_BYPASS // Make sure kernels are running sequentially even when using multiple // threads, lock resize_thread_team_data std::lock_guard instance_lock( internal_instance->m_instance_mutex); +#endif internal_instance->resize_thread_team_data( pool_reduce_size, team_reduce_size, team_shared_size, diff --git a/lib/kokkos/core/src/Serial/Kokkos_Serial_Parallel_Team.hpp b/lib/kokkos/core/src/Serial/Kokkos_Serial_Parallel_Team.hpp index a523cc86c9..7a6faf3d9f 100644 --- a/lib/kokkos/core/src/Serial/Kokkos_Serial_Parallel_Team.hpp +++ b/lib/kokkos/core/src/Serial/Kokkos_Serial_Parallel_Team.hpp @@ -223,7 +223,7 @@ class ParallelFor, const size_t m_shared; template - inline std::enable_if_t::value> exec( + inline std::enable_if_t> exec( HostThreadTeamData& data) const { for (int ileague = 0; ileague < m_league; ++ileague) { m_functor(Member(data, ileague, m_league)); @@ -231,7 +231,7 @@ class ParallelFor, } template - inline std::enable_if_t::value> exec( + inline std::enable_if_t> exec( HostThreadTeamData& data) const { const TagType t{}; for (int ileague = 0; ileague < m_league; ++ileague) { @@ -247,10 +247,16 @@ class ParallelFor, const size_t thread_local_size = 0; // Never shrinks auto* internal_instance = m_policy.space().impl_internal_space_instance(); + + // caused a possibly codegen-related slowdown, especially in GCC 9-11 + // with KOKKOS_ARCH_NATIVE + // https://github.com/kokkos/kokkos/issues/7268 +#ifndef KOKKOS_ENABLE_ATOMICS_BYPASS // Make sure kernels are running sequentially even when using multiple // threads, lock resize_thread_team_data std::lock_guard instance_lock( internal_instance->m_instance_mutex); +#endif internal_instance->resize_thread_team_data( pool_reduce_size, team_reduce_size, team_shared_size, @@ -293,7 +299,7 @@ class ParallelReduce - inline std::enable_if_t::value> exec( + inline std::enable_if_t> exec( HostThreadTeamData& data, reference_type update) const { for (int ileague = 0; ileague < m_league; ++ileague) { m_functor_reducer.get_functor()(Member(data, ileague, m_league), update); @@ -301,7 +307,7 @@ class ParallelReduce - inline std::enable_if_t::value> exec( + inline std::enable_if_t> exec( HostThreadTeamData& data, reference_type update) const { const TagType t{}; @@ -321,10 +327,16 @@ class ParallelReduce instance_lock( internal_instance->m_instance_mutex); +#endif internal_instance->resize_thread_team_data( pool_reduce_size, team_reduce_size, team_shared_size, diff --git a/lib/kokkos/core/src/Serial/Kokkos_Serial_Task.hpp b/lib/kokkos/core/src/Serial/Kokkos_Serial_Task.hpp index 5905d6d32e..678d182504 100644 --- a/lib/kokkos/core/src/Serial/Kokkos_Serial_Task.hpp +++ b/lib/kokkos/core/src/Serial/Kokkos_Serial_Task.hpp @@ -25,10 +25,16 @@ #include #include #include +#include //---------------------------------------------------------------------------- //---------------------------------------------------------------------------- +#ifdef KOKKOS_ENABLE_DEPRECATION_WARNINGS +// We allow using deprecated classes in this file +KOKKOS_IMPL_DISABLE_DEPRECATED_WARNINGS_PUSH() +#endif + namespace Kokkos { namespace Impl { @@ -102,9 +108,8 @@ class TaskQueueSpecialization> { template class TaskQueueSpecializationConstrained< - Scheduler, - std::enable_if_t::value>> { + Scheduler, std::enable_if_t>> { public: // Note: Scheduler may be an incomplete type at class scope (but not inside // of the methods, obviously) @@ -215,6 +220,10 @@ extern template class TaskQueue, FunctorType m_functor; template - std::enable_if_t::value> exec_one( + std::enable_if_t> exec_one( const std::int32_t w) const noexcept { m_functor(w); } template - std::enable_if_t::value> exec_one( + std::enable_if_t> exec_one( const std::int32_t w) const noexcept { const TagType t{}; m_functor(t, w); diff --git a/lib/kokkos/core/src/Serial/Kokkos_Serial_ZeroMemset.hpp b/lib/kokkos/core/src/Serial/Kokkos_Serial_ZeroMemset.hpp index 6ad6aabc5a..527e094079 100644 --- a/lib/kokkos/core/src/Serial/Kokkos_Serial_ZeroMemset.hpp +++ b/lib/kokkos/core/src/Serial/Kokkos_Serial_ZeroMemset.hpp @@ -31,15 +31,11 @@ namespace Impl { // parallel execution space since the specialization for // DefaultHostExecutionSpace is defined elsewhere. struct DummyExecutionSpace; -template +template <> struct ZeroMemset< - std::conditional_t::value, - Serial, DummyExecutionSpace>, - View> { - ZeroMemset(const Serial&, const View& dst) { - using ValueType = typename View::value_type; - std::memset(dst.data(), 0, sizeof(ValueType) * dst.size()); - } + std::conditional_t, + Serial, DummyExecutionSpace>> { + ZeroMemset(const Serial&, void* dst, size_t cnt) { std::memset(dst, 0, cnt); } }; } // namespace Impl diff --git a/lib/kokkos/core/src/Threads/Kokkos_Threads_Instance.cpp b/lib/kokkos/core/src/Threads/Kokkos_Threads_Instance.cpp index 3842966cd7..edc9489f67 100644 --- a/lib/kokkos/core/src/Threads/Kokkos_Threads_Instance.cpp +++ b/lib/kokkos/core/src/Threads/Kokkos_Threads_Instance.cpp @@ -67,8 +67,9 @@ std::pair int s_thread_pool_size[3] = {0, 0, 0}; -void (*volatile s_current_function)(ThreadsInternal &, const void *); -const void *volatile s_current_function_arg = nullptr; +using s_current_function_type = void (*)(ThreadsInternal &, const void *); +std::atomic s_current_function; +std::atomic s_current_function_arg = nullptr; inline unsigned fan_size(const unsigned rank, const unsigned size) { const unsigned rank_rev = size - (rank + 1); @@ -79,7 +80,7 @@ inline unsigned fan_size(const unsigned rank, const unsigned size) { return count; } -void wait_yield(volatile ThreadState &flag, const ThreadState value) { +void wait_yield(std::atomic &flag, const ThreadState value) { while (value == flag) { std::this_thread::yield(); } @@ -135,11 +136,12 @@ ThreadsInternal::ThreadsInternal() ThreadsInternal *const nil = nullptr; // Which entry in 's_threads_exec', possibly determined from hwloc binding - const int entry = reinterpret_cast(s_current_function_arg) < - size_t(s_thread_pool_size[0]) - ? reinterpret_cast(s_current_function_arg) - : size_t(Kokkos::hwloc::bind_this_thread( - s_thread_pool_size[0], s_threads_coord)); + const int entry = + reinterpret_cast(s_current_function_arg.load()) < + size_t(s_thread_pool_size[0]) + ? reinterpret_cast(s_current_function_arg.load()) + : size_t(Kokkos::hwloc::bind_this_thread(s_thread_pool_size[0], + s_threads_coord)); // Given a good entry set this thread in the 's_threads_exec' array if (entry < s_thread_pool_size[0] && @@ -543,7 +545,7 @@ void ThreadsInternal::initialize(int thread_count_arg) { for (unsigned ith = 1; ith < thread_count; ++ith) { // Try to protect against cache coherency failure by casting to volatile. ThreadsInternal *const th = - ((ThreadsInternal * volatile *)s_threads_exec)[ith]; + ((ThreadsInternal *volatile *)s_threads_exec)[ith]; if (th) { wait_yield(th->m_pool_state, ThreadState::Active); } else { diff --git a/lib/kokkos/core/src/Threads/Kokkos_Threads_Instance.hpp b/lib/kokkos/core/src/Threads/Kokkos_Threads_Instance.hpp index a5eb231cb0..130b3433d0 100644 --- a/lib/kokkos/core/src/Threads/Kokkos_Threads_Instance.hpp +++ b/lib/kokkos/core/src/Threads/Kokkos_Threads_Instance.hpp @@ -60,7 +60,7 @@ class ThreadsInternal { int m_pool_rank_rev; int m_pool_size; int m_pool_fan_size; - ThreadState volatile m_pool_state; ///< State for global synchronizations + std::atomic m_pool_state; ///< State for global synchronizations // Members for dynamic scheduling // Which thread am I stealing from currently @@ -96,7 +96,7 @@ class ThreadsInternal { return reinterpret_cast(m_scratch) + m_scratch_reduce_end; } - KOKKOS_INLINE_FUNCTION ThreadState volatile &state() { return m_pool_state; } + KOKKOS_INLINE_FUNCTION auto &state() { return m_pool_state; } KOKKOS_INLINE_FUNCTION ThreadsInternal *const *pool_base() const { return m_pool_base; } @@ -225,7 +225,7 @@ class ThreadsInternal { // to inactive triggers another thread to exit a spinwait // and read the 'reduce_memory'. // Must 'memory_fence()' to guarantee that storing the update to - // 'reduce_memory()' will complete before storing the the update to + // 'reduce_memory()' will complete before storing the update to // 'm_pool_state'. memory_fence(); @@ -403,7 +403,7 @@ class ThreadsInternal { static void start(void (*)(ThreadsInternal &, const void *), const void *); #ifdef KOKKOS_ENABLE_DEPRECATED_CODE_4 - KOKKOS_DEPRECATED static int in_parallel(); + static int in_parallel(); #endif static void fence(); static void fence(const std::string &); diff --git a/lib/kokkos/core/src/Threads/Kokkos_Threads_ParallelFor_MDRange.hpp b/lib/kokkos/core/src/Threads/Kokkos_Threads_ParallelFor_MDRange.hpp index 59577609ab..711b1b6926 100644 --- a/lib/kokkos/core/src/Threads/Kokkos_Threads_ParallelFor_MDRange.hpp +++ b/lib/kokkos/core/src/Threads/Kokkos_Threads_ParallelFor_MDRange.hpp @@ -51,7 +51,7 @@ class ParallelFor, } template - static std::enable_if_t::value> + static std::enable_if_t> exec_schedule(ThreadsInternal &instance, const void *arg) { const ParallelFor &self = *((const ParallelFor *)arg); @@ -65,7 +65,7 @@ class ParallelFor, } template - static std::enable_if_t::value> + static std::enable_if_t> exec_schedule(ThreadsInternal &instance, const void *arg) { const ParallelFor &self = *((const ParallelFor *)arg); diff --git a/lib/kokkos/core/src/Threads/Kokkos_Threads_ParallelFor_Range.hpp b/lib/kokkos/core/src/Threads/Kokkos_Threads_ParallelFor_Range.hpp index 4a89c4fad8..25aab9ebfb 100644 --- a/lib/kokkos/core/src/Threads/Kokkos_Threads_ParallelFor_Range.hpp +++ b/lib/kokkos/core/src/Threads/Kokkos_Threads_ParallelFor_Range.hpp @@ -35,7 +35,7 @@ class ParallelFor, const Policy m_policy; template - inline static std::enable_if_t::value> exec_range( + inline static std::enable_if_t> exec_range( const FunctorType &functor, const Member ibeg, const Member iend) { #if defined(KOKKOS_ENABLE_AGGRESSIVE_VECTORIZATION) && \ defined(KOKKOS_ENABLE_PRAGMA_IVDEP) @@ -47,7 +47,7 @@ class ParallelFor, } template - inline static std::enable_if_t::value> exec_range( + inline static std::enable_if_t> exec_range( const FunctorType &functor, const Member ibeg, const Member iend) { const TagType t{}; #if defined(KOKKOS_ENABLE_AGGRESSIVE_VECTORIZATION) && \ @@ -64,7 +64,7 @@ class ParallelFor, } template - static std::enable_if_t::value> + static std::enable_if_t> exec_schedule(ThreadsInternal &instance, const void *arg) { const ParallelFor &self = *((const ParallelFor *)arg); @@ -77,7 +77,7 @@ class ParallelFor, } template - static std::enable_if_t::value> + static std::enable_if_t> exec_schedule(ThreadsInternal &instance, const void *arg) { const ParallelFor &self = *((const ParallelFor *)arg); diff --git a/lib/kokkos/core/src/Threads/Kokkos_Threads_ParallelFor_Team.hpp b/lib/kokkos/core/src/Threads/Kokkos_Threads_ParallelFor_Team.hpp index f927d7c6a6..40be3884c3 100644 --- a/lib/kokkos/core/src/Threads/Kokkos_Threads_ParallelFor_Team.hpp +++ b/lib/kokkos/core/src/Threads/Kokkos_Threads_ParallelFor_Team.hpp @@ -36,8 +36,8 @@ class ParallelFor, const size_t m_shared; template - inline static std::enable_if_t::value && - std::is_same::value> + inline static std::enable_if_t && + std::is_same_v> exec_team(const FunctorType &functor, Member member) { for (; member.valid_static(); member.next_static()) { functor(member); @@ -45,8 +45,8 @@ class ParallelFor, } template - inline static std::enable_if_t::value && - std::is_same::value> + inline static std::enable_if_t && + std::is_same_v> exec_team(const FunctorType &functor, Member member) { const TagType t{}; for (; member.valid_static(); member.next_static()) { @@ -55,8 +55,8 @@ class ParallelFor, } template - inline static std::enable_if_t::value && - std::is_same::value> + inline static std::enable_if_t && + std::is_same_v> exec_team(const FunctorType &functor, Member member) { for (; member.valid_dynamic(); member.next_dynamic()) { functor(member); @@ -64,8 +64,8 @@ class ParallelFor, } template - inline static std::enable_if_t::value && - std::is_same::value> + inline static std::enable_if_t && + std::is_same_v> exec_team(const FunctorType &functor, Member member) { const TagType t{}; for (; member.valid_dynamic(); member.next_dynamic()) { @@ -88,8 +88,12 @@ class ParallelFor, policy.impl_set_vector_length(1); } if (policy.team_size() < 0) { - policy.impl_set_team_size( - policy.team_size_recommended(m_functor, ParallelForTag{})); + int team_size = policy.team_size_recommended(m_functor, ParallelForTag{}); + if (team_size <= 0) + Kokkos::Impl::throw_runtime_exception( + "Kokkos::Impl::ParallelFor could not find a " + "valid execution configuration."); + policy.impl_set_team_size(team_size); } return policy; } diff --git a/lib/kokkos/core/src/Threads/Kokkos_Threads_ParallelReduce_MDRange.hpp b/lib/kokkos/core/src/Threads/Kokkos_Threads_ParallelReduce_MDRange.hpp index fa63215a9e..9f28f9bbfc 100644 --- a/lib/kokkos/core/src/Threads/Kokkos_Threads_ParallelReduce_MDRange.hpp +++ b/lib/kokkos/core/src/Threads/Kokkos_Threads_ParallelReduce_MDRange.hpp @@ -59,7 +59,7 @@ class ParallelReduce - static std::enable_if_t::value> + static std::enable_if_t> exec_schedule(ThreadsInternal &instance, const void *arg) { const ParallelReduce &self = *((const ParallelReduce *)arg); @@ -76,7 +76,7 @@ class ParallelReduce - static std::enable_if_t::value> + static std::enable_if_t> exec_schedule(ThreadsInternal &instance, const void *arg) { const ParallelReduce &self = *((const ParallelReduce *)arg); @@ -91,8 +91,8 @@ class ParallelReduce(instance.reduce_memory())); + reference_type update = + reducer.init(static_cast(instance.reduce_memory())); while (work_index != -1) { const Member begin = static_cast(work_index); const Member end = begin + 1 < num_tiles ? begin + 1 : num_tiles; @@ -100,7 +100,7 @@ class ParallelReduce, const pointer_type m_result_ptr; template - inline static std::enable_if_t::value> exec_range( + inline static std::enable_if_t> exec_range( const FunctorType &functor, const Member &ibeg, const Member &iend, reference_type update) { #if defined(KOKKOS_ENABLE_AGGRESSIVE_VECTORIZATION) && \ @@ -55,7 +55,7 @@ class ParallelReduce, } template - inline static std::enable_if_t::value> exec_range( + inline static std::enable_if_t> exec_range( const FunctorType &functor, const Member &ibeg, const Member &iend, reference_type update) { const TagType t{}; @@ -73,7 +73,7 @@ class ParallelReduce, } template - static std::enable_if_t::value> + static std::enable_if_t> exec_schedule(ThreadsInternal &instance, const void *arg) { const ParallelReduce &self = *((const ParallelReduce *)arg); const WorkRange range(self.m_policy, instance.pool_rank(), @@ -89,7 +89,7 @@ class ParallelReduce, } template - static std::enable_if_t::value> + static std::enable_if_t> exec_schedule(ThreadsInternal &instance, const void *arg) { const ParallelReduce &self = *((const ParallelReduce *)arg); const WorkRange range(self.m_policy, instance.pool_rank(), diff --git a/lib/kokkos/core/src/Threads/Kokkos_Threads_ParallelReduce_Team.hpp b/lib/kokkos/core/src/Threads/Kokkos_Threads_ParallelReduce_Team.hpp index 4db310701f..69527ee3e6 100644 --- a/lib/kokkos/core/src/Threads/Kokkos_Threads_ParallelReduce_Team.hpp +++ b/lib/kokkos/core/src/Threads/Kokkos_Threads_ParallelReduce_Team.hpp @@ -42,7 +42,7 @@ class ParallelReduce - inline static std::enable_if_t::value> exec_team( + inline static std::enable_if_t> exec_team( const FunctorType &functor, Member member, reference_type update) { for (; member.valid_static(); member.next_static()) { functor(member, update); @@ -50,7 +50,7 @@ class ParallelReduce - inline static std::enable_if_t::value> exec_team( + inline static std::enable_if_t> exec_team( const FunctorType &functor, Member member, reference_type update) { const TagType t{}; for (; member.valid_static(); member.next_static()) { @@ -106,9 +106,14 @@ class ParallelReduce could not find " + "a valid execution configuration."); + policy.impl_set_team_size(team_size); } return policy; } diff --git a/lib/kokkos/core/src/Threads/Kokkos_Threads_ParallelScan_Range.hpp b/lib/kokkos/core/src/Threads/Kokkos_Threads_ParallelScan_Range.hpp index 62f34d741f..d54f4ca952 100644 --- a/lib/kokkos/core/src/Threads/Kokkos_Threads_ParallelScan_Range.hpp +++ b/lib/kokkos/core/src/Threads/Kokkos_Threads_ParallelScan_Range.hpp @@ -39,7 +39,7 @@ class ParallelScan, const Policy m_policy; template - inline static std::enable_if_t::value> exec_range( + inline static std::enable_if_t> exec_range( const FunctorType &functor, const Member &ibeg, const Member &iend, reference_type update, const bool final) { #if defined(KOKKOS_ENABLE_AGGRESSIVE_VECTORIZATION) && \ @@ -52,7 +52,7 @@ class ParallelScan, } template - inline static std::enable_if_t::value> exec_range( + inline static std::enable_if_t> exec_range( const FunctorType &functor, const Member &ibeg, const Member &iend, reference_type update, const bool final) { const TagType t{}; @@ -119,7 +119,7 @@ class ParallelScanWithTotal, const pointer_type m_result_ptr; template - inline static std::enable_if_t::value> exec_range( + inline static std::enable_if_t> exec_range( const FunctorType &functor, const Member &ibeg, const Member &iend, reference_type update, const bool final) { #if defined(KOKKOS_ENABLE_AGGRESSIVE_VECTORIZATION) && \ @@ -132,7 +132,7 @@ class ParallelScanWithTotal, } template - inline static std::enable_if_t::value> exec_range( + inline static std::enable_if_t> exec_range( const FunctorType &functor, const Member &ibeg, const Member &iend, reference_type update, const bool final) { const TagType t{}; diff --git a/lib/kokkos/core/src/Threads/Kokkos_Threads_Spinwait.cpp b/lib/kokkos/core/src/Threads/Kokkos_Threads_Spinwait.cpp index 3df9dc07bf..0f9a77f2af 100644 --- a/lib/kokkos/core/src/Threads/Kokkos_Threads_Spinwait.cpp +++ b/lib/kokkos/core/src/Threads/Kokkos_Threads_Spinwait.cpp @@ -108,7 +108,7 @@ void host_thread_yield(const uint32_t i, const WaitMode mode) { #endif /* defined( KOKKOS_ENABLE_ASM ) */ } -void spinwait_while_equal(ThreadState const volatile& flag, +void spinwait_while_equal(std::atomic const& flag, ThreadState const value) { Kokkos::store_fence(); uint32_t i = 0; diff --git a/lib/kokkos/core/src/Threads/Kokkos_Threads_Spinwait.hpp b/lib/kokkos/core/src/Threads/Kokkos_Threads_Spinwait.hpp index b98b6dbb73..7ab43cdb7a 100644 --- a/lib/kokkos/core/src/Threads/Kokkos_Threads_Spinwait.hpp +++ b/lib/kokkos/core/src/Threads/Kokkos_Threads_Spinwait.hpp @@ -20,6 +20,7 @@ #include #include +#include namespace Kokkos { namespace Impl { @@ -34,7 +35,7 @@ enum class WaitMode : int { void host_thread_yield(const uint32_t i, const WaitMode mode); -void spinwait_while_equal(ThreadState const volatile& flag, +void spinwait_while_equal(std::atomic const& flag, ThreadState const value); } // namespace Impl diff --git a/lib/kokkos/core/src/Threads/Kokkos_Threads_Team.hpp b/lib/kokkos/core/src/Threads/Kokkos_Threads_Team.hpp index a3501a437d..f627e0d47a 100644 --- a/lib/kokkos/core/src/Threads/Kokkos_Threads_Team.hpp +++ b/lib/kokkos/core/src/Threads/Kokkos_Threads_Team.hpp @@ -143,8 +143,8 @@ class ThreadsExecTeamMember { KOKKOS_IF_ON_HOST(( // Make sure there is enough scratch space: - using type = typename if_c::type; + using type = std::conditional_t; if (m_team_base) { type* const local_value = ((type*)m_team_base[0]->scratch_memory()); @@ -164,8 +164,8 @@ class ThreadsExecTeamMember { KOKKOS_IF_ON_HOST(( // Make sure there is enough scratch space: - using type = typename if_c::type; + using type = std::conditional_t; f(value); if (m_team_base) { type* const local_value = ((type*)m_team_base[0]->scratch_memory()); memory_fence(); @@ -186,7 +186,7 @@ class ThreadsExecTeamMember { KOKKOS_IF_ON_HOST(( // Make sure there is enough scratch space: using type = - typename if_c::type; + std::conditional_t; if (team_rank() != team_size() - 1) * ((volatile type*)m_instance->scratch_memory()) = value; @@ -215,52 +215,65 @@ class ThreadsExecTeamMember { } template - KOKKOS_INLINE_FUNCTION - std::enable_if_t::value> - team_reduce(const ReducerType& reducer, - const typename ReducerType::value_type contribution) const { + KOKKOS_INLINE_FUNCTION std::enable_if_t::value> + team_reduce(const ReducerType& reducer, + typename ReducerType::value_type& contribution) const { KOKKOS_IF_ON_DEVICE(((void)reducer; (void)contribution;)) - KOKKOS_IF_ON_HOST(( - using value_type = typename ReducerType::value_type; - // Make sure there is enough scratch space: - using type = typename if_c::type; + KOKKOS_IF_ON_HOST( + (using value_type = typename ReducerType::value_type; + using wrapped_reducer_type = typename Impl::FunctorAnalysis< + Impl::FunctorPatternInterface::REDUCE, TeamPolicy, + ReducerType, value_type>::Reducer; + impl_team_reduce(wrapped_reducer_type(reducer), contribution); + reducer.reference() = contribution;)) + } - type* const local_value = ((type*)m_instance->scratch_memory()); + template + KOKKOS_INLINE_FUNCTION std::enable_if_t::value> + impl_team_reduce( + const WrappedReducerType& wrapped_reducer, + typename WrappedReducerType::value_type& contribution) const { + using value_type = typename WrappedReducerType::value_type; + // Make sure there is enough scratch space: + using type = std::conditional_t; - // Set this thread's contribution - if (team_rank() != team_size() - 1) { *local_value = contribution; } + type* const local_value = ((type*)m_instance->scratch_memory()); - // Fence to make sure the base team member has access: - memory_fence(); + // Set this thread's contribution + if (team_rank() != team_size() - 1) { + *local_value = contribution; + } - if (team_fan_in()) { - // The last thread to synchronize returns true, all other threads - // wait for team_fan_out() - type* const team_value = ((type*)m_team_base[0]->scratch_memory()); + // Fence to make sure the base team member has access: + memory_fence(); - *team_value = contribution; - // Join to the team value: - for (int i = 1; i < m_team_size; ++i) { - reducer.join(*team_value, - *((type*)m_team_base[i]->scratch_memory())); - } + if (team_fan_in()) { + // The last thread to synchronize returns true, all other threads + // wait for team_fan_out() + type* const team_value = ((type*)m_team_base[0]->scratch_memory()); - // Team base thread may "lap" member threads so copy out to their - // local value. - for (int i = 1; i < m_team_size; ++i) { - *((type*)m_team_base[i]->scratch_memory()) = *team_value; - } + *team_value = contribution; + // Join to the team value: + for (int i = 1; i < m_team_size; ++i) { + wrapped_reducer.join(team_value, + ((type*)m_team_base[i]->scratch_memory())); + } - // Fence to make sure all team members have access - memory_fence(); - } + // Team base thread may "lap" member threads so copy out to their + // local value. + for (int i = 1; i < m_team_size; ++i) { + *((type*)m_team_base[i]->scratch_memory()) = *team_value; + } - team_fan_out(); + // Fence to make sure all team members have access + memory_fence(); + } - // Value was changed by the team base - reducer.reference() = *local_value;)) + team_fan_out(); + + contribution = *local_value; } /** \brief Intra-team exclusive prefix sum with team_rank() ordering @@ -278,8 +291,8 @@ class ThreadsExecTeamMember { KOKKOS_IF_ON_DEVICE(((void)global_accum; return value;)) KOKKOS_IF_ON_HOST(( // Make sure there is enough scratch space: - using type = typename if_c::type; + using type = std::conditional_t; volatile type* const work_value = ((type*)m_instance->scratch_memory()); @@ -887,19 +900,25 @@ KOKKOS_INLINE_FUNCTION std::enable_if_t::value> parallel_reduce(const Impl::TeamThreadRangeBoundariesStruct< iType, Impl::ThreadsExecTeamMember>& loop_boundaries, const Lambda& lambda, ValueType& result) { - ValueType intermediate; - Sum sum(intermediate); - sum.init(intermediate); + using functor_analysis_type = typename Impl::FunctorAnalysis< + Impl::FunctorPatternInterface::REDUCE, + TeamPolicy, Lambda, + ValueType>; + using wrapped_reducer_type = typename functor_analysis_type::Reducer; + using value_type = typename wrapped_reducer_type::value_type; + + wrapped_reducer_type wrapped_reducer(lambda); + value_type value; + wrapped_reducer.init(&value); for (iType i = loop_boundaries.start; i < loop_boundaries.end; i += loop_boundaries.increment) { - ValueType tmp = ValueType(); - lambda(i, tmp); - intermediate += tmp; + lambda(i, value); } - loop_boundaries.thread.team_reduce(sum, intermediate); - result = sum.reference(); + loop_boundaries.thread.impl_team_reduce(wrapped_reducer, value); + wrapped_reducer.final(&value); + result = value; } template @@ -907,15 +926,25 @@ KOKKOS_INLINE_FUNCTION std::enable_if_t::value> parallel_reduce(const Impl::TeamThreadRangeBoundariesStruct< iType, Impl::ThreadsExecTeamMember>& loop_boundaries, const Lambda& lambda, const ReducerType& reducer) { - typename ReducerType::value_type value; - reducer.init(value); + using value_type = typename ReducerType::value_type; + using functor_analysis_type = typename Impl::FunctorAnalysis< + Impl::FunctorPatternInterface::REDUCE, + TeamPolicy, + ReducerType, value_type>; + using wrapped_reducer_type = typename functor_analysis_type::Reducer; + + wrapped_reducer_type wrapped_reducer(reducer); + value_type value; + wrapped_reducer.init(&value); for (iType i = loop_boundaries.start; i < loop_boundaries.end; i += loop_boundaries.increment) { lambda(i, value); } - loop_boundaries.thread.team_reduce(reducer, value); + loop_boundaries.thread.impl_team_reduce(wrapped_reducer, value); + wrapped_reducer.final(&value); + reducer.reference() = value; } } // namespace Kokkos @@ -950,11 +979,24 @@ KOKKOS_INLINE_FUNCTION std::enable_if_t::value> parallel_reduce(const Impl::ThreadVectorRangeBoundariesStruct< iType, Impl::ThreadsExecTeamMember>& loop_boundaries, const Lambda& lambda, ValueType& result) { - result = ValueType(); + using functor_analysis_type = typename Impl::FunctorAnalysis< + Impl::FunctorPatternInterface::REDUCE, + TeamPolicy, Lambda, + ValueType>; + using wrapped_reducer_type = typename functor_analysis_type::Reducer; + using value_type = typename wrapped_reducer_type::value_type; + + wrapped_reducer_type wrapped_reducer(lambda); + value_type value; + wrapped_reducer.init(&value); + for (iType i = loop_boundaries.start; i < loop_boundaries.end; i += loop_boundaries.increment) { - lambda(i, result); + lambda(i, value); } + + wrapped_reducer.final(&value); + result = value; } template @@ -962,11 +1004,24 @@ KOKKOS_INLINE_FUNCTION std::enable_if_t::value> parallel_reduce(const Impl::ThreadVectorRangeBoundariesStruct< iType, Impl::ThreadsExecTeamMember>& loop_boundaries, const Lambda& lambda, const ReducerType& reducer) { - reducer.init(reducer.reference()); + using value_type = typename ReducerType::value_type; + using functor_analysis_type = typename Impl::FunctorAnalysis< + Impl::FunctorPatternInterface::REDUCE, + TeamPolicy, + ReducerType, value_type>; + using wrapped_reducer_type = typename functor_analysis_type::Reducer; + + wrapped_reducer_type wrapped_reducer(reducer); + value_type value; + wrapped_reducer.init(&value); + for (iType i = loop_boundaries.start; i < loop_boundaries.end; i += loop_boundaries.increment) { - lambda(i, reducer.reference()); + lambda(i, value); } + + wrapped_reducer.final(&value); + reducer.reference() = value; } /** \brief Inter-thread parallel exclusive prefix sum. Executes @@ -1049,7 +1104,7 @@ KOKKOS_INLINE_FUNCTION void parallel_scan( typename Impl::FunctorAnalysis, FunctorType, void>::value_type; - static_assert(std::is_same::value, + static_assert(std::is_same_v, "Non-matching value types of closure and return type"); ValueType scan_val = ValueType(); diff --git a/lib/kokkos/core/src/Threads/Kokkos_Threads_WorkGraphPolicy.hpp b/lib/kokkos/core/src/Threads/Kokkos_Threads_WorkGraphPolicy.hpp index c88d66db5f..5fed92db26 100644 --- a/lib/kokkos/core/src/Threads/Kokkos_Threads_WorkGraphPolicy.hpp +++ b/lib/kokkos/core/src/Threads/Kokkos_Threads_WorkGraphPolicy.hpp @@ -36,13 +36,13 @@ class ParallelFor, FunctorType m_functor; template - std::enable_if_t::value> exec_one( + std::enable_if_t> exec_one( const std::int32_t w) const noexcept { m_functor(w); } template - std::enable_if_t::value> exec_one( + std::enable_if_t> exec_one( const std::int32_t w) const noexcept { const TagType t{}; m_functor(t, w); diff --git a/lib/kokkos/core/src/View/Kokkos_BasicView.hpp b/lib/kokkos/core/src/View/Kokkos_BasicView.hpp new file mode 100644 index 0000000000..29eafca62e --- /dev/null +++ b/lib/kokkos/core/src/View/Kokkos_BasicView.hpp @@ -0,0 +1,652 @@ +//@HEADER +// ************************************************************************ +// +// Kokkos v. 4.0 +// Copyright (2022) National Technology & Engineering +// Solutions of Sandia, LLC (NTESS). +// +// Under the terms of Contract DE-NA0003525 with NTESS, +// the U.S. Government retains certain rights in this software. +// +// Part of Kokkos, under the Apache License v2.0 with LLVM Exceptions. +// See https://kokkos.org/LICENSE for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//@HEADER + +#ifndef KOKKOS_IMPL_PUBLIC_INCLUDE +static_assert(false, + "Including non-public Kokkos header files is not allowed."); +#endif + +#ifndef KOKKOS_BASIC_VIEW_HPP +#define KOKKOS_BASIC_VIEW_HPP +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +// FIXME: we need to make this work for not using our mdspan impl +#define KOKKOS_IMPL_NO_UNIQUE_ADDRESS _MDSPAN_NO_UNIQUE_ADDRESS +namespace Kokkos::Impl { + +constexpr inline struct SubViewCtorTag { + explicit SubViewCtorTag() = default; +} subview_ctor_tag{}; + +template +struct KokkosSliceToMDSpanSliceImpl { + using type = T; + KOKKOS_FUNCTION + static constexpr decltype(auto) transform(const T &s) { return s; } +}; + +template <> +struct KokkosSliceToMDSpanSliceImpl { + using type = full_extent_t; + KOKKOS_FUNCTION + static constexpr decltype(auto) transform(Kokkos::ALL_t) { + return full_extent; + } +}; + +template +using kokkos_slice_to_mdspan_slice = + typename KokkosSliceToMDSpanSliceImpl::type; + +template +KOKKOS_INLINE_FUNCTION constexpr decltype(auto) +transform_kokkos_slice_to_mdspan_slice(const T &s) { + return KokkosSliceToMDSpanSliceImpl::transform(s); +} + +// We do have implementation detail versions of these in our mdspan impl +// However they are not part of the public standard interface +template +struct is_layout_right_padded : public std::false_type {}; + +template +struct is_layout_right_padded> + : public std::true_type {}; + +template +struct is_layout_left_padded : public std::false_type {}; + +template +struct is_layout_left_padded> + : public std::true_type {}; + +template +class BasicView { + public: + using mdspan_type = + mdspan; + using extents_type = typename mdspan_type::extents_type; + using layout_type = typename mdspan_type::layout_type; + using accessor_type = typename mdspan_type::accessor_type; + using mapping_type = typename mdspan_type::mapping_type; + using element_type = typename mdspan_type::element_type; + using value_type = typename mdspan_type::value_type; + using index_type = typename mdspan_type::index_type; + using size_type = typename mdspan_type::size_type; + using rank_type = typename mdspan_type::rank_type; + using data_handle_type = typename mdspan_type::data_handle_type; + using reference = typename mdspan_type::reference; + using memory_space = typename accessor_type::memory_space; + using execution_space = typename memory_space::execution_space; + + // For now View and BasicView will have a restriction that the data handle + // needs to be convertible to element_type* and vice versa + static_assert(std::is_constructible_v); + static_assert(std::is_constructible_v); + + KOKKOS_FUNCTION static constexpr rank_type rank() noexcept { + return extents_type::rank(); + } + KOKKOS_FUNCTION static constexpr rank_type rank_dynamic() noexcept { + return extents_type::rank_dynamic(); + } + KOKKOS_FUNCTION static constexpr size_t static_extent(rank_type r) noexcept { + return extents_type::static_extent(r); + } + KOKKOS_FUNCTION constexpr index_type extent(rank_type r) const noexcept { + return m_map.extents().extent(r); + }; + + protected: + // These are pre-condition checks which are unconditionally (i.e. in release + // mode) enabled in Kokkos::View 4.4 + template + KOKKOS_FUNCTION static constexpr void check_basic_view_constructibility( + [[maybe_unused]] const OtherMapping &rhs) { + using src_t = typename OtherMapping::layout_type; + using dst_t = layout_type; + constexpr size_t rnk = mdspan_type::rank(); + if constexpr (!std::is_same_v) { + if constexpr (Impl::is_layout_left_padded::value) { + if constexpr (std::is_same_v) { + index_type stride = 1; + for (size_t r = 0; r < rnk; r++) { + if (rhs.stride(r) != stride) + Kokkos::abort("View assignment must have compatible layouts"); + if constexpr (rnk > 1) + stride *= (r == 0 ? rhs.stride(1) : rhs.extents().extent(r)); + } + } + } + if constexpr (Impl::is_layout_right_padded::value) { + if constexpr (std::is_same_v) { + index_type stride = 1; + if constexpr (rnk > 0) { + for (size_t r = rnk; r > 0; r--) { + if (rhs.stride(r - 1) != stride) + Kokkos::abort("View assignment must have compatible layouts"); + if constexpr (rnk > 1) + stride *= (r == rnk ? rhs.stride(r - 2) + : rhs.extents().extent(r - 1)); + } + } + } + } + if constexpr (std::is_same_v) { + if constexpr (std::is_same_v) { + index_type stride = 1; + for (size_t r = 0; r < rnk; r++) { + if (rhs.stride(r) != stride) + Kokkos::abort("View assignment must have compatible layouts"); + stride *= rhs.extents().extent(r); + } + } else if constexpr (Impl::is_layout_left_padded::value && + rnk > 1) { + if (rhs.stride(1) != rhs.extents().extent(0)) + Kokkos::abort("View assignment must have compatible layouts"); + } + } + if constexpr (std::is_same_v) { + if constexpr (std::is_same_v) { + index_type stride = 1; + if constexpr (rnk > 0) { + for (size_t r = rnk; r > 0; r--) { + if (rhs.stride(r - 1) != stride) + Kokkos::abort("View assignment must have compatible layouts"); + stride *= rhs.extents().extent(r - 1); + } + } + } else if constexpr (Impl::is_layout_right_padded::value && + rnk > 1) { + if (rhs.stride(rnk - 2) != rhs.extents().extent(rnk - 1)) + Kokkos::abort("View assignment must have compatible layouts"); + } + } + } + } + + public: + KOKKOS_DEFAULTED_FUNCTION constexpr BasicView() = default; + + KOKKOS_FUNCTION constexpr BasicView(const mdspan_type &other) + : m_ptr(other.data_handle()), + m_map(other.mapping()), + m_acc(other.accessor()){}; + KOKKOS_FUNCTION constexpr BasicView(mdspan_type &&other) + : m_ptr(std::move(other.data_handle())), + m_map(std::move(other.mapping())), + m_acc(std::move(other.accessor())){}; + + template + // requires(std::is_constructible_v) + KOKKOS_FUNCTION explicit constexpr BasicView( + std::enable_if_t, + data_handle_type> + p, + OtherIndexTypes... exts) + : m_ptr(std::move(p)), + m_map(extents_type(static_cast(std::move(exts))...)), + m_acc{} {} + + template + // When doing C++20 we should switch to this, the conditional explicit we + // can't do in 17 + // requires(std::is_constructible_v>) + // explicit(Size != rank_dynamic()) + KOKKOS_FUNCTION constexpr BasicView( + std::enable_if_t< + std::is_constructible_v>, + data_handle_type> + p, + const Array &exts) + : m_ptr(std::move(p)), m_map(extents_type(exts)), m_acc{} {} + + KOKKOS_FUNCTION constexpr BasicView(data_handle_type p, + const extents_type &exts) +// Compilation will simply fail in C++17 and overload set should not be an issue +#ifndef KOKKOS_ENABLE_CXX17 + requires(std::is_default_constructible_v && + std::is_constructible_v) +#endif + : m_ptr(std::move(p)), m_map(exts), m_acc{} { + } + + KOKKOS_FUNCTION constexpr BasicView(data_handle_type p, const mapping_type &m) +// Compilation will simply fail in C++17 and overload set should not be an issue +#ifndef KOKKOS_ENABLE_CXX17 + requires(std::is_default_constructible_v) +#endif + : m_ptr(std::move(p)), m_map(m), m_acc{} { + } + + KOKKOS_FUNCTION constexpr BasicView(data_handle_type p, const mapping_type &m, + const accessor_type &a) + : m_ptr(std::move(p)), m_map(m), m_acc(a) {} + + template +// requires(std::is_constructible_v::mdspan_type>) +#ifndef KOKKOS_ENABLE_CXX17 + explicit( + !std::is_convertible_v &, + mapping_type> || + !std::is_convertible_v) +#endif + KOKKOS_INLINE_FUNCTION + BasicView(const BasicView &other, + std::enable_if_t< + std::is_constructible_v< + mdspan_type, typename BasicView::mdspan_type>, + void *> = nullptr) + : m_ptr(other.m_ptr), m_map(other.m_map), m_acc(other.m_acc) { + // Kokkos View precondition checks happen in release builds + check_basic_view_constructibility(other.mapping()); + + static_assert( + std::is_constructible_v, + "Kokkos::View: incompatible data_handle_type for View construction"); + static_assert(std::is_constructible_v, + "Kokkos::View: incompatible extents for View construction"); + } + + template +// requires(std::is_constructible_v>) +#ifndef KOKKOS_ENABLE_CXX17 + explicit( + !std::is_convertible_v &, + mapping_type> || + !std::is_convertible_v) +#endif + KOKKOS_INLINE_FUNCTION + BasicView(const mdspan &other, + std::enable_if_t< + std::is_constructible_v< + mdspan_type, mdspan>, + void *> = nullptr) + : m_ptr(other.data_handle()), + m_map(other.mapping()), + m_acc(other.accessor()) { + // Kokkos View precondition checks happen in release builds + check_basic_view_constructibility(other.mapping()); + + static_assert( + std::is_constructible_v, + "Kokkos::View: incompatible data_handle_type for View construction"); + static_assert(std::is_constructible_v, + "Kokkos::View: incompatible extents for View construction"); + } + + // Allocating constructors specific to BasicView + /// + /// Construct from a given mapping + /// + explicit constexpr BasicView(const std::string &label, + const mapping_type &mapping) + : BasicView(view_alloc(label), mapping) {} + + /// + /// Construct from a given extents + /// + explicit constexpr BasicView(const std::string &label, + const extents_type &ext) + : BasicView(view_alloc(label), mapping_type{ext}) {} + + private: + template + data_handle_type create_data_handle( + const Impl::ViewCtorProp &arg_prop, + const typename mdspan_type::mapping_type &arg_mapping) { + constexpr bool has_exec = Impl::ViewCtorProp::has_execution_space; + // Copy the input allocation properties with possibly defaulted properties + // We need to split it in two to avoid MSVC compiler errors + auto prop_copy_tmp = + Impl::with_properties_if_unset(arg_prop, std::string{}); + auto prop_copy = Impl::with_properties_if_unset( + prop_copy_tmp, memory_space{}, execution_space{}); + using alloc_prop = decltype(prop_copy); + + if (alloc_prop::initialize && + !alloc_prop::execution_space::impl_is_initialized()) { + // If initializing view data then + // the execution space must be initialized. + Kokkos::Impl::throw_runtime_exception( + "Constructing View and initializing data with uninitialized " + "execution space"); + } + return data_handle_type(Impl::make_shared_allocation_record( + arg_mapping.required_span_size(), + Impl::get_property(prop_copy), + Impl::get_property(prop_copy), + has_exec ? std::optional{Impl::get_property< + Impl::ExecutionSpaceTag>(prop_copy)} + : std::optional{std::nullopt}, + std::integral_constant(), + std::integral_constant())); + } + + public: + template + // requires(!Impl::ViewCtorProp::has_pointer) + explicit inline BasicView( + const Impl::ViewCtorProp &arg_prop, + std::enable_if_t::has_pointer, + typename mdspan_type::mapping_type> const &arg_mapping) + : BasicView(create_data_handle(arg_prop, arg_mapping), arg_mapping) {} + + template + // requires(Impl::ViewCtorProp::has_pointer) + KOKKOS_FUNCTION explicit inline BasicView( + const Impl::ViewCtorProp &arg_prop, + std::enable_if_t::has_pointer, + typename mdspan_type::mapping_type> const &arg_mapping) + : BasicView( + data_handle_type(Impl::get_property(arg_prop)), + arg_mapping) {} + + protected: + template + KOKKOS_INLINE_FUNCTION BasicView( + Impl::SubViewCtorTag, + const BasicView &src_view, + SliceSpecifiers... slices) + : BasicView(submdspan( + src_view.to_mdspan(), + Impl::transform_kokkos_slice_to_mdspan_slice(slices)...)) {} + + public: + //---------------------------------------- + // Conversion to MDSpan + template , + mdspan_type>>> + KOKKOS_INLINE_FUNCTION constexpr + operator mdspan() const { + return mdspan_type(m_ptr, m_map, m_acc); + } + + // Here we use an overload instead of a default parameter as a workaround + // to a potential compiler bug with clang 17. It may be present in other + // compilers + template >> + KOKKOS_INLINE_FUNCTION constexpr auto to_mdspan() const { + using ret_mdspan_type = + mdspan; + return ret_mdspan_type( + static_cast( + data_handle()), + mapping(), static_cast(accessor())); + } + + template < + class OtherAccessorType = AccessorPolicy, + typename = std::enable_if_t>> + KOKKOS_INLINE_FUNCTION constexpr auto to_mdspan( + const OtherAccessorType &other_accessor) const { + using ret_mdspan_type = + mdspan; + return ret_mdspan_type( + static_cast( + data_handle()), + mapping(), other_accessor); + } + + KOKKOS_FUNCTION void assign_data(element_type *ptr) { m_ptr = ptr; } + + // ========================= mdspan ================================= + + // [mdspan.mdspan.members], members + +// Introducing the C++20 and C++23 variants of the operators already +#ifndef KOKKOS_ENABLE_CXX17 +#ifndef KOKKOS_ENABLE_CXX20 + // C++23 only operator[] + template + requires((std::is_convertible_v && ...) && + (std::is_nothrow_constructible_v && + ...) && + (sizeof...(OtherIndexTypes) == rank())) + KOKKOS_FUNCTION constexpr reference operator[]( + OtherIndexTypes... indices) const { + return m_acc.access(m_ptr, + m_map(static_cast(std::move(indices))...)); + } + + template + requires( + std::is_convertible_v && + std::is_nothrow_constructible_v) + KOKKOS_FUNCTION constexpr reference operator[]( + const Array &indices) const { + return m_acc.access(m_ptr, + [&](std::index_sequence) { + return m_map(indices[Idxs]...); + }(std::make_index_sequence())); + } + + template + requires( + std::is_convertible_v && + std::is_nothrow_constructible_v) + KOKKOS_FUNCTION constexpr reference operator[]( + std::span indices) const { + return m_acc.access(m_ptr, + [&](std::index_sequence) { + return m_map(indices[Idxs]...); + }(std::make_index_sequence())); + } +#endif + + // C++20 operator() + template + requires((std::is_convertible_v && ...) && + (std::is_nothrow_constructible_v && + ...) && + (sizeof...(OtherIndexTypes) == rank())) + KOKKOS_FUNCTION constexpr reference operator()( + OtherIndexTypes... indices) const { + return m_acc.access(m_ptr, + m_map(static_cast(std::move(indices))...)); + } + + template + requires( + std::is_convertible_v && + std::is_nothrow_constructible_v) + KOKKOS_FUNCTION constexpr reference operator()( + const Array &indices) const { + return m_acc.access(m_ptr, + [&](std::index_sequence) { + return m_map(indices[Idxs]...); + }(std::make_index_sequence())); + } + + template + requires( + std::is_convertible_v && + std::is_nothrow_constructible_v) + KOKKOS_FUNCTION constexpr reference operator()( + std::span indices) const { + return m_acc.access(m_ptr, + [&](std::index_sequence) { + return m_map(indices[Idxs]...); + }(std::make_index_sequence())); + } +#else + // C++17 variant of operator() + + // Some weird unexplained issue in compiling the SFINAE version with CUDA/MSVC + // So we just use post factor check here with static_assert +#if defined(KOKKOS_ENABLE_CUDA) && defined(_WIN32) + template + KOKKOS_FUNCTION constexpr reference operator()( + OtherIndexTypes... indices) const { + static_assert((std::is_convertible_v && ...)); + static_assert( + (std::is_nothrow_constructible_v && ...)); + static_assert((sizeof...(OtherIndexTypes)) == rank()); + return m_acc.access(m_ptr, + m_map(static_cast(std::move(indices))...)); + } +#else + template + KOKKOS_FUNCTION constexpr std::enable_if_t< + ((std::is_convertible_v && ...)) && + ((std::is_nothrow_constructible_v && + ...)) && + ((sizeof...(OtherIndexTypes)) == rank()), + reference> + operator()(OtherIndexTypes... indices) const { + return m_acc.access(m_ptr, + m_map(static_cast(std::move(indices))...)); + } +#endif +#endif + + private: + // FIXME_CXX20: could use inline templated lambda in C++20 mode inside size() + template + KOKKOS_FUNCTION constexpr size_type size_impl( + std::index_sequence) const noexcept { + // Note we restrict data_handle to be convertible to element_type* for now. + // This is also different from mdspan: mdspan can NOT be legally in a state + // where m_ptr is nullptr and the product of extents is non-zero + // The default constructor of mdspan is constrained to dynamic_rank > 0 + // For View we do not have that constraint today + if (data_handle() == nullptr) return 0u; + return ((static_cast(m_map.extents().extent(Idxs))) * ... * + size_type(1)); + } + + public: + KOKKOS_FUNCTION constexpr size_type size() const noexcept { + return size_impl(std::make_index_sequence()); + } + + private: + // FIXME_CXX20: could use inline templated lambda in C++20 mode inside empty() + template + KOKKOS_FUNCTION constexpr bool empty_impl( + std::index_sequence) const noexcept { + // Note we restrict data_handle to be convertible to element_type* for now. + // This is also different from mdspan: mdspan can NOT be legally in a state + // where m_ptr is nullptr and the product of extents is non-zero + // The default constructor of mdspan is constrained to dynamic_rank > 0 + // For View we do not have that constraint today + if (data_handle() == nullptr) return true; + return (rank() > 0) && + ((m_map.extents().extent(Idxs) == index_type(0)) || ... || false); + } + + public: + [[nodiscard]] KOKKOS_FUNCTION constexpr bool empty() const noexcept { + return empty_impl(std::make_index_sequence()); + } + + KOKKOS_FUNCTION friend constexpr void swap(BasicView &x, + BasicView &y) noexcept { + kokkos_swap(x.m_ptr, y.m_ptr); + kokkos_swap(x.m_map, y.m_map); + kokkos_swap(x.m_acc, y.m_acc); + } + + KOKKOS_FUNCTION constexpr const extents_type &extents() const noexcept { + return m_map.extents(); + }; + KOKKOS_FUNCTION constexpr const data_handle_type &data_handle() + const noexcept { + return m_ptr; + }; + KOKKOS_FUNCTION constexpr const mapping_type &mapping() const noexcept { + return m_map; + }; + KOKKOS_FUNCTION constexpr const accessor_type &accessor() const noexcept { + return m_acc; + }; + + KOKKOS_FUNCTION static constexpr bool is_always_unique() noexcept { + return mapping_type::is_always_unique(); + }; + KOKKOS_FUNCTION static constexpr bool is_always_exhaustive() noexcept { + return mapping_type::is_always_exhaustive(); + }; + KOKKOS_FUNCTION static constexpr bool is_always_strided() noexcept { + return mapping_type::is_always_strided(); + }; + + KOKKOS_FUNCTION constexpr bool is_unique() const { + return m_map.is_unique(); + }; + KOKKOS_FUNCTION constexpr bool is_exhaustive() const { + return m_map.is_exhaustive(); + }; + KOKKOS_FUNCTION constexpr bool is_strided() const { + return m_map.is_strided(); + }; + KOKKOS_FUNCTION constexpr index_type stride(rank_type r) const { + return m_map.stride(r); + }; + + protected: +#ifndef __NVCC__ + KOKKOS_IMPL_NO_UNIQUE_ADDRESS data_handle_type m_ptr{}; + KOKKOS_IMPL_NO_UNIQUE_ADDRESS mapping_type m_map{}; + KOKKOS_IMPL_NO_UNIQUE_ADDRESS accessor_type m_acc{}; +#else + data_handle_type m_ptr{}; + mapping_type m_map{}; + accessor_type m_acc{}; +#endif + + template + friend class BasicView; +}; +} // namespace Kokkos::Impl + +#endif diff --git a/lib/kokkos/core/src/View/Kokkos_ViewAlloc.hpp b/lib/kokkos/core/src/View/Kokkos_ViewAlloc.hpp index 1ade75692f..eb11630b21 100644 --- a/lib/kokkos/core/src/View/Kokkos_ViewAlloc.hpp +++ b/lib/kokkos/core/src/View/Kokkos_ViewAlloc.hpp @@ -26,6 +26,7 @@ static_assert(false, #include #include #include +#include #include #include @@ -41,22 +42,8 @@ bool is_zero_byte(const T& x) { return std::memcmp(&x, all_zeroes, sizeof(T)) == 0; } -//---------------------------------------------------------------------------- - -/* - * The construction, assignment to default, and destruction - * are merged into a single functor. - * Primarily to work around an unresolved CUDA back-end bug - * that would lose the destruction cuda device function when - * called from the shared memory tracking destruction. - * Secondarily to have two fewer partial specializations. - */ -template ::value> -struct ViewValueFunctor; - template -struct ViewValueFunctor { +struct ViewValueFunctor { using ExecSpace = typename DeviceType::execution_space; struct DestroyTag {}; @@ -68,20 +55,31 @@ struct ViewValueFunctor { std::string name; bool default_exec_space; - template - KOKKOS_INLINE_FUNCTION - std::enable_if_t::value> - operator()(ConstructTag const&, const size_t i) const { + template + KOKKOS_FUNCTION + std::enable_if_t> + operator()(ConstructTag, const size_t i) const { new (ptr + i) ValueType(); } - KOKKOS_INLINE_FUNCTION void operator()(DestroyTag const&, - const size_t i) const { + KOKKOS_FUNCTION void operator()(DestroyTag, const size_t i) const { + // When instantiating a View on host execution space with a host only + // destructor the workaround for CUDA device symbol instantiation tries to + // still compile a destruction kernel for the device, and issues a warning + // for host from host-device +#ifdef KOKKOS_ENABLE_CUDA + if constexpr (std::is_same_v) { + KOKKOS_IF_ON_DEVICE(((ptr + i)->~ValueType();)) + } else { + KOKKOS_IF_ON_HOST(((ptr + i)->~ValueType();)) + } +#else (ptr + i)->~ValueType(); +#endif } - ViewValueFunctor() = default; - ViewValueFunctor(const ViewValueFunctor&) = default; + ViewValueFunctor() = default; + ViewValueFunctor(const ViewValueFunctor&) = default; ViewValueFunctor& operator=(const ViewValueFunctor&) = default; ViewValueFunctor(ExecSpace const& arg_space, ValueType* const arg_ptr, @@ -104,49 +102,6 @@ struct ViewValueFunctor { functor_instantiate_workaround(); } - template - std::enable_if_t::value && - std::is_trivially_copy_assignable::value> - construct_dispatch() { - ValueType value{}; -// On A64FX memset seems to do the wrong thing with regards to first touch -// leading to the significant performance issues -#ifndef KOKKOS_ARCH_A64FX - if (Impl::is_zero_byte(value)) { - uint64_t kpID = 0; - if (Kokkos::Profiling::profileLibraryLoaded()) { - // We are not really using parallel_for here but using beginParallelFor - // instead of begin_parallel_for (and adding "via memset") is the best - // we can do to indicate that this is not supposed to be tunable (and - // doesn't really execute a parallel_for). - Kokkos::Profiling::beginParallelFor( - "Kokkos::View::initialization [" + name + "] via memset", - Kokkos::Profiling::Experimental::device_id(space), &kpID); - } - (void)ZeroMemset( - space, Kokkos::View>(ptr, n)); - - if (Kokkos::Profiling::profileLibraryLoaded()) { - Kokkos::Profiling::endParallelFor(kpID); - } - if (default_exec_space) - space.fence("Kokkos::Impl::ViewValueFunctor: View init/destroy fence"); - } else { -#endif - parallel_for_implementation(); -#ifndef KOKKOS_ARCH_A64FX - } -#endif - } - - template - std::enable_if_t::value && - std::is_trivially_copy_assignable::value)> - construct_dispatch() { - parallel_for_implementation(); - } - template void parallel_for_implementation() { using PolicyType = @@ -172,24 +127,62 @@ struct ViewValueFunctor { const Kokkos::Impl::ParallelFor closure( *this, policy); closure.execute(); - if (default_exec_space || std::is_same_v) - space.fence("Kokkos::Impl::ViewValueFunctor: View init/destroy fence"); if (Kokkos::Profiling::profileLibraryLoaded()) { Kokkos::Profiling::endParallelFor(kpID); } + if (default_exec_space || std::is_same_v) { + space.fence(std::is_same_v + ? "Kokkos::View::destruction before deallocate" + : "Kokkos::View::initialization"); + } } - void construct_shared_allocation() { construct_dispatch(); } + // Shortcut for zero initialization + void zero_memset_implementation() { + uint64_t kpID = 0; + if (Kokkos::Profiling::profileLibraryLoaded()) { + // We are not really using parallel_for here but using beginParallelFor + // instead of begin_parallel_for (and adding "via memset") is the best + // we can do to indicate that this is not supposed to be tunable (and + // doesn't really execute a parallel_for). + Kokkos::Profiling::beginParallelFor( + "Kokkos::View::initialization [" + name + "] via memset", + Kokkos::Profiling::Experimental::device_id(space), &kpID); + } + + (void)ZeroMemset(space, ptr, n * sizeof(ValueType)); + + if (Kokkos::Profiling::profileLibraryLoaded()) { + Kokkos::Profiling::endParallelFor(kpID); + } + if (default_exec_space) { + space.fence("Kokkos::View::initialization via memset"); + } + } + + void construct_shared_allocation() { +// On A64FX memset seems to do the wrong thing with regards to first touch +// leading to the significant performance issues +#ifndef KOKKOS_ARCH_A64FX + if constexpr (std::is_trivial_v) { + // value-initialization is equivalent to filling with zeros + zero_memset_implementation(); + } else +#endif + parallel_for_implementation(); + } void destroy_shared_allocation() { + if constexpr (std::is_trivially_destructible_v) { + // do nothing, don't bother calling the destructor + } else { #ifdef KOKKOS_ENABLE_IMPL_VIEW_OF_VIEWS_DESTRUCTOR_PRECONDITION_VIOLATION_WORKAROUND - if constexpr (std::is_same_v) - for (size_t i = 0; i < n; ++i) (ptr + i)->~ValueType(); - else + if constexpr (std::is_same_v) + for (size_t i = 0; i < n; ++i) (ptr + i)->~ValueType(); + else #endif - { - parallel_for_implementation(); + parallel_for_implementation(); } } @@ -206,114 +199,6 @@ struct ViewValueFunctor { } }; -template -struct ViewValueFunctor { - using ExecSpace = typename DeviceType::execution_space; - using PolicyType = Kokkos::RangePolicy>; - - ExecSpace space; - ValueType* ptr; - size_t n; - std::string name; - bool default_exec_space; - - KOKKOS_INLINE_FUNCTION - void operator()(const size_t i) const { ptr[i] = ValueType(); } - - ViewValueFunctor() = default; - ViewValueFunctor(const ViewValueFunctor&) = default; - ViewValueFunctor& operator=(const ViewValueFunctor&) = default; - - ViewValueFunctor(ExecSpace const& arg_space, ValueType* const arg_ptr, - size_t const arg_n, std::string arg_name) - : space(arg_space), - ptr(arg_ptr), - n(arg_n), - name(std::move(arg_name)), - default_exec_space(false) {} - - ViewValueFunctor(ValueType* const arg_ptr, size_t const arg_n, - std::string arg_name) - : space(ExecSpace{}), - ptr(arg_ptr), - n(arg_n), - name(std::move(arg_name)), - default_exec_space(true) {} - - template - std::enable_if_t::value && - std::is_trivially_copy_assignable::value> - construct_shared_allocation() { - // Shortcut for zero initialization -// On A64FX memset seems to do the wrong thing with regards to first touch -// leading to the significant performance issues -#ifndef KOKKOS_ARCH_A64FX - ValueType value{}; - if (Impl::is_zero_byte(value)) { - uint64_t kpID = 0; - if (Kokkos::Profiling::profileLibraryLoaded()) { - // We are not really using parallel_for here but using beginParallelFor - // instead of begin_parallel_for (and adding "via memset") is the best - // we can do to indicate that this is not supposed to be tunable (and - // doesn't really execute a parallel_for). - Kokkos::Profiling::beginParallelFor( - "Kokkos::View::initialization [" + name + "] via memset", - Kokkos::Profiling::Experimental::device_id(space), &kpID); - } - - (void)ZeroMemset( - space, Kokkos::View>(ptr, n)); - - if (Kokkos::Profiling::profileLibraryLoaded()) { - Kokkos::Profiling::endParallelFor(kpID); - } - if (default_exec_space) - space.fence("Kokkos::Impl::ViewValueFunctor: View init/destroy fence"); - } else { -#endif - parallel_for_implementation(); -#ifndef KOKKOS_ARCH_A64FX - } -#endif - } - - template - std::enable_if_t::value && - std::is_trivially_copy_assignable::value)> - construct_shared_allocation() { - parallel_for_implementation(); - } - - void parallel_for_implementation() { - PolicyType policy(space, 0, n); - uint64_t kpID = 0; - if (Kokkos::Profiling::profileLibraryLoaded()) { - Kokkos::Profiling::beginParallelFor( - "Kokkos::View::initialization [" + name + "]", - Kokkos::Profiling::Experimental::device_id(space), &kpID); - } -#ifdef KOKKOS_ENABLE_CUDA - if (std::is_same::value) { - Kokkos::Impl::cuda_prefetch_pointer(space, ptr, sizeof(ValueType) * n, - true); - } -#endif - const Kokkos::Impl::ParallelFor closure( - *this, policy); - closure.execute(); - if (default_exec_space) - space.fence( - "Kokkos::Impl::ViewValueFunctor: Fence after setting values in " - "view"); - if (Kokkos::Profiling::profileLibraryLoaded()) { - Kokkos::Profiling::endParallelFor(kpID); - } - } - - void destroy_shared_allocation() {} -}; - template struct ViewValueFunctorSequentialHostInit { using ExecSpace = typename DeviceType::execution_space; @@ -358,6 +243,63 @@ struct ViewValueFunctorSequentialHostInit { } }; +template +Kokkos::Impl::SharedAllocationRecord* make_shared_allocation_record( + const size_t& required_span_size, std::string_view label, + const MemorySpace& memory_space, + const std::optional exec_space, + std::bool_constant, std::bool_constant) { + static_assert(SpaceAccessibility::accessible); + + // Use this for constructing and destroying the view + using device_type = Kokkos::Device; + using functor_type = std::conditional_t< + SequentialInit, + ViewValueFunctorSequentialHostInit, + ViewValueFunctor>; + using record_type = + Kokkos::Impl::SharedAllocationRecord; + + /* Force alignment of allocations on on 8 byte boundaries even for + * element types smaller than 8 bytes */ + static constexpr std::size_t align_mask = 0x7; + + // Calculate the total size of the memory, in bytes, and make sure it is + // byte-aligned + const std::size_t alloc_size = + (required_span_size * sizeof(ElementType) + align_mask) & ~align_mask; + + auto* record = + exec_space + ? record_type::allocate(*exec_space, memory_space, std::string{label}, + alloc_size) + : record_type::allocate(memory_space, std::string{label}, alloc_size); + + auto ptr = static_cast(record->data()); + + auto functor = + exec_space ? functor_type(*exec_space, ptr, required_span_size, + std::string{label}) + : functor_type(ptr, required_span_size, std::string{label}); + + // Only initialize if the allocation is non-zero. + // May be zero if one of the dimensions is zero. + if constexpr (Initialize) { + if (alloc_size) { + // Assume destruction is only required when construction is requested. + // The ViewValueFunctor has both value construction and destruction + // operators. + record->m_destroy = std::move(functor); + + // Construct values + record->m_destroy.construct_shared_allocation(); + } + } + + return record; +} + } // namespace Kokkos::Impl #endif // KOKKOS_VIEW_ALLOC_HPP diff --git a/lib/kokkos/core/src/impl/Kokkos_Atomic_View.hpp b/lib/kokkos/core/src/View/Kokkos_ViewAtomic.hpp similarity index 96% rename from lib/kokkos/core/src/impl/Kokkos_Atomic_View.hpp rename to lib/kokkos/core/src/View/Kokkos_ViewAtomic.hpp index 23d4c2524c..f77066b70f 100644 --- a/lib/kokkos/core/src/impl/Kokkos_Atomic_View.hpp +++ b/lib/kokkos/core/src/View/Kokkos_ViewAtomic.hpp @@ -13,8 +13,8 @@ // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //@HEADER -#ifndef KOKKOS_ATOMIC_VIEW_HPP -#define KOKKOS_ATOMIC_VIEW_HPP +#ifndef KOKKOS_VIEWATOMIC_HPP +#define KOKKOS_VIEWATOMIC_HPP #include #include @@ -44,10 +44,10 @@ class AtomicDataElement { } KOKKOS_INLINE_FUNCTION - void inc() const { Kokkos::atomic_increment(ptr); } + void inc() const { Kokkos::atomic_inc(ptr); } KOKKOS_INLINE_FUNCTION - void dec() const { Kokkos::atomic_decrement(ptr); } + void dec() const { Kokkos::atomic_dec(ptr); } KOKKOS_INLINE_FUNCTION const_value_type operator++() const { @@ -215,7 +215,7 @@ class AtomicViewDataHandle { } KOKKOS_INLINE_FUNCTION - operator typename ViewTraits::value_type*() const { return ptr; } + operator typename ViewTraits::value_type *() const { return ptr; } }; } // namespace Impl diff --git a/lib/kokkos/core/src/impl/Kokkos_ViewCtor.hpp b/lib/kokkos/core/src/View/Kokkos_ViewCtor.hpp similarity index 84% rename from lib/kokkos/core/src/impl/Kokkos_ViewCtor.hpp rename to lib/kokkos/core/src/View/Kokkos_ViewCtor.hpp index 379180ae64..f080474717 100644 --- a/lib/kokkos/core/src/impl/Kokkos_ViewCtor.hpp +++ b/lib/kokkos/core/src/View/Kokkos_ViewCtor.hpp @@ -72,8 +72,8 @@ struct ViewCtorProp {}; */ template struct ViewCtorProp> { - ViewCtorProp() = default; - ViewCtorProp(const ViewCtorProp &) = default; + ViewCtorProp() = default; + ViewCtorProp(const ViewCtorProp &) = default; ViewCtorProp &operator=(const ViewCtorProp &) = default; using type = CommonViewAllocProp; @@ -92,8 +92,8 @@ struct ViewCtorProp || std::is_same_v || std::is_same_v>, P> { - ViewCtorProp() = default; - ViewCtorProp(const ViewCtorProp &) = default; + ViewCtorProp() = default; + ViewCtorProp(const ViewCtorProp &) = default; ViewCtorProp &operator=(const ViewCtorProp &) = default; using type = P; @@ -106,14 +106,14 @@ struct ViewCtorProp || /* Map input label type to std::string */ template struct ViewCtorProp::value>, Label> { - ViewCtorProp() = default; - ViewCtorProp(const ViewCtorProp &) = default; + ViewCtorProp() = default; + ViewCtorProp(const ViewCtorProp &) = default; ViewCtorProp &operator=(const ViewCtorProp &) = default; using type = std::string; ViewCtorProp(const type &arg) : value(arg) {} - ViewCtorProp(type &&arg) : value(arg) {} + ViewCtorProp(type &&arg) : value(std::move(arg)) {} type value; }; @@ -122,8 +122,8 @@ template struct ViewCtorProp::value || Kokkos::is_execution_space::value>, Space> { - ViewCtorProp() = default; - ViewCtorProp(const ViewCtorProp &) = default; + ViewCtorProp() = default; + ViewCtorProp(const ViewCtorProp &) = default; ViewCtorProp &operator=(const ViewCtorProp &) = default; using type = Space; @@ -135,8 +135,8 @@ struct ViewCtorProp::value || template struct ViewCtorProp { - ViewCtorProp() = default; - ViewCtorProp(const ViewCtorProp &) = default; + ViewCtorProp() = default; + ViewCtorProp(const ViewCtorProp &) = default; ViewCtorProp &operator=(const ViewCtorProp &) = default; using type = T *; @@ -213,14 +213,19 @@ struct ViewCtorProp : public ViewCtorProp... { using execution_space = typename var_execution_space::type; using pointer_type = typename var_pointer::type; - /* Copy from a matching argument list. - * Requires std::is_same< P , ViewCtorProp< void , Args >::value ... - */ - template - inline ViewCtorProp(Args const &... args) : ViewCtorProp(args)... {} + // Construct from a matching argument list. + // + // Note that if P is empty, this constructor is the default constructor. + // On the other hand, if P is not empty, the constraint implies that + // there is no default constructor. + template , Args &&>...>>> + ViewCtorProp(Args &&...args) + : ViewCtorProp(std::forward(args))... {} template - KOKKOS_FUNCTION ViewCtorProp(pointer_type arg0, Args const &... args) + KOKKOS_FUNCTION ViewCtorProp(pointer_type arg0, Args const &...args) : ViewCtorProp(arg0), ViewCtorProp::type>(args)... {} @@ -252,7 +257,7 @@ auto with_properties_if_unset(const ViewCtorProp &view_ctor_prop) { template auto with_properties_if_unset(const ViewCtorProp &view_ctor_prop, [[maybe_unused]] const Property &property, - const Properties &... properties) { + const Properties &...properties) { if constexpr ((is_execution_space::value && !ViewCtorProp::has_execution_space) || (is_memory_space::value && @@ -302,7 +307,7 @@ template struct WithPropertiesIfUnset, Property, Properties...> { static constexpr auto apply_prop(const ViewCtorProp &view_ctor_prop, const Property &prop, - const Properties &... properties) { + const Properties &...properties) { if constexpr ((is_execution_space::value && !ViewCtorProp::has_execution_space) || (is_memory_space::value && @@ -328,7 +333,7 @@ struct WithPropertiesIfUnset, Property, Properties...> { template auto with_properties_if_unset(const ViewCtorProp &view_ctor_prop, - const Properties &... properties) { + const Properties &...properties) { return WithPropertiesIfUnset, Properties...>::apply_prop( view_ctor_prop, properties...); } @@ -437,6 +442,48 @@ using ViewAllocateWithoutInitializing = Impl::ViewCtorProp; +inline constexpr Kokkos::Impl::SequentialHostInit_t SequentialHostInit{}; + +inline constexpr Kokkos::Impl::WithoutInitializing_t WithoutInitializing{}; + +inline constexpr Kokkos::Impl::AllowPadding_t AllowPadding{}; + +/** \brief Create View allocation parameter bundle from argument list. + * + * Valid argument list members are: + * 1) label as a "string" or std::string + * 2) memory space instance of the View::memory_space type + * 3) execution space instance compatible with the View::memory_space + * 4) Kokkos::WithoutInitializing to bypass initialization + * 4) Kokkos::AllowPadding to allow allocation to pad dimensions for memory + * alignment + */ +template +auto view_alloc(Args &&...args) { + using return_type = Impl::ViewCtorProp>::type...>; + + static_assert(!return_type::has_pointer, + "Cannot give pointer-to-memory for view allocation"); + + return return_type(std::forward(args)...); +} + +template +KOKKOS_INLINE_FUNCTION + Impl::ViewCtorProp::type...> + view_wrap(Args const &...args) { + using return_type = + Impl::ViewCtorProp::type...>; + + static_assert(!return_type::has_memory_space && + !return_type::has_execution_space && + !return_type::has_label && return_type::has_pointer, + "Must only give pointer-to-memory for view wrapping"); + + return return_type(args...); +} + } /* namespace Kokkos */ //---------------------------------------------------------------------------- diff --git a/lib/kokkos/core/src/impl/Kokkos_ViewDataAnalysis.hpp b/lib/kokkos/core/src/View/Kokkos_ViewDataAnalysis.hpp similarity index 96% rename from lib/kokkos/core/src/impl/Kokkos_ViewDataAnalysis.hpp rename to lib/kokkos/core/src/View/Kokkos_ViewDataAnalysis.hpp index 04c0c9aeed..37b6e2802f 100644 --- a/lib/kokkos/core/src/impl/Kokkos_ViewDataAnalysis.hpp +++ b/lib/kokkos/core/src/View/Kokkos_ViewDataAnalysis.hpp @@ -60,8 +60,8 @@ struct rank_dynamic { static constexpr size_t ArgN##R = (V != KOKKOS_INVALID_INDEX ? V : 1); \ static constexpr size_t N##R = (V != KOKKOS_INVALID_INDEX ? V : 1); \ KOKKOS_INLINE_FUNCTION explicit ViewDimension##R(size_t) {} \ - ViewDimension##R() = default; \ - ViewDimension##R(const ViewDimension##R&) = default; \ + ViewDimension##R() = default; \ + ViewDimension##R(const ViewDimension##R&) = default; \ ViewDimension##R& operator=(const ViewDimension##R&) = default; \ }; \ template \ @@ -72,8 +72,8 @@ struct rank_dynamic { struct ViewDimension##R<0u, RD> { \ static constexpr size_t ArgN##R = 0; \ std::conditional_t<(RD < 3), size_t, unsigned> N##R; \ - ViewDimension##R() = default; \ - ViewDimension##R(const ViewDimension##R&) = default; \ + ViewDimension##R() = default; \ + ViewDimension##R(const ViewDimension##R&) = default; \ ViewDimension##R& operator=(const ViewDimension##R&) = default; \ KOKKOS_INLINE_FUNCTION explicit ViewDimension##R(size_t V) : N##R(V) {} \ }; \ @@ -149,8 +149,8 @@ struct KOKKOS_IMPL_ENFORCE_EMPTY_BASE_OPTIMIZATION ViewDimension static constexpr unsigned rank = sizeof...(Vals); static constexpr unsigned rank_dynamic = Impl::rank_dynamic::value; - ViewDimension() = default; - ViewDimension(const ViewDimension&) = default; + ViewDimension() = default; + ViewDimension(const ViewDimension&) = default; ViewDimension& operator=(const ViewDimension&) = default; KOKKOS_INLINE_FUNCTION @@ -370,8 +370,7 @@ struct ViewDataAnalysis { // ValueType is opportunity for partial specialization. // Must match array analysis when this default template is used. static_assert( - std::is_same::value); + std::is_same_v); public: using specialize = void; // No specialization diff --git a/lib/kokkos/core/src/View/Kokkos_ViewLegacy.hpp b/lib/kokkos/core/src/View/Kokkos_ViewLegacy.hpp new file mode 100644 index 0000000000..fd406d58cc --- /dev/null +++ b/lib/kokkos/core/src/View/Kokkos_ViewLegacy.hpp @@ -0,0 +1,1604 @@ +//@HEADER +// ************************************************************************ +// +// Kokkos v. 4.0 +// Copyright (2022) National Technology & Engineering +// Solutions of Sandia, LLC (NTESS). +// +// Under the terms of Contract DE-NA0003525 with NTESS, +// the U.S. Government retains certain rights in this software. +// +// Part of Kokkos, under the Apache License v2.0 with LLVM Exceptions. +// See https://kokkos.org/LICENSE for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//@HEADER + +#ifndef KOKKOS_IMPL_PUBLIC_INCLUDE +#include +static_assert(false, + "Including non-public Kokkos header files is not allowed."); +#endif +#ifndef KOKKOS_VIEWLEGACY_HPP +#define KOKKOS_VIEWLEGACY_HPP + +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +#include +#include + +#ifdef KOKKOS_ENABLE_IMPL_MDSPAN +#include +#include +#include +#endif +#include + +#include + +//---------------------------------------------------------------------------- +//---------------------------------------------------------------------------- + +namespace Kokkos { + +/** \class View + * \brief View to an array of data. + * + * A View represents an array of one or more dimensions. + * For details, please refer to Kokkos' tutorial materials. + * + * \section Kokkos_View_TemplateParameters Template parameters + * + * This class has both required and optional template parameters. The + * \c DataType parameter must always be provided, and must always be + * first. The parameters \c Arg1Type, \c Arg2Type, and \c Arg3Type are + * placeholders for different template parameters. The default value + * of the fifth template parameter \c Specialize suffices for most use + * cases. When explaining the template parameters, we won't refer to + * \c Arg1Type, \c Arg2Type, and \c Arg3Type; instead, we will refer + * to the valid categories of template parameters, in whatever order + * they may occur. + * + * Valid ways in which template arguments may be specified: + * - View< DataType > + * - View< DataType , Layout > + * - View< DataType , Layout , Space > + * - View< DataType , Layout , Space , MemoryTraits > + * - View< DataType , Space > + * - View< DataType , Space , MemoryTraits > + * - View< DataType , MemoryTraits > + * + * \tparam DataType (required) This indicates both the type of each + * entry of the array, and the combination of compile-time and + * run-time array dimension(s). For example, double* + * indicates a one-dimensional array of \c double with run-time + * dimension, and int*[3] a two-dimensional array of \c int + * with run-time first dimension and compile-time second dimension + * (of 3). In general, the run-time dimensions (if any) must go + * first, followed by zero or more compile-time dimensions. For + * more examples, please refer to the tutorial materials. + * + * \tparam Space (required) The memory space. + * + * \tparam Layout (optional) The array's layout in memory. For + * example, LayoutLeft indicates a column-major (Fortran style) + * layout, and LayoutRight a row-major (C style) layout. If not + * specified, this defaults to the preferred layout for the + * Space. + * + * \tparam MemoryTraits (optional) Assertion of the user's intended + * access behavior. For example, RandomAccess indicates read-only + * access with limited spatial locality, and Unmanaged lets users + * wrap externally allocated memory in a View without automatic + * deallocation. + * + * \section Kokkos_View_MT MemoryTraits discussion + * + * \subsection Kokkos_View_MT_Interp MemoryTraits interpretation depends on + * Space + * + * Some \c MemoryTraits options may have different interpretations for + * different \c Space types. For example, with the Cuda device, + * \c RandomAccess tells Kokkos to fetch the data through the texture + * cache, whereas the non-GPU devices have no such hardware construct. + * + * \subsection Kokkos_View_MT_PrefUse Preferred use of MemoryTraits + * + * Users should defer applying the optional \c MemoryTraits parameter + * until the point at which they actually plan to rely on it in a + * computational kernel. This minimizes the number of template + * parameters exposed in their code, which reduces the cost of + * compilation. Users may always assign a View without specified + * \c MemoryTraits to a compatible View with that specification. + * For example: + * \code + * // Pass in the simplest types of View possible. + * void + * doSomething (View out, + * View in) + * { + * // Assign the "generic" View in to a RandomAccess View in_rr. + * // Note that RandomAccess View objects must have const data. + * View in_rr = in; + * // ... do something with in_rr and out ... + * } + * \endcode + */ + +} // namespace Kokkos + +namespace Kokkos { + +template +struct is_always_assignable_impl; + +template +struct is_always_assignable_impl, + Kokkos::View> { + using mapping_type = Kokkos::Impl::ViewMapping< + typename Kokkos::View::traits, + typename Kokkos::View::traits, + typename Kokkos::View::traits::specialize>; + + constexpr static bool value = + mapping_type::is_assignable && + static_cast(Kokkos::View::rank_dynamic) >= + static_cast(Kokkos::View::rank_dynamic); +}; + +template +using is_always_assignable = is_always_assignable_impl< + std::remove_reference_t, + std::remove_const_t>>; + +template +inline constexpr bool is_always_assignable_v = + is_always_assignable::value; + +template +constexpr bool is_assignable(const Kokkos::View& dst, + const Kokkos::View& src) { + using DstTraits = typename Kokkos::View::traits; + using SrcTraits = typename Kokkos::View::traits; + using mapping_type = + Kokkos::Impl::ViewMapping; + + return is_always_assignable_v, + Kokkos::View> || + (mapping_type::is_assignable && + ((DstTraits::dimension::rank_dynamic >= 1) || + (dst.static_extent(0) == src.extent(0))) && + ((DstTraits::dimension::rank_dynamic >= 2) || + (dst.static_extent(1) == src.extent(1))) && + ((DstTraits::dimension::rank_dynamic >= 3) || + (dst.static_extent(2) == src.extent(2))) && + ((DstTraits::dimension::rank_dynamic >= 4) || + (dst.static_extent(3) == src.extent(3))) && + ((DstTraits::dimension::rank_dynamic >= 5) || + (dst.static_extent(4) == src.extent(4))) && + ((DstTraits::dimension::rank_dynamic >= 6) || + (dst.static_extent(5) == src.extent(5))) && + ((DstTraits::dimension::rank_dynamic >= 7) || + (dst.static_extent(6) == src.extent(6))) && + ((DstTraits::dimension::rank_dynamic >= 8) || + (dst.static_extent(7) == src.extent(7)))); +} + +} /* namespace Kokkos */ + +//---------------------------------------------------------------------------- +//---------------------------------------------------------------------------- + +#include + +//---------------------------------------------------------------------------- +//---------------------------------------------------------------------------- + +namespace Kokkos { + +template +class View; + +template +struct is_view : public std::false_type {}; + +template +struct is_view> : public std::true_type {}; + +template +struct is_view> : public std::true_type {}; + +template +inline constexpr bool is_view_v = is_view::value; + +template +class View : public ViewTraits { + private: + template + friend class View; + template + friend class Kokkos::Impl::ViewMapping; + + using view_tracker_type = Kokkos::Impl::ViewTracker; + + public: + using traits = ViewTraits; + + private: + using map_type = + Kokkos::Impl::ViewMapping; + template + friend struct Kokkos::Impl::ViewTracker; + using hooks_policy = typename traits::hooks_policy; + + view_tracker_type m_track; + map_type m_map; + + public: + //---------------------------------------- + /** \brief Compatible view of array of scalar types */ + using array_type = + View; + + /** \brief Compatible view of const data type */ + using const_type = + View; + + /** \brief Compatible view of non-const data type */ + using non_const_type = + View; + + /** \brief Compatible host mirror view */ + using host_mirror_type = + View, + typename traits::hooks_policy>; + + /** \brief Compatible host mirror view */ + using HostMirror = host_mirror_type; + + /** \brief Unified types */ + using uniform_type = typename Impl::ViewUniformType::type; + using uniform_const_type = + typename Impl::ViewUniformType::const_type; + using uniform_runtime_type = + typename Impl::ViewUniformType::runtime_type; + using uniform_runtime_const_type = + typename Impl::ViewUniformType::runtime_const_type; + using uniform_nomemspace_type = + typename Impl::ViewUniformType::nomemspace_type; + using uniform_const_nomemspace_type = + typename Impl::ViewUniformType::const_nomemspace_type; + using uniform_runtime_nomemspace_type = + typename Impl::ViewUniformType::runtime_nomemspace_type; + using uniform_runtime_const_nomemspace_type = + typename Impl::ViewUniformType::runtime_const_nomemspace_type; + + using reference_type = typename map_type::reference_type; + using pointer_type = typename map_type::pointer_type; + + // Typedefs from mdspan + // using extents_type -> not applicable + // Defining layout_type here made MSVC+CUDA fail + // using layout_type = typename traits::array_layout; + // using accessor_type -> not applicable + // using mapping_type -> not applicable + using element_type = typename traits::value_type; + // using value_type -> conflicts with traits::value_type + using index_type = typename traits::memory_space::size_type; + // using size_type -> already from traits::size_type; where it is + // memory_space::size_type + using rank_type = size_t; + using data_handle_type = pointer_type; + using reference = reference_type; + + //---------------------------------------- + // Domain rank and extents + + static constexpr Impl::integral_constant + rank = {}; + static constexpr Impl::integral_constant + rank_dynamic = {}; +#ifdef KOKKOS_ENABLE_DEPRECATED_CODE_4 + enum {Rank KOKKOS_DEPRECATED_WITH_COMMENT("Use rank instead.") = + map_type::Rank}; +#endif + + template + KOKKOS_INLINE_FUNCTION constexpr std::enable_if_t, + size_t> + extent(const iType& r) const noexcept { + return m_map.extent(r); + } + + static KOKKOS_INLINE_FUNCTION constexpr size_t static_extent( + const unsigned r) noexcept { + return map_type::static_extent(r); + } + + template + KOKKOS_INLINE_FUNCTION constexpr std::enable_if_t, + int> + extent_int(const iType& r) const noexcept { + return static_cast(m_map.extent(r)); + } + + KOKKOS_INLINE_FUNCTION constexpr typename traits::array_layout layout() + const { + return m_map.layout(); + } + + //---------------------------------------- + /* Deprecate all 'dimension' functions in favor of + * ISO/C++ vocabulary 'extent'. + */ + + KOKKOS_INLINE_FUNCTION constexpr size_t size() const { + return m_map.dimension_0() * m_map.dimension_1() * m_map.dimension_2() * + m_map.dimension_3() * m_map.dimension_4() * m_map.dimension_5() * + m_map.dimension_6() * m_map.dimension_7(); + } + + KOKKOS_INLINE_FUNCTION constexpr size_t stride_0() const { + return m_map.stride_0(); + } + KOKKOS_INLINE_FUNCTION constexpr size_t stride_1() const { + return m_map.stride_1(); + } + KOKKOS_INLINE_FUNCTION constexpr size_t stride_2() const { + return m_map.stride_2(); + } + KOKKOS_INLINE_FUNCTION constexpr size_t stride_3() const { + return m_map.stride_3(); + } + KOKKOS_INLINE_FUNCTION constexpr size_t stride_4() const { + return m_map.stride_4(); + } + KOKKOS_INLINE_FUNCTION constexpr size_t stride_5() const { + return m_map.stride_5(); + } + KOKKOS_INLINE_FUNCTION constexpr size_t stride_6() const { + return m_map.stride_6(); + } + KOKKOS_INLINE_FUNCTION constexpr size_t stride_7() const { + return m_map.stride_7(); + } + + template + KOKKOS_INLINE_FUNCTION constexpr std::enable_if_t, + size_t> + stride(iType r) const { + return ( + r == 0 + ? m_map.stride_0() + : (r == 1 + ? m_map.stride_1() + : (r == 2 + ? m_map.stride_2() + : (r == 3 + ? m_map.stride_3() + : (r == 4 + ? m_map.stride_4() + : (r == 5 + ? m_map.stride_5() + : (r == 6 + ? m_map.stride_6() + : m_map.stride_7()))))))); + } + + template + KOKKOS_INLINE_FUNCTION void stride(iType* const s) const { + m_map.stride(s); + } + + //---------------------------------------- + // Range span is the span which contains all members. + + enum { + reference_type_is_lvalue_reference = + std::is_lvalue_reference_v + }; + + KOKKOS_INLINE_FUNCTION constexpr size_t span() const { return m_map.span(); } + KOKKOS_INLINE_FUNCTION bool span_is_contiguous() const { + return m_map.span_is_contiguous(); + } + KOKKOS_INLINE_FUNCTION constexpr bool is_allocated() const { + return m_map.data() != nullptr; + } + KOKKOS_INLINE_FUNCTION constexpr pointer_type data() const { + return m_map.data(); + } + + //---------------------------------------- + // Allow specializations to query their specialized map + + KOKKOS_INLINE_FUNCTION + const Kokkos::Impl::ViewMapping& + impl_map() const { + return m_map; + } + KOKKOS_INLINE_FUNCTION + const Kokkos::Impl::SharedAllocationTracker& impl_track() const { + return m_track.m_tracker; + } + //---------------------------------------- + + private: + static constexpr bool is_layout_left = + std::is_same_v; + + static constexpr bool is_layout_right = + std::is_same_v; + + static constexpr bool is_layout_stride = + std::is_same_v; + + static constexpr bool is_default_map = + std::is_void_v && + (is_layout_left || is_layout_right || is_layout_stride); + +#if defined(KOKKOS_ENABLE_DEBUG_BOUNDS_CHECK) + +#define KOKKOS_IMPL_VIEW_OPERATOR_VERIFY(...) \ + Kokkos::Impl::runtime_check_memory_access_violation< \ + typename traits::memory_space>( \ + "Kokkos::View ERROR: attempt to access inaccessible memory space", \ + __VA_ARGS__); \ + Kokkos::Impl::view_verify_operator_bounds( \ + __VA_ARGS__); + +#else + +#define KOKKOS_IMPL_VIEW_OPERATOR_VERIFY(...) \ + Kokkos::Impl::runtime_check_memory_access_violation< \ + typename traits::memory_space>( \ + "Kokkos::View ERROR: attempt to access inaccessible memory space", \ + __VA_ARGS__); + +#endif + + template + static KOKKOS_FUNCTION void check_access_member_function_valid_args(Is...) { + static_assert(rank <= sizeof...(Is)); + static_assert(sizeof...(Is) <= 8); + static_assert(Kokkos::Impl::are_integral::value); + } + + template + static KOKKOS_FUNCTION void check_operator_parens_valid_args(Is...) { + static_assert(rank == sizeof...(Is)); + static_assert(Kokkos::Impl::are_integral::value); + } + + public: + //------------------------------ + // Rank 1 default map operator() + + template + KOKKOS_FORCEINLINE_FUNCTION + std::enable_if_t<(Kokkos::Impl::always_true::value && // + (1 == rank) && is_default_map && !is_layout_stride), + reference_type> + operator()(I0 i0) const { + check_operator_parens_valid_args(i0); + KOKKOS_IMPL_VIEW_OPERATOR_VERIFY(m_track, m_map, i0) + return m_map.m_impl_handle[i0]; + } + + template + KOKKOS_FORCEINLINE_FUNCTION + std::enable_if_t<(Kokkos::Impl::always_true::value && // + (1 == rank) && is_default_map && is_layout_stride), + reference_type> + operator()(I0 i0) const { + check_operator_parens_valid_args(i0); + KOKKOS_IMPL_VIEW_OPERATOR_VERIFY(m_track, m_map, i0) + return m_map.m_impl_handle[m_map.m_impl_offset.m_stride.S0 * i0]; + } + + //------------------------------ + // Rank 1 operator[] + + template + KOKKOS_FORCEINLINE_FUNCTION std::enable_if_t< + ((1 == rank) && Kokkos::Impl::are_integral::value && !is_default_map), + reference_type> + operator[](I0 i0) const { + KOKKOS_IMPL_VIEW_OPERATOR_VERIFY(m_track, m_map, i0) + return m_map.reference(i0); + } + + template + KOKKOS_FORCEINLINE_FUNCTION + std::enable_if_t<((1 == rank) && Kokkos::Impl::are_integral::value && + is_default_map && !is_layout_stride), + reference_type> + operator[](I0 i0) const { + KOKKOS_IMPL_VIEW_OPERATOR_VERIFY(m_track, m_map, i0) + return m_map.m_impl_handle[i0]; + } + + template + KOKKOS_FORCEINLINE_FUNCTION + std::enable_if_t<((1 == rank) && Kokkos::Impl::are_integral::value && + is_default_map && is_layout_stride), + reference_type> + operator[](I0 i0) const { + KOKKOS_IMPL_VIEW_OPERATOR_VERIFY(m_track, m_map, i0) + return m_map.m_impl_handle[m_map.m_impl_offset.m_stride.S0 * i0]; + } + + //------------------------------ + // Rank 2 default map operator() + + template + KOKKOS_FORCEINLINE_FUNCTION std::enable_if_t< + (Kokkos::Impl::always_true::value && // + (2 == rank) && is_default_map && + (is_layout_left || is_layout_right || is_layout_stride)), + reference_type> + operator()(I0 i0, I1 i1) const { + check_operator_parens_valid_args(i0, i1); + KOKKOS_IMPL_VIEW_OPERATOR_VERIFY(m_track, m_map, i0, i1) + if constexpr (is_layout_left) { + if constexpr (rank_dynamic == 0) + return m_map.m_impl_handle[i0 + m_map.m_impl_offset.m_dim.N0 * i1]; + else + return m_map.m_impl_handle[i0 + m_map.m_impl_offset.m_stride * i1]; + } else if constexpr (is_layout_right) { + if constexpr (rank_dynamic == 0) + return m_map.m_impl_handle[i1 + m_map.m_impl_offset.m_dim.N1 * i0]; + else + return m_map.m_impl_handle[i1 + m_map.m_impl_offset.m_stride * i0]; + } else { + static_assert(is_layout_stride); + return m_map.m_impl_handle[i0 * m_map.m_impl_offset.m_stride.S0 + + i1 * m_map.m_impl_offset.m_stride.S1]; + } +#if defined KOKKOS_COMPILER_INTEL + __builtin_unreachable(); +#endif + } + + // Rank 0 -> 8 operator() except for rank-1 and rank-2 with default map which + // have "inlined" versions above + + template + KOKKOS_FORCEINLINE_FUNCTION std::enable_if_t< + (Kokkos::Impl::always_true::value && // + (2 != rank) && (1 != rank) && (0 != rank) && is_default_map), + reference_type> + operator()(Is... indices) const { + check_operator_parens_valid_args(indices...); + KOKKOS_IMPL_VIEW_OPERATOR_VERIFY(m_track, m_map, indices...) + return m_map.m_impl_handle[m_map.m_impl_offset(indices...)]; + } + + template + KOKKOS_FORCEINLINE_FUNCTION + std::enable_if_t<(Kokkos::Impl::always_true::value && // + ((0 == rank) || !is_default_map)), + reference_type> + operator()(Is... indices) const { + check_operator_parens_valid_args(indices...); + KOKKOS_IMPL_VIEW_OPERATOR_VERIFY(m_track, m_map, indices...) + return m_map.reference(indices...); + } + + //------------------------------ + // Rank 0 + + template + KOKKOS_FORCEINLINE_FUNCTION std::enable_if_t< + (Kokkos::Impl::always_true::value && (0 == rank)), reference_type> + access(Is... extra) const { + check_access_member_function_valid_args(extra...); + KOKKOS_IMPL_VIEW_OPERATOR_VERIFY(m_track, m_map, extra...) + return m_map.reference(); + } + + //------------------------------ + // Rank 1 + + template + KOKKOS_FORCEINLINE_FUNCTION + std::enable_if_t<(Kokkos::Impl::always_true::value && + (1 == rank) && !is_default_map), + reference_type> + access(I0 i0, Is... extra) const { + check_access_member_function_valid_args(i0, extra...); + KOKKOS_IMPL_VIEW_OPERATOR_VERIFY(m_track, m_map, i0, extra...) + return m_map.reference(i0); + } + + template + KOKKOS_FORCEINLINE_FUNCTION + std::enable_if_t<(Kokkos::Impl::always_true::value && + (1 == rank) && is_default_map && !is_layout_stride), + reference_type> + access(I0 i0, Is... extra) const { + check_access_member_function_valid_args(i0, extra...); + KOKKOS_IMPL_VIEW_OPERATOR_VERIFY(m_track, m_map, i0, extra...) + return m_map.m_impl_handle[i0]; + } + + template + KOKKOS_FORCEINLINE_FUNCTION + std::enable_if_t<(Kokkos::Impl::always_true::value && + (1 == rank) && is_default_map && is_layout_stride), + reference_type> + access(I0 i0, Is... extra) const { + check_access_member_function_valid_args(i0, extra...); + KOKKOS_IMPL_VIEW_OPERATOR_VERIFY(m_track, m_map, i0, extra...) + return m_map.m_impl_handle[m_map.m_impl_offset.m_stride.S0 * i0]; + } + + //------------------------------ + // Rank 2 + + template + KOKKOS_FORCEINLINE_FUNCTION + std::enable_if_t<(Kokkos::Impl::always_true::value && + (2 == rank) && !is_default_map), + reference_type> + access(I0 i0, I1 i1, Is... extra) const { + check_access_member_function_valid_args(i0, i1, extra...); + KOKKOS_IMPL_VIEW_OPERATOR_VERIFY(m_track, m_map, i0, i1, extra...) + return m_map.reference(i0, i1); + } + + template + KOKKOS_FORCEINLINE_FUNCTION std::enable_if_t< + (Kokkos::Impl::always_true::value && (2 == rank) && + is_default_map && + (is_layout_left || is_layout_right || is_layout_stride)), + reference_type> + access(I0 i0, I1 i1, Is... extra) const { + check_access_member_function_valid_args(i0, i1, extra...); + KOKKOS_IMPL_VIEW_OPERATOR_VERIFY(m_track, m_map, i0, i1, extra...) + if constexpr (is_layout_left) { + if constexpr (rank_dynamic == 0) + return m_map.m_impl_handle[i0 + m_map.m_impl_offset.m_dim.N0 * i1]; + else + return m_map.m_impl_handle[i0 + m_map.m_impl_offset.m_stride * i1]; + } else if constexpr (is_layout_right) { + if constexpr (rank_dynamic == 0) + return m_map.m_impl_handle[i1 + m_map.m_impl_offset.m_dim.N1 * i0]; + else + return m_map.m_impl_handle[i1 + m_map.m_impl_offset.m_stride * i0]; + } else { + static_assert(is_layout_stride); + return m_map.m_impl_handle[i0 * m_map.m_impl_offset.m_stride.S0 + + i1 * m_map.m_impl_offset.m_stride.S1]; + } +#if defined KOKKOS_COMPILER_INTEL + __builtin_unreachable(); +#endif + } + + //------------------------------ + // Rank 3 + + template + KOKKOS_FORCEINLINE_FUNCTION + std::enable_if_t<(Kokkos::Impl::always_true::value && + (3 == rank) && is_default_map), + reference_type> + access(I0 i0, I1 i1, I2 i2, Is... extra) const { + check_access_member_function_valid_args(i0, i1, i2, extra...); + KOKKOS_IMPL_VIEW_OPERATOR_VERIFY(m_track, m_map, i0, i1, i2, extra...) + return m_map.m_impl_handle[m_map.m_impl_offset(i0, i1, i2)]; + } + + template + KOKKOS_FORCEINLINE_FUNCTION + std::enable_if_t<(Kokkos::Impl::always_true::value && + (3 == rank) && !is_default_map), + reference_type> + access(I0 i0, I1 i1, I2 i2, Is... extra) const { + check_access_member_function_valid_args(i0, i1, i2, extra...); + KOKKOS_IMPL_VIEW_OPERATOR_VERIFY(m_track, m_map, i0, i1, i2, extra...) + return m_map.reference(i0, i1, i2); + } + + //------------------------------ + // Rank 4 + + template + KOKKOS_FORCEINLINE_FUNCTION std::enable_if_t< + (Kokkos::Impl::always_true::value && (4 == rank) && + is_default_map), + reference_type> + access(I0 i0, I1 i1, I2 i2, I3 i3, Is... extra) const { + check_access_member_function_valid_args(i0, i1, i2, i3, extra...); + KOKKOS_IMPL_VIEW_OPERATOR_VERIFY(m_track, m_map, i0, i1, i2, i3, extra...) + return m_map.m_impl_handle[m_map.m_impl_offset(i0, i1, i2, i3)]; + } + + template + KOKKOS_FORCEINLINE_FUNCTION std::enable_if_t< + (Kokkos::Impl::always_true::value && (4 == rank) && + !is_default_map), + reference_type> + access(I0 i0, I1 i1, I2 i2, I3 i3, Is... extra) const { + check_access_member_function_valid_args(i0, i1, i2, i3, extra...); + KOKKOS_IMPL_VIEW_OPERATOR_VERIFY(m_track, m_map, i0, i1, i2, i3, extra...) + return m_map.reference(i0, i1, i2, i3); + } + + //------------------------------ + // Rank 5 + + template + KOKKOS_FORCEINLINE_FUNCTION std::enable_if_t< + (Kokkos::Impl::always_true::value && + (5 == rank) && is_default_map), + reference_type> + access(I0 i0, I1 i1, I2 i2, I3 i3, I4 i4, Is... extra) const { + check_access_member_function_valid_args(i0, i1, i2, i3, i4, extra...); + KOKKOS_IMPL_VIEW_OPERATOR_VERIFY(m_track, m_map, i0, i1, i2, i3, i4, + extra...) + return m_map.m_impl_handle[m_map.m_impl_offset(i0, i1, i2, i3, i4)]; + } + + template + KOKKOS_FORCEINLINE_FUNCTION std::enable_if_t< + (Kokkos::Impl::always_true::value && + (5 == rank) && !is_default_map), + reference_type> + access(I0 i0, I1 i1, I2 i2, I3 i3, I4 i4, Is... extra) const { + check_access_member_function_valid_args(i0, i1, i2, i3, i4, extra...); + KOKKOS_IMPL_VIEW_OPERATOR_VERIFY(m_track, m_map, i0, i1, i2, i3, i4, + extra...) + return m_map.reference(i0, i1, i2, i3, i4); + } + + //------------------------------ + // Rank 6 + + template + KOKKOS_FORCEINLINE_FUNCTION std::enable_if_t< + (Kokkos::Impl::always_true::value && + (6 == rank) && is_default_map), + reference_type> + access(I0 i0, I1 i1, I2 i2, I3 i3, I4 i4, I5 i5, Is... extra) const { + check_access_member_function_valid_args(i0, i1, i2, i3, i4, i5, extra...); + KOKKOS_IMPL_VIEW_OPERATOR_VERIFY(m_track, m_map, i0, i1, i2, i3, i4, i5, + extra...) + return m_map.m_impl_handle[m_map.m_impl_offset(i0, i1, i2, i3, i4, i5)]; + } + + template + KOKKOS_FORCEINLINE_FUNCTION std::enable_if_t< + (Kokkos::Impl::always_true::value && + (6 == rank) && !is_default_map), + reference_type> + access(I0 i0, I1 i1, I2 i2, I3 i3, I4 i4, I5 i5, Is... extra) const { + check_access_member_function_valid_args(i0, i1, i2, i3, i4, i5, extra...); + KOKKOS_IMPL_VIEW_OPERATOR_VERIFY(m_track, m_map, i0, i1, i2, i3, i4, i5, + extra...) + return m_map.reference(i0, i1, i2, i3, i4, i5); + } + + //------------------------------ + // Rank 7 + + template + KOKKOS_FORCEINLINE_FUNCTION std::enable_if_t< + (Kokkos::Impl::always_true::value && + (7 == rank) && is_default_map), + reference_type> + access(I0 i0, I1 i1, I2 i2, I3 i3, I4 i4, I5 i5, I6 i6, Is... extra) const { + check_access_member_function_valid_args(i0, i1, i2, i3, i4, i5, i6, + extra...); + KOKKOS_IMPL_VIEW_OPERATOR_VERIFY(m_track, m_map, i0, i1, i2, i3, i4, i5, i6, + extra...) + return m_map.m_impl_handle[m_map.m_impl_offset(i0, i1, i2, i3, i4, i5, i6)]; + } + + template + KOKKOS_FORCEINLINE_FUNCTION std::enable_if_t< + (Kokkos::Impl::always_true::value && + (7 == rank) && !is_default_map), + reference_type> + access(I0 i0, I1 i1, I2 i2, I3 i3, I4 i4, I5 i5, I6 i6, Is... extra) const { + check_access_member_function_valid_args(i0, i1, i2, i3, i4, i5, i6, + extra...); + KOKKOS_IMPL_VIEW_OPERATOR_VERIFY(m_track, m_map, i0, i1, i2, i3, i4, i5, i6, + extra...) + return m_map.reference(i0, i1, i2, i3, i4, i5, i6); + } + + //------------------------------ + // Rank 8 + + template + KOKKOS_FORCEINLINE_FUNCTION + std::enable_if_t<(Kokkos::Impl::always_true::value && + (8 == rank) && is_default_map), + reference_type> + access(I0 i0, I1 i1, I2 i2, I3 i3, I4 i4, I5 i5, I6 i6, I7 i7, + Is... extra) const { + check_access_member_function_valid_args(i0, i1, i2, i3, i4, i5, i6, i7, + extra...); + KOKKOS_IMPL_VIEW_OPERATOR_VERIFY(m_track, m_map, i0, i1, i2, i3, i4, i5, i6, + i7, extra...) + return m_map + .m_impl_handle[m_map.m_impl_offset(i0, i1, i2, i3, i4, i5, i6, i7)]; + } + + template + KOKKOS_FORCEINLINE_FUNCTION + std::enable_if_t<(Kokkos::Impl::always_true::value && + (8 == rank) && !is_default_map), + reference_type> + access(I0 i0, I1 i1, I2 i2, I3 i3, I4 i4, I5 i5, I6 i6, I7 i7, + Is... extra) const { + check_access_member_function_valid_args(i0, i1, i2, i3, i4, i5, i6, i7, + extra...); + KOKKOS_IMPL_VIEW_OPERATOR_VERIFY(m_track, m_map, i0, i1, i2, i3, i4, i5, i6, + i7, extra...) + return m_map.reference(i0, i1, i2, i3, i4, i5, i6, i7); + } + +#undef KOKKOS_IMPL_VIEW_OPERATOR_VERIFY + + //---------------------------------------- + // Standard destructor, constructors, and assignment operators + + KOKKOS_DEFAULTED_FUNCTION + ~View() = default; + + KOKKOS_DEFAULTED_FUNCTION + View() = default; + + KOKKOS_FUNCTION + View(const View& other) : m_track(other.m_track), m_map(other.m_map) { + KOKKOS_IF_ON_HOST((hooks_policy::copy_construct(*this, other);)) + } + + KOKKOS_FUNCTION + View(View&& other) + : m_track{std::move(other.m_track)}, m_map{std::move(other.m_map)} { + KOKKOS_IF_ON_HOST((hooks_policy::move_construct(*this, other);)) + } + + KOKKOS_FUNCTION + View& operator=(const View& other) { + m_map = other.m_map; + m_track = other.m_track; + + KOKKOS_IF_ON_HOST((hooks_policy::copy_assign(*this, other);)) + + return *this; + } + + KOKKOS_FUNCTION + View& operator=(View&& other) { + m_map = std::move(other.m_map); + m_track = std::move(other.m_track); + + KOKKOS_IF_ON_HOST((hooks_policy::move_assign(*this, other);)) + + return *this; + } + + //---------------------------------------- + // Compatible view copy constructor and assignment + // may assign unmanaged from managed. + + template + KOKKOS_INLINE_FUNCTION View( + const View& rhs, + std::enable_if_t::traits, + typename traits::specialize>::is_assignable_data_type>* = nullptr) + : m_track(rhs), m_map() { + using SrcTraits = typename View::traits; + using Mapping = Kokkos::Impl::ViewMapping; + static_assert(Mapping::is_assignable, + "Incompatible View copy construction"); + Mapping::assign(m_map, rhs.m_map, rhs.m_track.m_tracker); + } + + template + KOKKOS_INLINE_FUNCTION std::enable_if_t< + Kokkos::Impl::ViewMapping< + traits, typename View::traits, + typename traits::specialize>::is_assignable_data_type, + View>& + operator=(const View& rhs) { + using SrcTraits = typename View::traits; + using Mapping = Kokkos::Impl::ViewMapping; + static_assert(Mapping::is_assignable, "Incompatible View copy assignment"); + Mapping::assign(m_map, rhs.m_map, rhs.m_track.m_tracker); + m_track.assign(rhs); + return *this; + } + + //---------------------------------------- + // Compatible subview constructor + // may assign unmanaged from managed. + + template + KOKKOS_INLINE_FUNCTION View(const View& src_view, const Arg0 arg0, + Args... args) + : m_track(src_view), m_map() { + using SrcType = View; + + using Mapping = Kokkos::Impl::ViewMapping; + + using DstType = typename Mapping::type; + + static_assert( + Kokkos::Impl::ViewMapping::is_assignable, + "Subview construction requires compatible view and subview arguments"); + + Mapping::assign(m_map, src_view.m_map, arg0, args...); + } + + //---------------------------------------- + // Allocation tracking properties + + KOKKOS_INLINE_FUNCTION + int use_count() const { return m_track.m_tracker.use_count(); } + + inline const std::string label() const { + return m_track.m_tracker + .template get_label(); + } + + public: + //---------------------------------------- + // Allocation according to allocation properties and array layout + + template + explicit inline View( + const Impl::ViewCtorProp& arg_prop, + std::enable_if_t::has_pointer, + typename traits::array_layout> const& arg_layout) + : m_track(), m_map() { + // Copy the input allocation properties with possibly defaulted properties + // We need to split it in two to avoid MSVC compiler errors + auto prop_copy_tmp = + Impl::with_properties_if_unset(arg_prop, std::string{}); + auto prop_copy = Impl::with_properties_if_unset( + prop_copy_tmp, typename traits::device_type::memory_space{}, + typename traits::device_type::execution_space{}); + using alloc_prop = decltype(prop_copy); + + static_assert(traits::is_managed, + "View allocation constructor requires managed memory"); + + if (alloc_prop::initialize && + !alloc_prop::execution_space::impl_is_initialized()) { + // If initializing view data then + // the execution space must be initialized. + Kokkos::Impl::throw_runtime_exception( + "Constructing View and initializing data with uninitialized " + "execution space"); + } + +#ifdef KOKKOS_ENABLE_DEBUG_BOUNDS_CHECK + if constexpr (std::is_same_v || + std::is_same_v || + std::is_same_v) { + size_t i0 = arg_layout.dimension[0]; + size_t i1 = arg_layout.dimension[1]; + size_t i2 = arg_layout.dimension[2]; + size_t i3 = arg_layout.dimension[3]; + size_t i4 = arg_layout.dimension[4]; + size_t i5 = arg_layout.dimension[5]; + size_t i6 = arg_layout.dimension[6]; + size_t i7 = arg_layout.dimension[7]; + + const std::string& alloc_name = + Impl::get_property(prop_copy); + Impl::runtime_check_rank( + *this, std::is_same::value, i0, i1, + i2, i3, i4, i5, i6, i7, alloc_name.c_str()); + } +#endif + + Kokkos::Impl::SharedAllocationRecord<>* record = m_map.allocate_shared( + prop_copy, arg_layout, Impl::ViewCtorProp::has_execution_space); + + // Setup and initialization complete, start tracking + m_track.m_tracker.assign_allocated_record_to_uninitialized(record); + } + + KOKKOS_INLINE_FUNCTION + void assign_data(pointer_type arg_data) { + m_track.m_tracker.clear(); + m_map.assign_data(arg_data); + } + + // Wrap memory according to properties and array layout + template + explicit KOKKOS_INLINE_FUNCTION View( + const Impl::ViewCtorProp& arg_prop, + std::enable_if_t::has_pointer, + typename traits::array_layout> const& arg_layout) + : m_track() // No memory tracking + , + m_map(arg_prop, arg_layout) { + static_assert( + std::is_same::pointer_type>::value, + "Constructing View to wrap user memory must supply matching pointer " + "type"); + +#ifdef KOKKOS_ENABLE_DEBUG_BOUNDS_CHECK + if constexpr (std::is_same_v || + std::is_same_v || + std::is_same_v) { + size_t i0 = arg_layout.dimension[0]; + size_t i1 = arg_layout.dimension[1]; + size_t i2 = arg_layout.dimension[2]; + size_t i3 = arg_layout.dimension[3]; + size_t i4 = arg_layout.dimension[4]; + size_t i5 = arg_layout.dimension[5]; + size_t i6 = arg_layout.dimension[6]; + size_t i7 = arg_layout.dimension[7]; + + Impl::runtime_check_rank( + *this, std::is_same::value, i0, i1, + i2, i3, i4, i5, i6, i7, "UNMANAGED"); + } +#endif + } + + // Simple dimension-only layout + template + explicit inline View( + const Impl::ViewCtorProp& arg_prop, + std::enable_if_t::has_pointer, size_t> const + arg_N0 = KOKKOS_IMPL_CTOR_DEFAULT_ARG, + const size_t arg_N1 = KOKKOS_IMPL_CTOR_DEFAULT_ARG, + const size_t arg_N2 = KOKKOS_IMPL_CTOR_DEFAULT_ARG, + const size_t arg_N3 = KOKKOS_IMPL_CTOR_DEFAULT_ARG, + const size_t arg_N4 = KOKKOS_IMPL_CTOR_DEFAULT_ARG, + const size_t arg_N5 = KOKKOS_IMPL_CTOR_DEFAULT_ARG, + const size_t arg_N6 = KOKKOS_IMPL_CTOR_DEFAULT_ARG, + const size_t arg_N7 = KOKKOS_IMPL_CTOR_DEFAULT_ARG) + : View(arg_prop, + typename traits::array_layout(arg_N0, arg_N1, arg_N2, arg_N3, + arg_N4, arg_N5, arg_N6, arg_N7)) { + static_assert(traits::array_layout::is_extent_constructible, + "Layout is not constructible from extent arguments. Use " + "overload taking a layout object instead."); + } + + template + explicit KOKKOS_INLINE_FUNCTION View( + const Impl::ViewCtorProp& arg_prop, + std::enable_if_t::has_pointer, size_t> const + arg_N0 = KOKKOS_IMPL_CTOR_DEFAULT_ARG, + const size_t arg_N1 = KOKKOS_IMPL_CTOR_DEFAULT_ARG, + const size_t arg_N2 = KOKKOS_IMPL_CTOR_DEFAULT_ARG, + const size_t arg_N3 = KOKKOS_IMPL_CTOR_DEFAULT_ARG, + const size_t arg_N4 = KOKKOS_IMPL_CTOR_DEFAULT_ARG, + const size_t arg_N5 = KOKKOS_IMPL_CTOR_DEFAULT_ARG, + const size_t arg_N6 = KOKKOS_IMPL_CTOR_DEFAULT_ARG, + const size_t arg_N7 = KOKKOS_IMPL_CTOR_DEFAULT_ARG) + : View(arg_prop, + typename traits::array_layout(arg_N0, arg_N1, arg_N2, arg_N3, + arg_N4, arg_N5, arg_N6, arg_N7)) { + static_assert(traits::array_layout::is_extent_constructible, + "Layout is not constructible from extent arguments. Use " + "overload taking a layout object instead."); + } + + // Allocate with label and layout + template + explicit inline View( + const Label& arg_label, + std::enable_if_t::value, + typename traits::array_layout> const& arg_layout) + : View(Impl::ViewCtorProp(arg_label), arg_layout) {} + + // Allocate label and layout, must disambiguate from subview constructor. + template + explicit inline View( + const Label& arg_label, + std::enable_if_t::value, const size_t> + arg_N0 = KOKKOS_IMPL_CTOR_DEFAULT_ARG, + const size_t arg_N1 = KOKKOS_IMPL_CTOR_DEFAULT_ARG, + const size_t arg_N2 = KOKKOS_IMPL_CTOR_DEFAULT_ARG, + const size_t arg_N3 = KOKKOS_IMPL_CTOR_DEFAULT_ARG, + const size_t arg_N4 = KOKKOS_IMPL_CTOR_DEFAULT_ARG, + const size_t arg_N5 = KOKKOS_IMPL_CTOR_DEFAULT_ARG, + const size_t arg_N6 = KOKKOS_IMPL_CTOR_DEFAULT_ARG, + const size_t arg_N7 = KOKKOS_IMPL_CTOR_DEFAULT_ARG) + : View(Impl::ViewCtorProp(arg_label), + typename traits::array_layout(arg_N0, arg_N1, arg_N2, arg_N3, + arg_N4, arg_N5, arg_N6, arg_N7)) { + static_assert(traits::array_layout::is_extent_constructible, + "Layout is not constructible from extent arguments. Use " + "overload taking a layout object instead."); + } + + // Construct view from ViewTracker and map + // This should be the preferred method because future extensions may need to + // use the ViewTracker class. + template + KOKKOS_INLINE_FUNCTION View( + const view_tracker_type& track, + const Kokkos::Impl::ViewMapping& map) + : m_track(track), m_map() { + using Mapping = + Kokkos::Impl::ViewMapping; + static_assert(Mapping::is_assignable, + "Incompatible View copy construction"); + Mapping::assign(m_map, map, track.m_tracker); + } + + // Construct View from internal shared allocation tracker object and map + // This is here for backwards compatibility for classes that derive from + // Kokkos::View + template + KOKKOS_INLINE_FUNCTION View( + const typename view_tracker_type::track_type& track, + const Kokkos::Impl::ViewMapping& map) + : m_track(track), m_map() { + using Mapping = + Kokkos::Impl::ViewMapping; + static_assert(Mapping::is_assignable, + "Incompatible View copy construction"); + Mapping::assign(m_map, map, track); + } + + //---------------------------------------- + // Memory span required to wrap these dimensions. + static constexpr size_t required_allocation_size( + typename traits::array_layout const& layout) { + return map_type::memory_span(layout); + } + + static constexpr size_t required_allocation_size( + const size_t arg_N0 = 0, const size_t arg_N1 = 0, const size_t arg_N2 = 0, + const size_t arg_N3 = 0, const size_t arg_N4 = 0, const size_t arg_N5 = 0, + const size_t arg_N6 = 0, const size_t arg_N7 = 0) { + static_assert(traits::array_layout::is_extent_constructible, + "Layout is not constructible from extent arguments. Use " + "overload taking a layout object instead."); + return map_type::memory_span(typename traits::array_layout( + arg_N0, arg_N1, arg_N2, arg_N3, arg_N4, arg_N5, arg_N6, arg_N7)); + } + + explicit KOKKOS_INLINE_FUNCTION View( + pointer_type arg_ptr, const size_t arg_N0 = KOKKOS_IMPL_CTOR_DEFAULT_ARG, + const size_t arg_N1 = KOKKOS_IMPL_CTOR_DEFAULT_ARG, + const size_t arg_N2 = KOKKOS_IMPL_CTOR_DEFAULT_ARG, + const size_t arg_N3 = KOKKOS_IMPL_CTOR_DEFAULT_ARG, + const size_t arg_N4 = KOKKOS_IMPL_CTOR_DEFAULT_ARG, + const size_t arg_N5 = KOKKOS_IMPL_CTOR_DEFAULT_ARG, + const size_t arg_N6 = KOKKOS_IMPL_CTOR_DEFAULT_ARG, + const size_t arg_N7 = KOKKOS_IMPL_CTOR_DEFAULT_ARG) + : View(Impl::ViewCtorProp(arg_ptr), + typename traits::array_layout(arg_N0, arg_N1, arg_N2, arg_N3, + arg_N4, arg_N5, arg_N6, arg_N7)) { + static_assert(traits::array_layout::is_extent_constructible, + "Layout is not constructible from extent arguments. Use " + "overload taking a layout object instead."); + } + + explicit KOKKOS_INLINE_FUNCTION View( + pointer_type arg_ptr, const typename traits::array_layout& arg_layout) + : View(Impl::ViewCtorProp(arg_ptr), arg_layout) {} + + //---------------------------------------- + // Shared scratch memory constructor + + static KOKKOS_INLINE_FUNCTION size_t + shmem_size(const size_t arg_N0 = KOKKOS_INVALID_INDEX, + const size_t arg_N1 = KOKKOS_INVALID_INDEX, + const size_t arg_N2 = KOKKOS_INVALID_INDEX, + const size_t arg_N3 = KOKKOS_INVALID_INDEX, + const size_t arg_N4 = KOKKOS_INVALID_INDEX, + const size_t arg_N5 = KOKKOS_INVALID_INDEX, + const size_t arg_N6 = KOKKOS_INVALID_INDEX, + const size_t arg_N7 = KOKKOS_INVALID_INDEX) { + static_assert(traits::array_layout::is_extent_constructible, + "Layout is not constructible from extent arguments. Use " + "overload taking a layout object instead."); + const size_t num_passed_args = Impl::count_valid_integers( + arg_N0, arg_N1, arg_N2, arg_N3, arg_N4, arg_N5, arg_N6, arg_N7); + + if (std::is_void_v && + num_passed_args != rank_dynamic) { + Kokkos::abort( + "Kokkos::View::shmem_size() rank_dynamic != number of arguments.\n"); + } + + return View::shmem_size(typename traits::array_layout( + arg_N0, arg_N1, arg_N2, arg_N3, arg_N4, arg_N5, arg_N6, arg_N7)); + } + + private: + // Want to be able to align to minimum scratch alignment or sizeof or alignof + // elements + static constexpr size_t scratch_value_alignment = + max({sizeof(typename traits::value_type), + alignof(typename traits::value_type), + static_cast( + traits::execution_space::scratch_memory_space::ALIGN)}); + + public: + static KOKKOS_INLINE_FUNCTION size_t + shmem_size(typename traits::array_layout const& arg_layout) { + return map_type::memory_span(arg_layout) + scratch_value_alignment; + } + + explicit KOKKOS_INLINE_FUNCTION View( + const typename traits::execution_space::scratch_memory_space& arg_space, + const typename traits::array_layout& arg_layout) + : View(Impl::ViewCtorProp(reinterpret_cast( + arg_space.get_shmem_aligned(map_type::memory_span(arg_layout), + scratch_value_alignment))), + arg_layout) {} + + explicit KOKKOS_INLINE_FUNCTION View( + const typename traits::execution_space::scratch_memory_space& arg_space, + const size_t arg_N0 = KOKKOS_IMPL_CTOR_DEFAULT_ARG, + const size_t arg_N1 = KOKKOS_IMPL_CTOR_DEFAULT_ARG, + const size_t arg_N2 = KOKKOS_IMPL_CTOR_DEFAULT_ARG, + const size_t arg_N3 = KOKKOS_IMPL_CTOR_DEFAULT_ARG, + const size_t arg_N4 = KOKKOS_IMPL_CTOR_DEFAULT_ARG, + const size_t arg_N5 = KOKKOS_IMPL_CTOR_DEFAULT_ARG, + const size_t arg_N6 = KOKKOS_IMPL_CTOR_DEFAULT_ARG, + const size_t arg_N7 = KOKKOS_IMPL_CTOR_DEFAULT_ARG) + : View(Impl::ViewCtorProp( + reinterpret_cast(arg_space.get_shmem_aligned( + map_type::memory_span(typename traits::array_layout( + arg_N0, arg_N1, arg_N2, arg_N3, arg_N4, arg_N5, arg_N6, + arg_N7)), + scratch_value_alignment))), + typename traits::array_layout(arg_N0, arg_N1, arg_N2, arg_N3, + arg_N4, arg_N5, arg_N6, arg_N7)) { + static_assert(traits::array_layout::is_extent_constructible, + "Layout is not constructible from extent arguments. Use " + "overload taking a layout object instead."); + } + + //---------------------------------------- + // MDSpan converting constructors +#ifdef KOKKOS_ENABLE_IMPL_MDSPAN + template ::mdspan_type> + KOKKOS_INLINE_FUNCTION +#ifndef KOKKOS_ENABLE_CXX17 + explicit(traits::is_managed) +#endif + View(const typename Impl::MDSpanViewTraits::mdspan_type& mds, + std::enable_if_t< + !std::is_same_v>* = + nullptr) + : View(mds.data_handle(), + Impl::array_layout_from_mapping< + typename traits::array_layout, + typename Impl::MDSpanViewTraits::mdspan_type>( + mds.mapping())) { + } + + template + KOKKOS_INLINE_FUNCTION +#ifndef KOKKOS_ENABLE_CXX17 + explicit(!std::is_convertible_v< + Kokkos::mdspan, + typename Impl::MDSpanViewTraits::mdspan_type>) +#endif + View(const Kokkos::mdspan& mds) + : View(typename Impl::MDSpanViewTraits::mdspan_type(mds)) { + } + + //---------------------------------------- + // Conversion to MDSpan + template ::mdspan_type, + typename = std::enable_if_t, + std::false_type, + std::is_assignable, + ImplNaturalMDSpanType>>::value>> + KOKKOS_INLINE_FUNCTION constexpr operator mdspan< + OtherElementType, OtherExtents, OtherLayoutPolicy, OtherAccessor>() { + using mdspan_type = typename Impl::MDSpanViewTraits::mdspan_type; + return mdspan_type{data(), + Impl::mapping_from_view_mapping(m_map)}; + } + + template >, + typename = std::enable_if_t>> + KOKKOS_INLINE_FUNCTION constexpr auto to_mdspan( + const OtherAccessorType& other_accessor = + typename Impl::MDSpanViewTraits::accessor_type()) { + using mdspan_type = typename Impl::MDSpanViewTraits::mdspan_type; + using ret_mdspan_type = + mdspan; + return ret_mdspan_type{data(), + Impl::mapping_from_view_mapping(m_map), + other_accessor}; + } +#endif // KOKKOS_ENABLE_IMPL_MDSPAN +}; + +template +KOKKOS_INLINE_FUNCTION constexpr unsigned rank(const View&) { + return View::rank(); +} + +namespace Impl { + +template +struct RankDataType { + using type = typename RankDataType::type*; +}; + +template +struct RankDataType { + using type = ValueType; +}; + +template +KOKKOS_FUNCTION std::enable_if_t< + N == View::rank() && + std::is_same_v::specialize, void>, + View> +as_view_of_rank_n(View v) { + return v; +} + +// Placeholder implementation to compile generic code for DynRankView; should +// never be called +template +KOKKOS_FUNCTION std::enable_if_t< + N != View::rank() && + std::is_same_v::specialize, void>, + View::value_type, N>::type, + Args...>> +as_view_of_rank_n(View) { + Kokkos::abort("Trying to get at a View of the wrong rank"); + return {}; +} + +template +void apply_to_view_of_static_rank(Function&& f, View a) { + f(a); +} + +} // namespace Impl + +template +KOKKOS_INLINE_FUNCTION auto subview(const View& src, Args... args) { + static_assert(View::rank == sizeof...(Args), + "subview requires one argument for each source View rank"); + + return typename Kokkos::Impl::ViewMapping< + void /* deduce subview type from source view traits */ + , + typename Impl::RemoveAlignedMemoryTrait::type, + Args...>::type(src, args...); +} + +#ifdef KOKKOS_ENABLE_DEPRECATED_CODE_4 +template +KOKKOS_DEPRECATED KOKKOS_INLINE_FUNCTION auto subview(const View& src, + Args... args) { + static_assert(View::rank == sizeof...(Args), + "subview requires one argument for each source View rank"); + static_assert(Kokkos::is_memory_traits::value); + + return typename Kokkos::Impl::ViewMapping< + void /* deduce subview type from source view traits */ + , + typename Impl::RemoveAlignedMemoryTrait::type, + Args...>::type(src, args...); +} +#endif + +template +using Subview = decltype(subview(std::declval(), std::declval()...)); + +} /* namespace Kokkos */ + +//---------------------------------------------------------------------------- +//---------------------------------------------------------------------------- + +namespace Kokkos { + +template +KOKKOS_INLINE_FUNCTION bool operator==(const View& lhs, + const View& rhs) { + // Same data, layout, dimensions + using lhs_traits = ViewTraits; + using rhs_traits = ViewTraits; + + return std::is_same_v && + std::is_same_v && + std::is_same_v && + View::rank() == View::rank() && + lhs.data() == rhs.data() && lhs.span() == rhs.span() && + lhs.extent(0) == rhs.extent(0) && lhs.extent(1) == rhs.extent(1) && + lhs.extent(2) == rhs.extent(2) && lhs.extent(3) == rhs.extent(3) && + lhs.extent(4) == rhs.extent(4) && lhs.extent(5) == rhs.extent(5) && + lhs.extent(6) == rhs.extent(6) && lhs.extent(7) == rhs.extent(7); +} + +template +KOKKOS_INLINE_FUNCTION bool operator!=(const View& lhs, + const View& rhs) { + return !(operator==(lhs, rhs)); +} + +} /* namespace Kokkos */ + +//---------------------------------------------------------------------------- +//---------------------------------------------------------------------------- + +namespace Kokkos { +namespace Impl { + +template +struct CommonViewValueType; + +template +struct CommonViewValueType { + using value_type = std::common_type_t; +}; + +template +struct CommonViewAllocProp; + +template +struct CommonViewAllocProp { + using value_type = ValueType; + using scalar_array_type = ValueType; + + template + KOKKOS_INLINE_FUNCTION CommonViewAllocProp(const Views&...) {} +}; + +template +struct DeduceCommonViewAllocProp; + +// Base case must provide types for: +// 1. specialize 2. value_type 3. is_view 4. prop_type +template +struct DeduceCommonViewAllocProp { + using specialize = typename FirstView::traits::specialize; + + using value_type = typename FirstView::traits::value_type; + + enum : bool { is_view = is_view::value }; + + using prop_type = CommonViewAllocProp; +}; + +template +struct DeduceCommonViewAllocProp { + using NextTraits = DeduceCommonViewAllocProp; + + using first_specialize = typename FirstView::traits::specialize; + using first_value_type = typename FirstView::traits::value_type; + + enum : bool { first_is_view = is_view::value }; + + using next_specialize = typename NextTraits::specialize; + using next_value_type = typename NextTraits::value_type; + + enum : bool { next_is_view = NextTraits::is_view }; + + // common types + + // determine specialize type + // if first and next specialize differ, but are not the same specialize, error + // out + static_assert(!(!std::is_same_v && + !std::is_void_v && + !std::is_void_v), + "Kokkos DeduceCommonViewAllocProp ERROR: Only one non-void " + "specialize trait allowed"); + + // otherwise choose non-void specialize if either/both are non-void + using specialize = + std::conditional_t, + first_specialize, + std::conditional_t<(std::is_void_v && + !std::is_void_v), + next_specialize, first_specialize>>; + + using value_type = typename CommonViewValueType::value_type; + + enum : bool { is_view = (first_is_view && next_is_view) }; + + using prop_type = CommonViewAllocProp; +}; + +} // end namespace Impl + +template +using DeducedCommonPropsType = + typename Impl::DeduceCommonViewAllocProp::prop_type; + +// This function is required in certain scenarios where users customize +// Kokkos View internals. One example are dynamic length embedded ensemble +// types. The function is used to propagate necessary information +// (like the ensemble size) when creating new views. +// However, most of the time it is called with a single view. +// Furthermore, the propagated information is not just for view allocations. +// From what I can tell, the type of functionality provided by +// common_view_alloc_prop is the equivalent of propagating accessors in mdspan, +// a mechanism we will eventually use to replace this clunky approach here, when +// we are finally mdspan based. +// TODO: get rid of this when we have mdspan +template +KOKKOS_INLINE_FUNCTION DeducedCommonPropsType common_view_alloc_prop( + Views const&... views) { + return DeducedCommonPropsType(views...); +} + +} // namespace Kokkos + +#include +#include + +//---------------------------------------------------------------------------- +//---------------------------------------------------------------------------- + +#endif /* #ifndef KOKKOS_VIEWLEGACY_HPP */ diff --git a/lib/kokkos/core/src/impl/Kokkos_ViewMapping.hpp b/lib/kokkos/core/src/View/Kokkos_ViewMapping.hpp similarity index 90% rename from lib/kokkos/core/src/impl/Kokkos_ViewMapping.hpp rename to lib/kokkos/core/src/View/Kokkos_ViewMapping.hpp index 10aaa63b7c..ecc19eaf5e 100644 --- a/lib/kokkos/core/src/impl/Kokkos_ViewMapping.hpp +++ b/lib/kokkos/core/src/View/Kokkos_ViewMapping.hpp @@ -28,61 +28,41 @@ #include #include #include -#include -#include -#include +#include +#include +#include +#include #include #include #include -#include +#include #include //---------------------------------------------------------------------------- //---------------------------------------------------------------------------- -namespace Kokkos { - -struct ALL_t { - KOKKOS_INLINE_FUNCTION - constexpr const ALL_t& operator()() const { return *this; } - - KOKKOS_INLINE_FUNCTION - constexpr bool operator==(const ALL_t&) const { return true; } -}; - -#ifdef KOKKOS_ENABLE_DEPRECATED_CODE_4 -namespace Impl { -// TODO This alias declaration forces us to fully qualify ALL_t inside the -// Kokkos::Impl namespace to avoid deprecation warnings. Replace the -// fully-qualified name when we remove Kokkos::Impl::ALL_t. -using ALL_t KOKKOS_DEPRECATED_WITH_COMMENT("Use Kokkos::ALL_t instead!") = - Kokkos::ALL_t; -} // namespace Impl -#endif -} // namespace Kokkos - namespace Kokkos { namespace Impl { template struct is_integral_extent_type { - enum : bool { value = std::is_same::value ? 1 : 0 }; + enum : bool { value = std::is_same_v ? 1 : 0 }; }; template struct is_integral_extent_type> { - enum : bool { value = std::is_integral::value ? 1 : 0 }; + enum : bool { value = std::is_integral_v ? 1 : 0 }; }; template struct is_integral_extent_type> { - enum : bool { value = std::is_integral::value ? 1 : 0 }; + enum : bool { value = std::is_integral_v ? 1 : 0 }; }; // Assuming '2 == initializer_list::size()' template struct is_integral_extent_type> { - enum : bool { value = std::is_integral::value ? 1 : 0 }; + enum : bool { value = std::is_integral_v ? 1 : 0 }; }; template @@ -93,8 +73,7 @@ struct is_integral_extent { enum : bool { value = is_integral_extent_type::value }; - static_assert(value || std::is_integral::value || - std::is_void::value, + static_assert(value || std::is_integral_v || std::is_void_v, "subview argument must be either integral or integral extent"); }; @@ -112,16 +91,16 @@ struct SubviewLegalArgsCompileTime { enum { - value = (((CurrentArg == RankDest - 1) && - (Kokkos::Impl::is_integral_extent_type::value)) || - ((CurrentArg >= RankDest) && (std::is_integral::value)) || - ((CurrentArg < RankDest) && - (std::is_same::value)) || - ((CurrentArg == 0) && - (Kokkos::Impl::is_integral_extent_type::value))) && - (SubviewLegalArgsCompileTime::value) + value = + (((CurrentArg == RankDest - 1) && + (Kokkos::Impl::is_integral_extent_type::value)) || + ((CurrentArg >= RankDest) && (std::is_integral_v)) || + ((CurrentArg < RankDest) && (std::is_same_v)) || + ((CurrentArg == 0) && + (Kokkos::Impl::is_integral_extent_type::value))) && + (SubviewLegalArgsCompileTime::value) }; }; @@ -129,7 +108,7 @@ template struct SubviewLegalArgsCompileTime { enum { - value = ((CurrentArg == RankDest - 1) || (std::is_integral::value)) && + value = ((CurrentArg == RankDest - 1) || (std::is_integral_v)) && (CurrentArg == RankSrc - 1) }; }; @@ -144,10 +123,9 @@ struct SubviewLegalArgsCompileTime::value)) || - ((CurrentArg < RankSrc - RankDest) && - (std::is_integral::value)) || + ((CurrentArg < RankSrc - RankDest) && (std::is_integral_v)) || ((CurrentArg >= RankSrc - RankDest) && - (std::is_same::value))) && + (std::is_same_v))) && (SubviewLegalArgsCompileTime::value) @@ -158,8 +136,8 @@ template struct SubviewLegalArgsCompileTime { enum { - value = ((CurrentArg == RankSrc - 1) && - (std::is_same::value)) + value = + ((CurrentArg == RankSrc - 1) && (std::is_same_v)) }; }; @@ -392,7 +370,7 @@ struct SubviewExtents { const int n = snprintf(buffer, LEN, "Kokkos::subview bounds error ("); error(buffer + n, LEN - n, 0, 0, dim, args...); - Kokkos::Impl::throw_runtime_exception(std::string(buffer));)) + Kokkos::abort(buffer);)) KOKKOS_IF_ON_DEVICE(((void)dim; Kokkos::abort("Kokkos::subview bounds error"); @@ -718,8 +696,8 @@ struct ViewOffset< return *this; } #else - ViewOffset() = default; - ViewOffset(const ViewOffset&) = default; + ViewOffset() = default; + ViewOffset(const ViewOffset&) = default; ViewOffset& operator=(const ViewOffset&) = default; #endif @@ -885,14 +863,17 @@ struct ViewOffset< KOKKOS_INLINE_FUNCTION constexpr array_layout layout() const { constexpr auto r = dimension_type::rank; - return array_layout((r > 0 ? m_dim.N0 : KOKKOS_INVALID_INDEX), - (r > 1 ? m_dim.N1 : KOKKOS_INVALID_INDEX), - (r > 2 ? m_dim.N2 : KOKKOS_INVALID_INDEX), - (r > 3 ? m_dim.N3 : KOKKOS_INVALID_INDEX), - (r > 4 ? m_dim.N4 : KOKKOS_INVALID_INDEX), - (r > 5 ? m_dim.N5 : KOKKOS_INVALID_INDEX), - (r > 6 ? m_dim.N6 : KOKKOS_INVALID_INDEX), - (r > 7 ? m_dim.N7 : KOKKOS_INVALID_INDEX)); + array_layout l((r > 0 ? m_dim.N0 : KOKKOS_INVALID_INDEX), + (r > 1 ? m_dim.N1 : KOKKOS_INVALID_INDEX), + (r > 2 ? m_dim.N2 : KOKKOS_INVALID_INDEX), + (r > 3 ? m_dim.N3 : KOKKOS_INVALID_INDEX), + (r > 4 ? m_dim.N4 : KOKKOS_INVALID_INDEX), + (r > 5 ? m_dim.N5 : KOKKOS_INVALID_INDEX), + (r > 6 ? m_dim.N6 : KOKKOS_INVALID_INDEX), + (r > 7 ? m_dim.N7 : KOKKOS_INVALID_INDEX)); + // Without span_is_contiguous Sacado hidden dimensions get messed up + l.stride = span_is_contiguous() ? KOKKOS_IMPL_CTOR_DEFAULT_ARG : m_stride; + return l; } KOKKOS_INLINE_FUNCTION constexpr size_type dimension_0() const { @@ -1071,8 +1052,8 @@ struct ViewOffset< } #else - ViewOffset() = default; - ViewOffset(const ViewOffset&) = default; + ViewOffset() = default; + ViewOffset(const ViewOffset&) = default; ViewOffset& operator=(const ViewOffset&) = default; #endif @@ -1086,7 +1067,11 @@ struct ViewOffset< arg_layout.dimension[2], arg_layout.dimension[3], arg_layout.dimension[4], arg_layout.dimension[5], arg_layout.dimension[6], arg_layout.dimension[7]), - m_stride(Padding::stride(arg_layout.dimension[0])) {} + m_stride( + arg_layout.stride != KOKKOS_IMPL_CTOR_DEFAULT_ARG + ? arg_layout.stride + : Padding::stride(arg_layout.dimension[0])) { + } template KOKKOS_INLINE_FUNCTION constexpr ViewOffset( @@ -1407,8 +1392,8 @@ struct ViewOffset< } #else - ViewOffset() = default; - ViewOffset(const ViewOffset&) = default; + ViewOffset() = default; + ViewOffset(const ViewOffset&) = default; ViewOffset& operator=(const ViewOffset&) = default; #endif @@ -1565,14 +1550,17 @@ struct ViewOffset< KOKKOS_INLINE_FUNCTION constexpr array_layout layout() const { constexpr auto r = dimension_type::rank; - return array_layout((r > 0 ? m_dim.N0 : KOKKOS_INVALID_INDEX), - (r > 1 ? m_dim.N1 : KOKKOS_INVALID_INDEX), - (r > 2 ? m_dim.N2 : KOKKOS_INVALID_INDEX), - (r > 3 ? m_dim.N3 : KOKKOS_INVALID_INDEX), - (r > 4 ? m_dim.N4 : KOKKOS_INVALID_INDEX), - (r > 5 ? m_dim.N5 : KOKKOS_INVALID_INDEX), - (r > 6 ? m_dim.N6 : KOKKOS_INVALID_INDEX), - (r > 7 ? m_dim.N7 : KOKKOS_INVALID_INDEX)); + array_layout l((r > 0 ? m_dim.N0 : KOKKOS_INVALID_INDEX), + (r > 1 ? m_dim.N1 : KOKKOS_INVALID_INDEX), + (r > 2 ? m_dim.N2 : KOKKOS_INVALID_INDEX), + (r > 3 ? m_dim.N3 : KOKKOS_INVALID_INDEX), + (r > 4 ? m_dim.N4 : KOKKOS_INVALID_INDEX), + (r > 5 ? m_dim.N5 : KOKKOS_INVALID_INDEX), + (r > 6 ? m_dim.N6 : KOKKOS_INVALID_INDEX), + (r > 7 ? m_dim.N7 : KOKKOS_INVALID_INDEX)); + // Without span_is_contiguous Sacado hidden dimensions get messed up + l.stride = span_is_contiguous() ? KOKKOS_IMPL_CTOR_DEFAULT_ARG : m_stride; + return l; } KOKKOS_INLINE_FUNCTION constexpr size_type dimension_0() const { @@ -1614,8 +1602,8 @@ struct ViewOffset< } KOKKOS_INLINE_FUNCTION constexpr bool span_is_contiguous() const { - return m_stride == m_dim.N7 * m_dim.N6 * m_dim.N5 * m_dim.N4 * m_dim.N3 * - m_dim.N2 * m_dim.N1; + return m_stride == static_cast(m_dim.N7) * m_dim.N6 * m_dim.N5 * + m_dim.N4 * m_dim.N3 * m_dim.N2 * m_dim.N1; } /* Strides of dimensions */ @@ -1624,19 +1612,21 @@ struct ViewOffset< return m_dim.N7; } KOKKOS_INLINE_FUNCTION constexpr size_type stride_5() const { - return m_dim.N7 * m_dim.N6; + return static_cast(m_dim.N7) * m_dim.N6; } KOKKOS_INLINE_FUNCTION constexpr size_type stride_4() const { - return m_dim.N7 * m_dim.N6 * m_dim.N5; + return static_cast(m_dim.N7) * m_dim.N6 * m_dim.N5; } KOKKOS_INLINE_FUNCTION constexpr size_type stride_3() const { - return m_dim.N7 * m_dim.N6 * m_dim.N5 * m_dim.N4; + return static_cast(m_dim.N7) * m_dim.N6 * m_dim.N5 * m_dim.N4; } KOKKOS_INLINE_FUNCTION constexpr size_type stride_2() const { - return m_dim.N7 * m_dim.N6 * m_dim.N5 * m_dim.N4 * m_dim.N3; + return static_cast(m_dim.N7) * m_dim.N6 * m_dim.N5 * m_dim.N4 * + m_dim.N3; } KOKKOS_INLINE_FUNCTION constexpr size_type stride_1() const { - return m_dim.N7 * m_dim.N6 * m_dim.N5 * m_dim.N4 * m_dim.N3 * m_dim.N2; + return static_cast(m_dim.N7) * m_dim.N6 * m_dim.N5 * m_dim.N4 * + m_dim.N3 * m_dim.N2; } KOKKOS_INLINE_FUNCTION constexpr size_type stride_0() const { return m_stride; @@ -1749,13 +1739,31 @@ struct ViewOffset< } #else - ViewOffset() = default; - ViewOffset(const ViewOffset&) = default; + ViewOffset() = default; + ViewOffset(const ViewOffset&) = default; ViewOffset& operator=(const ViewOffset&) = default; #endif /* Enable padding for trivial scalar types with non-zero trivial scalar size. */ + + private: + template + KOKKOS_FUNCTION constexpr size_type compute_stride( + const Kokkos::LayoutRight& arg_layout) { + if (arg_layout.stride != KOKKOS_IMPL_CTOR_DEFAULT_ARG) + return arg_layout.stride; + size_type value = m_dim.N1; + if constexpr (dimension_type::rank > 2) value *= m_dim.N2; + if constexpr (dimension_type::rank > 3) value *= m_dim.N3; + if constexpr (dimension_type::rank > 4) value *= m_dim.N4; + if constexpr (dimension_type::rank > 5) value *= m_dim.N5; + if constexpr (dimension_type::rank > 6) value *= m_dim.N6; + if constexpr (dimension_type::rank > 7) value *= m_dim.N7; + return Padding::stride(value); + } + + public: template KOKKOS_INLINE_FUNCTION constexpr ViewOffset( std::integral_constant const&, @@ -1764,37 +1772,7 @@ struct ViewOffset< arg_layout.dimension[2], arg_layout.dimension[3], arg_layout.dimension[4], arg_layout.dimension[5], arg_layout.dimension[6], arg_layout.dimension[7]), - m_stride( - Padding:: - stride(/* 2 <= rank */ - m_dim.N1 * - (dimension_type::rank == 2 - ? size_t(1) - : m_dim.N2 * - (dimension_type::rank == 3 - ? size_t(1) - : m_dim.N3 * - (dimension_type::rank == 4 - ? size_t(1) - : m_dim.N4 * - (dimension_type::rank == - 5 - ? size_t(1) - : m_dim.N5 * - (dimension_type:: - rank == - 6 - ? size_t( - 1) - : m_dim.N6 * - (dimension_type:: - rank == - 7 - ? size_t( - 1) - : m_dim - .N7)))))))) { - } + m_stride(compute_stride(arg_layout)) {} template KOKKOS_INLINE_FUNCTION constexpr ViewOffset( @@ -1886,8 +1864,8 @@ struct ViewStride<0> { static constexpr size_t S0 = 0, S1 = 0, S2 = 0, S3 = 0, S4 = 0, S5 = 0, S6 = 0, S7 = 0; - ViewStride() = default; - ViewStride(const ViewStride&) = default; + ViewStride() = default; + ViewStride(const ViewStride&) = default; ViewStride& operator=(const ViewStride&) = default; KOKKOS_INLINE_FUNCTION @@ -1901,8 +1879,8 @@ struct ViewStride<1> { static constexpr size_t S1 = 0, S2 = 0, S3 = 0, S4 = 0, S5 = 0, S6 = 0, S7 = 0; - ViewStride() = default; - ViewStride(const ViewStride&) = default; + ViewStride() = default; + ViewStride(const ViewStride&) = default; ViewStride& operator=(const ViewStride&) = default; KOKKOS_INLINE_FUNCTION @@ -1916,8 +1894,8 @@ struct ViewStride<2> { size_t S0, S1; static constexpr size_t S2 = 0, S3 = 0, S4 = 0, S5 = 0, S6 = 0, S7 = 0; - ViewStride() = default; - ViewStride(const ViewStride&) = default; + ViewStride() = default; + ViewStride(const ViewStride&) = default; ViewStride& operator=(const ViewStride&) = default; KOKKOS_INLINE_FUNCTION @@ -1931,8 +1909,8 @@ struct ViewStride<3> { size_t S0, S1, S2; static constexpr size_t S3 = 0, S4 = 0, S5 = 0, S6 = 0, S7 = 0; - ViewStride() = default; - ViewStride(const ViewStride&) = default; + ViewStride() = default; + ViewStride(const ViewStride&) = default; ViewStride& operator=(const ViewStride&) = default; KOKKOS_INLINE_FUNCTION @@ -1946,8 +1924,8 @@ struct ViewStride<4> { size_t S0, S1, S2, S3; static constexpr size_t S4 = 0, S5 = 0, S6 = 0, S7 = 0; - ViewStride() = default; - ViewStride(const ViewStride&) = default; + ViewStride() = default; + ViewStride(const ViewStride&) = default; ViewStride& operator=(const ViewStride&) = default; KOKKOS_INLINE_FUNCTION @@ -1961,8 +1939,8 @@ struct ViewStride<5> { size_t S0, S1, S2, S3, S4; static constexpr size_t S5 = 0, S6 = 0, S7 = 0; - ViewStride() = default; - ViewStride(const ViewStride&) = default; + ViewStride() = default; + ViewStride(const ViewStride&) = default; ViewStride& operator=(const ViewStride&) = default; KOKKOS_INLINE_FUNCTION @@ -1976,8 +1954,8 @@ struct ViewStride<6> { size_t S0, S1, S2, S3, S4, S5; static constexpr size_t S6 = 0, S7 = 0; - ViewStride() = default; - ViewStride(const ViewStride&) = default; + ViewStride() = default; + ViewStride(const ViewStride&) = default; ViewStride& operator=(const ViewStride&) = default; KOKKOS_INLINE_FUNCTION @@ -1991,8 +1969,8 @@ struct ViewStride<7> { size_t S0, S1, S2, S3, S4, S5, S6; static constexpr size_t S7 = 0; - ViewStride() = default; - ViewStride(const ViewStride&) = default; + ViewStride() = default; + ViewStride(const ViewStride&) = default; ViewStride& operator=(const ViewStride&) = default; KOKKOS_INLINE_FUNCTION @@ -2005,8 +1983,8 @@ template <> struct ViewStride<8> { size_t S0, S1, S2, S3, S4, S5, S6, S7; - ViewStride() = default; - ViewStride(const ViewStride&) = default; + ViewStride() = default; + ViewStride(const ViewStride&) = default; ViewStride& operator=(const ViewStride&) = default; KOKKOS_INLINE_FUNCTION @@ -2283,8 +2261,8 @@ struct ViewOffset { } #else - ViewOffset() = default; - ViewOffset(const ViewOffset&) = default; + ViewOffset() = default; + ViewOffset(const ViewOffset&) = default; ViewOffset& operator=(const ViewOffset&) = default; #endif @@ -2398,9 +2376,9 @@ struct ViewDataHandle { template struct ViewDataHandle< Traits, - std::enable_if_t<(std::is_same::value && - std::is_void::value && + std::enable_if_t<(std::is_same_v && + std::is_void_v && Traits::memory_traits::is_atomic)>> { using value_type = typename Traits::value_type; using handle_type = typename Kokkos::Impl::AtomicViewDataHandle; @@ -2422,11 +2400,10 @@ struct ViewDataHandle< template struct ViewDataHandle< - Traits, - std::enable_if_t<(std::is_void::value && - (!Traits::memory_traits::is_aligned) && - Traits::memory_traits::is_restrict && - (!Traits::memory_traits::is_atomic))>> { + Traits, std::enable_if_t<(std::is_void_v && + (!Traits::memory_traits::is_aligned) && + Traits::memory_traits::is_restrict && + (!Traits::memory_traits::is_atomic))>> { using value_type = typename Traits::value_type; using handle_type = typename Traits::value_type* KOKKOS_RESTRICT; using return_type = typename Traits::value_type& KOKKOS_RESTRICT; @@ -2446,11 +2423,10 @@ struct ViewDataHandle< template struct ViewDataHandle< - Traits, - std::enable_if_t<(std::is_void::value && - Traits::memory_traits::is_aligned && - (!Traits::memory_traits::is_restrict) && - (!Traits::memory_traits::is_atomic))>> { + Traits, std::enable_if_t<(std::is_void_v && + Traits::memory_traits::is_aligned && + (!Traits::memory_traits::is_restrict) && + (!Traits::memory_traits::is_atomic))>> { using value_type = typename Traits::value_type; // typedef work-around for intel compilers error #3186: expected typedef // declaration @@ -2485,11 +2461,10 @@ struct ViewDataHandle< template struct ViewDataHandle< - Traits, - std::enable_if_t<(std::is_void::value && - Traits::memory_traits::is_aligned && - Traits::memory_traits::is_restrict && - (!Traits::memory_traits::is_atomic))>> { + Traits, std::enable_if_t<(std::is_void_v && + Traits::memory_traits::is_aligned && + Traits::memory_traits::is_restrict && + (!Traits::memory_traits::is_atomic))>> { using value_type = typename Traits::value_type; // typedef work-around for intel compilers error #3186: expected typedef // declaration @@ -2533,11 +2508,10 @@ namespace Impl { /** \brief View mapping for non-specialized data type and standard layout */ template class ViewMapping< - Traits, - std::enable_if_t<( - std::is_void::value && - ViewOffset::is_mapping_plugin::value)>> { + Traits, std::enable_if_t<(std::is_void_v && + ViewOffset::is_mapping_plugin::value)>> { public: using offset_type = ViewOffset; @@ -2680,28 +2654,26 @@ class ViewMapping< reference_type reference() const { return m_impl_handle[0]; } template - KOKKOS_FORCEINLINE_FUNCTION - std::enable_if_t<(std::is_integral::value && - // if layout is neither stride nor irregular, - // then just use the handle directly - !(std::is_same::value || - !is_regular::value)), - reference_type> - reference(const I0& i0) const { + KOKKOS_FORCEINLINE_FUNCTION std::enable_if_t< + (std::is_integral_v && + // if layout is neither stride nor irregular, + // then just use the handle directly + !(std::is_same_v || + !is_regular::value)), + reference_type> + reference(const I0& i0) const { return m_impl_handle[i0]; } template - KOKKOS_FORCEINLINE_FUNCTION - std::enable_if_t<(std::is_integral::value && - // if the layout is strided or irregular, then - // we have to use the offset - (std::is_same::value || - !is_regular::value)), - reference_type> - reference(const I0& i0) const { + KOKKOS_FORCEINLINE_FUNCTION std::enable_if_t< + (std::is_integral_v && + // if the layout is strided or irregular, then + // we have to use the offset + (std::is_same_v || + !is_regular::value)), + reference_type> + reference(const I0& i0) const { return m_impl_handle[m_impl_offset(i0)]; } @@ -2780,7 +2752,7 @@ class ViewMapping< KOKKOS_DEFAULTED_FUNCTION ViewMapping& operator=(const ViewMapping&) = default; - KOKKOS_DEFAULTED_FUNCTION ViewMapping(ViewMapping&&) = default; + KOKKOS_DEFAULTED_FUNCTION ViewMapping(ViewMapping&&) = default; KOKKOS_DEFAULTED_FUNCTION ViewMapping& operator=(ViewMapping&&) = default; //---------------------------------------- @@ -2894,29 +2866,34 @@ template class ViewMapping< DstTraits, SrcTraits, std::enable_if_t<( - !(std::is_same:: - value) && // Added to have a new specialization for SrcType of - // LayoutStride + !(std::is_same_v)&& // Added to have a new + // specialization for + // SrcType of + // LayoutStride // default mappings - std::is_void::value && - std::is_void::value && + std::is_void_v && + std::is_void_v && ( // same layout - std::is_same::value || + std::is_same_v || // known layout - ((std::is_same::value || - std::is_same::value || - std::is_same::value) && - (std::is_same::value || - std::is_same::value || - std::is_same::value))))>> { + ((std::is_same_v || + std::is_same_v || + std::is_same_v< + typename DstTraits::array_layout, + Kokkos::LayoutStride>)&&(std::is_same_v || + std::is_same_v< + typename SrcTraits::array_layout, + Kokkos::LayoutRight> || + std::is_same_v< + typename SrcTraits::array_layout, + Kokkos::LayoutStride>))))>> { private: enum { is_assignable_space = Kokkos::Impl::MemorySpaceAccess< @@ -2926,10 +2903,10 @@ class ViewMapping< enum { is_assignable_value_type = - std::is_same::value || - std::is_same::value + std::is_same_v || + std::is_same_v }; enum { @@ -2939,12 +2916,12 @@ class ViewMapping< }; enum { - is_assignable_layout = - std::is_same::value || - std::is_same::value || - (DstTraits::dimension::rank == 0) || (DstTraits::dimension::rank == 1) + is_assignable_layout = std::is_same_v || + std::is_same_v || + (DstTraits::dimension::rank == 0) || + (DstTraits::dimension::rank == 1) }; public: @@ -3032,22 +3009,21 @@ class ViewMapping< template class ViewMapping< DstTraits, SrcTraits, - std::enable_if_t<( - std::is_same::value && - std::is_void::value && - std::is_void::value && - ( - // same layout - std::is_same::value || - // known layout - (std::is_same::value || - std::is_same::value || - std::is_same::value)))>> { + std::enable_if_t<(std::is_same_v && + std::is_void_v && + std::is_void_v && + ( + // same layout + std::is_same_v || + // known layout + (std::is_same_v || + std::is_same_v || + std::is_same_v)))>> { private: enum { is_assignable_space = Kokkos::Impl::MemorySpaceAccess< @@ -3057,10 +3033,10 @@ class ViewMapping< enum { is_assignable_value_type = - std::is_same::value || - std::is_same::value + std::is_same_v || + std::is_same_v }; enum { @@ -3091,8 +3067,7 @@ class ViewMapping< bool assignable = true; src.stride(strides); size_t exp_stride = 1; - if (std::is_same::value) { + if (std::is_same_v) { for (int i = 0; i < (int)src.Rank; i++) { if (i > 0) exp_stride *= src.extent(i - 1); if (strides[i] != exp_stride) { @@ -3100,8 +3075,8 @@ class ViewMapping< break; } } - } else if (std::is_same::value) { + } else if (std::is_same_v) { for (int i = 0; i < (int)src.Rank; i++) { if (i > 0) exp_stride *= src.extent(src.Rank - i); if (strides[src.Rank - 1 - i] != exp_stride) { @@ -3197,8 +3172,8 @@ struct SubViewDataTypeImpl> { template struct SubViewDataTypeImpl< - std::enable_if_t>::value>, - ValueType, Kokkos::Experimental::Extents, Integral, Args...> + std::enable_if_t>>, ValueType, + Kokkos::Experimental::Extents, Integral, Args...> : SubViewDataTypeImpl, Args...> {}; @@ -3230,13 +3205,13 @@ struct SubViewDataType : SubViewDataTypeImpl {}; template class ViewMapping< - std::enable_if_t<(std::is_void::value && - (std::is_same::value || - std::is_same::value || - std::is_same::value))>, + std::enable_if_t<( + std::is_void_v && + (std::is_same_v || + std::is_same_v || + std::is_same_v))>, SrcTraits, Args...> { private: static_assert(SrcTraits::rank == sizeof...(Args), @@ -3292,14 +3267,14 @@ class ViewMapping< // OutputRank 1 or 2, InputLayout Left, Interval 0 // because single stride one or second index has a stride. (rank <= 2 && R0 && - std::is_same::value) // replace with input rank + std::is_same_v) // replace with input rank || // OutputRank 1 or 2, InputLayout Right, Interval [InputRank-1] // because single stride one or second index has a stride. (rank <= 2 && R0_rev && - std::is_same::value) // replace input rank + std::is_same_v) // replace input rank ), typename SrcTraits::array_layout, Kokkos::LayoutStride>; diff --git a/lib/kokkos/core/src/impl/Kokkos_ViewTracker.hpp b/lib/kokkos/core/src/View/Kokkos_ViewTracker.hpp similarity index 100% rename from lib/kokkos/core/src/impl/Kokkos_ViewTracker.hpp rename to lib/kokkos/core/src/View/Kokkos_ViewTracker.hpp diff --git a/lib/kokkos/core/src/View/Kokkos_ViewTraits.hpp b/lib/kokkos/core/src/View/Kokkos_ViewTraits.hpp new file mode 100644 index 0000000000..5eddfc68e0 --- /dev/null +++ b/lib/kokkos/core/src/View/Kokkos_ViewTraits.hpp @@ -0,0 +1,457 @@ +//@HEADER +// ************************************************************************ +// +// Kokkos v. 4.0 +// Copyright (2022) National Technology & Engineering +// Solutions of Sandia, LLC (NTESS). +// +// Under the terms of Contract DE-NA0003525 with NTESS, +// the U.S. Government retains certain rights in this software. +// +// Part of Kokkos, under the Apache License v2.0 with LLVM Exceptions. +// See https://kokkos.org/LICENSE for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//@HEADER + +#ifndef KOKKOS_IMPL_PUBLIC_INCLUDE +#include +static_assert(false, + "Including non-public Kokkos header files is not allowed."); +#endif +#ifndef KOKKOS_VIEWTRAITS_HPP +#define KOKKOS_VIEWTRAITS_HPP + +#include +#include +#include +#include +#include +#ifdef KOKKOS_ENABLE_IMPL_MDSPAN +#include +#include +#endif + +//---------------------------------------------------------------------------- +//---------------------------------------------------------------------------- + +namespace Kokkos { + +struct ALL_t { + KOKKOS_FUNCTION + constexpr const ALL_t& operator()() const { return *this; } + + KOKKOS_FUNCTION + constexpr bool operator==(const ALL_t&) const { return true; } +}; + +#ifdef KOKKOS_ENABLE_DEPRECATED_CODE_4 +namespace Impl { +// TODO This alias declaration forces us to fully qualify ALL_t inside the +// Kokkos::Impl namespace to avoid deprecation warnings. Replace the +// fully-qualified name when we remove Kokkos::Impl::ALL_t. +using ALL_t KOKKOS_DEPRECATED_WITH_COMMENT("Use Kokkos::ALL_t instead!") = + Kokkos::ALL_t; +} // namespace Impl +#endif + +// FIXME_OPENMPTARGET - The `declare target` is needed for the Intel GPUs with +// the OpenMPTarget backend +#if defined(KOKKOS_ENABLE_OPENMPTARGET) && defined(KOKKOS_COMPILER_INTEL_LLVM) +#pragma omp declare target +#endif + +inline constexpr Kokkos::ALL_t ALL{}; + +#if defined(KOKKOS_ENABLE_OPENMPTARGET) && defined(KOKKOS_COMPILER_INTEL_LLVM) +#pragma omp end declare target +#endif + +namespace Impl { + +template +struct ViewArrayAnalysis; + +template ::non_const_value_type> +struct ViewDataAnalysis; + +template +class ViewMapping { + public: + enum : bool { is_assignable_data_type = false }; + enum : bool { is_assignable = false }; +}; + +template +constexpr KOKKOS_INLINE_FUNCTION std::size_t count_valid_integers( + const IntType i0, const IntType i1, const IntType i2, const IntType i3, + const IntType i4, const IntType i5, const IntType i6, const IntType i7) { + static_assert(std::is_integral_v, + "count_valid_integers() must have integer arguments."); + + return (i0 != KOKKOS_INVALID_INDEX) + (i1 != KOKKOS_INVALID_INDEX) + + (i2 != KOKKOS_INVALID_INDEX) + (i3 != KOKKOS_INVALID_INDEX) + + (i4 != KOKKOS_INVALID_INDEX) + (i5 != KOKKOS_INVALID_INDEX) + + (i6 != KOKKOS_INVALID_INDEX) + (i7 != KOKKOS_INVALID_INDEX); +} + +// FIXME Ideally, we would not instantiate this function for every possible View +// type. We should be able to only pass "extent" when we use mdspan. +template +KOKKOS_INLINE_FUNCTION void runtime_check_rank( + const View&, const bool is_void_spec, const size_t i0, const size_t i1, + const size_t i2, const size_t i3, const size_t i4, const size_t i5, + const size_t i6, const size_t i7, const char* label) { + (void)(label); + + if (is_void_spec) { + const size_t num_passed_args = + count_valid_integers(i0, i1, i2, i3, i4, i5, i6, i7); + // We either allow to pass as many extents as the dynamic rank is, or + // as many extents as the total rank is. In the latter case, the given + // extents for the static dimensions must match the + // compile-time extents. + constexpr int rank = View::rank(); + constexpr int dyn_rank = View::rank_dynamic(); + const bool n_args_is_dyn_rank = num_passed_args == dyn_rank; + const bool n_args_is_rank = num_passed_args == rank; + + if constexpr (rank != dyn_rank) { + if (n_args_is_rank) { + size_t new_extents[8] = {i0, i1, i2, i3, i4, i5, i6, i7}; + for (int i = dyn_rank; i < rank; ++i) + if (new_extents[i] != View::static_extent(i)) { + KOKKOS_IF_ON_HOST( + const std::string message = + "The specified run-time extent for Kokkos::View '" + + std::string(label) + + "' does not match the compile-time extent in dimension " + + std::to_string(i) + ". The given extent is " + + std::to_string(new_extents[i]) + " but should be " + + std::to_string(View::static_extent(i)) + ".\n"; + Kokkos::abort(message.c_str());) + KOKKOS_IF_ON_DEVICE( + Kokkos::abort( + "The specified run-time extents for a Kokkos::View " + "do not match the compile-time extents.");) + } + } + } + + if (!n_args_is_dyn_rank && !n_args_is_rank) { + KOKKOS_IF_ON_HOST( + const std::string message = + "Constructor for Kokkos::View '" + std::string(label) + + "' has mismatched number of arguments. The number " + "of arguments = " + + std::to_string(num_passed_args) + + " neither matches the dynamic rank = " + + std::to_string(dyn_rank) + + " nor the total rank = " + std::to_string(rank) + "\n"; + Kokkos::abort(message.c_str());) + KOKKOS_IF_ON_DEVICE(Kokkos::abort("Constructor for Kokkos View has " + "mismatched number of arguments.");) + } + } +} + +} /* namespace Impl */ +} /* namespace Kokkos */ + +// Class to provide a uniform type +namespace Kokkos { +namespace Impl { +template +struct ViewUniformType; +} +} // namespace Kokkos + +namespace Kokkos { + +#ifdef KOKKOS_ENABLE_IMPL_MDSPAN +namespace Impl { +struct UnsupportedKokkosArrayLayout; + +template +struct MDSpanViewTraits { + using mdspan_type = UnsupportedKokkosArrayLayout; +}; + +// "Natural" mdspan for a view if the View's ArrayLayout is supported. +template +struct MDSpanViewTraits::type>> { + using index_type = std::size_t; + using extents_type = + typename Impl::ExtentsFromDataType::type; + using mdspan_layout_type = + typename LayoutFromArrayLayout::type; + using accessor_type = + SpaceAwareAccessor>; + using mdspan_type = mdspan; +}; +} // namespace Impl +#endif // KOKKOS_ENABLE_IMPL_MDSPAN + +/** \class ViewTraits + * \brief Traits class for accessing attributes of a View. + * + * This is an implementation detail of View. It is only of interest + * to developers implementing a new specialization of View. + * + * Template argument options: + * - View< DataType > + * - View< DataType , Space > + * - View< DataType , Space , MemoryTraits > + * - View< DataType , ArrayLayout > + * - View< DataType , ArrayLayout , Space > + * - View< DataType , ArrayLayout , MemoryTraits > + * - View< DataType , ArrayLayout , Space , MemoryTraits > + * - View< DataType , MemoryTraits > + */ + +template +struct ViewTraits; + +template <> +struct ViewTraits { + using execution_space = void; + using memory_space = void; + using HostMirrorSpace = void; + using array_layout = void; + using memory_traits = void; + using specialize = void; + using hooks_policy = void; +}; + +template +struct ViewTraits { + // Ignore an extraneous 'void' + using execution_space = typename ViewTraits::execution_space; + using memory_space = typename ViewTraits::memory_space; + using HostMirrorSpace = typename ViewTraits::HostMirrorSpace; + using array_layout = typename ViewTraits::array_layout; + using memory_traits = typename ViewTraits::memory_traits; + using specialize = typename ViewTraits::specialize; + using hooks_policy = typename ViewTraits::hooks_policy; +}; + +template +struct ViewTraits< + std::enable_if_t::value>, + HooksPolicy, Prop...> { + using execution_space = typename ViewTraits::execution_space; + using memory_space = typename ViewTraits::memory_space; + using HostMirrorSpace = typename ViewTraits::HostMirrorSpace; + using array_layout = typename ViewTraits::array_layout; + using memory_traits = typename ViewTraits::memory_traits; + using specialize = typename ViewTraits::specialize; + using hooks_policy = HooksPolicy; +}; + +template +struct ViewTraits::value>, + ArrayLayout, Prop...> { + // Specify layout, keep subsequent space and memory traits arguments + + using execution_space = typename ViewTraits::execution_space; + using memory_space = typename ViewTraits::memory_space; + using HostMirrorSpace = typename ViewTraits::HostMirrorSpace; + using array_layout = ArrayLayout; + using memory_traits = typename ViewTraits::memory_traits; + using specialize = typename ViewTraits::specialize; + using hooks_policy = typename ViewTraits::hooks_policy; +}; + +template +struct ViewTraits::value>, Space, + Prop...> { + // Specify Space, memory traits should be the only subsequent argument. + + static_assert( + std::is_same_v::execution_space, + void> && + std::is_same_v::memory_space, + void> && + std::is_same_v::HostMirrorSpace, + void> && + std::is_same_v::array_layout, + void>, + "Only one View Execution or Memory Space template argument"); + + using execution_space = typename Space::execution_space; + using memory_space = typename Space::memory_space; + using HostMirrorSpace = + typename Kokkos::Impl::HostMirror::Space::memory_space; + using array_layout = typename execution_space::array_layout; + using memory_traits = typename ViewTraits::memory_traits; + using specialize = typename ViewTraits::specialize; + using hooks_policy = typename ViewTraits::hooks_policy; +}; + +template +struct ViewTraits< + std::enable_if_t::value>, + MemoryTraits, Prop...> { + // Specify memory trait, should not be any subsequent arguments + + static_assert( + std::is_same_v::execution_space, + void> && + std::is_same_v::memory_space, + void> && + std::is_same_v::array_layout, + void> && + std::is_same_v::memory_traits, + void> && + std::is_same_v::hooks_policy, + void>, + "MemoryTrait is the final optional template argument for a View"); + + using execution_space = void; + using memory_space = void; + using HostMirrorSpace = void; + using array_layout = void; + using memory_traits = MemoryTraits; + using specialize = void; + using hooks_policy = void; +}; + +template +struct ViewTraits { + private: + // Unpack the properties arguments + using prop = ViewTraits; + + using ExecutionSpace = + std::conditional_t, + typename prop::execution_space, + Kokkos::DefaultExecutionSpace>; + + using MemorySpace = + std::conditional_t, + typename prop::memory_space, + typename ExecutionSpace::memory_space>; + + using ArrayLayout = + std::conditional_t, + typename prop::array_layout, + typename ExecutionSpace::array_layout>; + + using HostMirrorSpace = std::conditional_t< + !std::is_void_v, + typename prop::HostMirrorSpace, + typename Kokkos::Impl::HostMirror::Space>; + + using MemoryTraits = + std::conditional_t, + typename prop::memory_traits, + typename Kokkos::MemoryManaged>; + + using HooksPolicy = + std::conditional_t, + typename prop::hooks_policy, + Kokkos::Experimental::DefaultViewHooks>; + + // Analyze data type's properties, + // May be specialized based upon the layout and value type + using data_analysis = Kokkos::Impl::ViewDataAnalysis; + + public: + //------------------------------------ + // Data type traits: + + using data_type = typename data_analysis::type; + using const_data_type = typename data_analysis::const_type; + using non_const_data_type = typename data_analysis::non_const_type; + + //------------------------------------ + // Compatible array of trivial type traits: + + using scalar_array_type = typename data_analysis::scalar_array_type; + using const_scalar_array_type = + typename data_analysis::const_scalar_array_type; + using non_const_scalar_array_type = + typename data_analysis::non_const_scalar_array_type; + + //------------------------------------ + // Value type traits: + + using value_type = typename data_analysis::value_type; + using const_value_type = typename data_analysis::const_value_type; + using non_const_value_type = typename data_analysis::non_const_value_type; + + //------------------------------------ + // Mapping traits: + + using array_layout = ArrayLayout; + using dimension = typename data_analysis::dimension; + + using specialize = std::conditional_t< + std::is_void_v, + typename prop::specialize, + typename data_analysis::specialize>; /* mapping specialization tag */ + + static constexpr unsigned rank = dimension::rank; + static constexpr unsigned rank_dynamic = dimension::rank_dynamic; + + //------------------------------------ + // Execution space, memory space, memory access traits, and host mirror space. + + using execution_space = ExecutionSpace; + using memory_space = MemorySpace; + using device_type = Kokkos::Device; + using memory_traits = MemoryTraits; + using host_mirror_space = HostMirrorSpace; + using hooks_policy = HooksPolicy; + + using size_type = typename MemorySpace::size_type; + + enum { is_hostspace = std::is_same_v }; + enum { is_managed = MemoryTraits::is_unmanaged == 0 }; + enum { is_random_access = MemoryTraits::is_random_access == 1 }; + + //------------------------------------ +}; + +//---------------------------------------------------------------------------- +//---------------------------------------------------------------------------- + +namespace Impl { +template +struct TypeListToViewTraits; + +template +struct TypeListToViewTraits> { + using type = ViewTraits; +}; + +// It is not safe to assume that subviews of views with the Aligned memory trait +// are also aligned. Hence, just remove that attribute for subviews. +template +struct RemoveAlignedMemoryTrait { + private: + using type_list_in = Kokkos::Impl::type_list; + using memory_traits = typename ViewTraits::memory_traits; + using type_list_in_wo_memory_traits = + typename Kokkos::Impl::type_list_remove_first::type; + using new_memory_traits = + Kokkos::MemoryTraits; + using new_type_list = typename Kokkos::Impl::concat_type_list< + type_list_in_wo_memory_traits, + Kokkos::Impl::type_list>::type; + + public: + using type = typename TypeListToViewTraits::type; +}; +} // namespace Impl + +} /* namespace Kokkos */ + +#endif /* KOKKOS_VIEWTRAITS_HPP */ diff --git a/lib/kokkos/core/src/impl/Kokkos_ViewUniformType.hpp b/lib/kokkos/core/src/View/Kokkos_ViewUniformType.hpp similarity index 88% rename from lib/kokkos/core/src/impl/Kokkos_ViewUniformType.hpp rename to lib/kokkos/core/src/View/Kokkos_ViewUniformType.hpp index 7de2869a0d..1e47613285 100644 --- a/lib/kokkos/core/src/impl/Kokkos_ViewUniformType.hpp +++ b/lib/kokkos/core/src/View/Kokkos_ViewUniformType.hpp @@ -24,11 +24,14 @@ namespace Impl { template struct ViewScalarToDataType { using type = typename ViewScalarToDataType::type *; + using const_type = + typename ViewScalarToDataType::const_type *; }; template struct ViewScalarToDataType { - using type = ScalarType; + using type = ScalarType; + using const_type = const ScalarType; }; template @@ -49,12 +52,13 @@ struct ViewUniformLayout { template struct ViewUniformType { using data_type = typename ViewType::data_type; - using const_data_type = std::add_const_t; + using const_data_type = typename ViewType::const_data_type; using runtime_data_type = typename ViewScalarToDataType::type; - using runtime_const_data_type = typename ViewScalarToDataType< - std::add_const_t, ViewType::rank>::type; + using runtime_const_data_type = + typename ViewScalarToDataType::const_type; using array_layout = typename ViewUniformLayout { } KOKKOS_FUNCTION - constexpr typename offset_policy::data_handle_type offset(data_handle_type p, - size_t i) const - noexcept { + constexpr typename offset_policy::data_handle_type offset( + data_handle_type p, size_t i) const noexcept { return nested_acc.offset(p, i); } @@ -214,6 +212,199 @@ struct AtomicAccessorRelaxed { } }; +//===================================================================== +//============= Reference Counted Accessor and DataHandle ============= +//===================================================================== + +template +class ReferenceCountedDataHandle { + public: + using value_type = ElementType; + using pointer = value_type*; + using reference = value_type&; + using memory_space = MemorySpace; + + KOKKOS_DEFAULTED_FUNCTION + ReferenceCountedDataHandle() = default; + + // this only ever works on host + explicit ReferenceCountedDataHandle(SharedAllocationRecord* rec) { + m_tracker.assign_allocated_record_to_uninitialized(rec); + m_handle = static_cast(get_record()->data()); + } + + KOKKOS_FUNCTION + ReferenceCountedDataHandle(const SharedAllocationTracker& tracker, + pointer data_handle) + : m_tracker(tracker), m_handle(data_handle) {} + + // unmanaged ctor + template >> + KOKKOS_FUNCTION ReferenceCountedDataHandle(OtherElementType* ptr) + : m_tracker(), m_handle(ptr) {} + + // subview ctor + template >> + KOKKOS_FUNCTION ReferenceCountedDataHandle( + const ReferenceCountedDataHandle& other, OtherElementType* ptr) + : m_tracker(other.m_tracker), m_handle(ptr) {} + + // converting ctor + template >> + KOKKOS_FUNCTION ReferenceCountedDataHandle( + const ReferenceCountedDataHandle& other) + : m_tracker(other.m_tracker), m_handle(other.m_handle) {} + + template < + class OtherElementType, class OtherSpace, + class = std::enable_if_t< + std::is_convertible_v && + (std::is_same_v || + std::is_same_v)>> + KOKKOS_FUNCTION ReferenceCountedDataHandle( + const ReferenceCountedDataHandle& other) + : m_tracker(other.m_tracker), m_handle(other.m_handle) {} + + KOKKOS_FUNCTION + pointer get() const noexcept { return m_handle; } + KOKKOS_FUNCTION + explicit operator pointer() const noexcept { return m_handle; } + + bool has_record() const { return m_tracker.has_record(); } + auto* get_record() const { return m_tracker.get_record(); } + int use_count() const noexcept { return m_tracker.use_count(); } + + std::string get_label() const { return m_tracker.get_label(); } + KOKKOS_FUNCTION + const SharedAllocationTracker& tracker() const noexcept { return m_tracker; } + + KOKKOS_FUNCTION + friend bool operator==(const ReferenceCountedDataHandle& lhs, + const value_type* rhs) { + return lhs.m_handle == rhs; + } + + KOKKOS_FUNCTION + friend bool operator==(const value_type* lhs, + const ReferenceCountedDataHandle& rhs) { + return lhs == rhs.m_handle; + } + + private: + template + friend class ReferenceCountedDataHandle; + + template + friend class ReferenceCountedAccessor; + + SharedAllocationTracker m_tracker; + pointer m_handle = nullptr; +}; + +template +class ReferenceCountedAccessor; + +template +struct IsReferenceCountedAccessor : std::false_type {}; + +template +struct IsReferenceCountedAccessor< + ReferenceCountedAccessor> + : std::true_type {}; + +template +class ReferenceCountedAccessor { + public: + using element_type = ElementType; + using data_handle_type = ReferenceCountedDataHandle; + using reference = typename NestedAccessor::reference; + using offset_policy = + ReferenceCountedAccessor; + using memory_space = MemorySpace; + + KOKKOS_DEFAULTED_FUNCTION + constexpr ReferenceCountedAccessor() noexcept = default; + + template < + class OtherElementType, class OtherNestedAccessor, + class = std::enable_if_t< + std::is_convertible_v && + std::is_constructible_v>> + KOKKOS_FUNCTION constexpr ReferenceCountedAccessor( + const ReferenceCountedAccessor&) {} + + template < + class OtherElementType, class OtherSpace, class OtherNestedAccessor, + class = std::enable_if_t< + std::is_convertible_v && + (std::is_same_v || + std::is_same_v)&&std:: + is_constructible_v>> + KOKKOS_FUNCTION constexpr ReferenceCountedAccessor( + const ReferenceCountedAccessor&) {} + + template >> + KOKKOS_FUNCTION constexpr ReferenceCountedAccessor( + const default_accessor&) {} + + template ::value && + std::is_convertible_v>> + KOKKOS_FUNCTION operator DstAccessor() const { + return m_nested_acc; + } + + KOKKOS_FUNCTION + constexpr reference access(data_handle_type p, size_t i) const { + return m_nested_acc.access(p.get(), i); + } + + KOKKOS_FUNCTION + constexpr data_handle_type offset(data_handle_type p, size_t i) const { + return data_handle_type(p, m_nested_acc.offset(p.get(), i)); + } + + KOKKOS_FUNCTION + constexpr auto nested_accessor() const { return m_nested_acc; } + + private: +#ifdef _MDSPAN_NO_UNIQUE_ADDRESS + _MDSPAN_NO_UNIQUE_ADDRESS +#else + [[no_unique_address]] +#endif + NestedAccessor m_nested_acc; +}; + +template +using CheckedReferenceCountedAccessor = + SpaceAwareAccessor>>; + +template +using CheckedRelaxedAtomicAccessor = + SpaceAwareAccessor>; + +template +using CheckedReferenceCountedRelaxedAtomicAccessor = SpaceAwareAccessor< + MemorySpace, ReferenceCountedAccessor>>; + } // namespace Impl } // namespace Kokkos diff --git a/lib/kokkos/core/src/View/MDSpan/Kokkos_MDSpan_Layout.hpp b/lib/kokkos/core/src/View/MDSpan/Kokkos_MDSpan_Layout.hpp index 089628137d..f990d158bf 100644 --- a/lib/kokkos/core/src/View/MDSpan/Kokkos_MDSpan_Layout.hpp +++ b/lib/kokkos/core/src/View/MDSpan/Kokkos_MDSpan_Layout.hpp @@ -23,7 +23,11 @@ static_assert(false, #define KOKKOS_EXPERIMENTAL_MDSPAN_LAYOUT_HPP #include "Kokkos_MDSpan_Extents.hpp" -#include +#include + +// The difference between a legacy Kokkos array layout and an +// mdspan layout is that the array layouts can have state, but don't have the +// nested mapping. This file provides interoperability helpers. namespace Kokkos::Impl { @@ -77,32 +81,7 @@ KOKKOS_INLINE_FUNCTION auto array_layout_from_mapping( rank > 7 ? mapping.stride(7) : 0, }; } else { - // FIXME: Kokkos Layouts don't store stride (it's in the mapping) - // We could conceivably fix this by adding an extra ViewCtorProp for - // an abritrary padding. For now we will check for this. - if constexpr (rank > 1 && - (std::is_same_v> || - std::is_same_v>)) { - [[maybe_unused]] constexpr size_t strided_index = - std::is_same_v< - typename mapping_type::layout_type, - Kokkos::Experimental::layout_left_padded> - ? 1 - : rank - 2; - [[maybe_unused]] constexpr size_t extent_index = - std::is_same_v< - typename mapping_type::layout_type, - Kokkos::Experimental::layout_left_padded> - ? 0 - : rank - 1; - KOKKOS_ASSERT(mapping.stride(strided_index) == ext.extent(extent_index)); - } - - return ArrayLayout{rank > 0 ? ext.extent(0) : KOKKOS_IMPL_CTOR_DEFAULT_ARG, + ArrayLayout layout{rank > 0 ? ext.extent(0) : KOKKOS_IMPL_CTOR_DEFAULT_ARG, rank > 1 ? ext.extent(1) : KOKKOS_IMPL_CTOR_DEFAULT_ARG, rank > 2 ? ext.extent(2) : KOKKOS_IMPL_CTOR_DEFAULT_ARG, rank > 3 ? ext.extent(3) : KOKKOS_IMPL_CTOR_DEFAULT_ARG, @@ -110,12 +89,98 @@ KOKKOS_INLINE_FUNCTION auto array_layout_from_mapping( rank > 5 ? ext.extent(5) : KOKKOS_IMPL_CTOR_DEFAULT_ARG, rank > 6 ? ext.extent(6) : KOKKOS_IMPL_CTOR_DEFAULT_ARG, rank > 7 ? ext.extent(7) : KOKKOS_IMPL_CTOR_DEFAULT_ARG}; + + if constexpr (rank > 1 && + std::is_same_v>) { + layout.stride = mapping.stride(1); + } + if constexpr (std::is_same_v>) { + if constexpr (rank == 2) { + layout.stride = mapping.stride(0); + } + if constexpr (rank > 2) { + if (mapping.stride(rank - 2) != mapping.extents().extent(rank - 1)) + Kokkos::abort( + "Invalid conversion from layout_right_padded to LayoutRight"); + } + } + return layout; } #ifdef KOKKOS_COMPILER_INTEL __builtin_unreachable(); #endif } +template +KOKKOS_INLINE_FUNCTION auto mapping_from_array_layout_impl( + ArrayLayout layout, std::index_sequence) { + using index_type = typename MappingType::index_type; + using extents_type = typename MappingType::extents_type; + if constexpr (std::is_same_v || + std::is_same_v) { + return MappingType{ + extents_type{dextents{ + layout.dimension[Idx]...}}}; + } else { + if (layout.stride == KOKKOS_IMPL_CTOR_DEFAULT_ARG || + extents_type::rank() < 2) { + return MappingType{ + extents_type{dextents{ + layout.dimension[Idx]...}}}; + } else { + if constexpr (std::is_same_v && + extents_type::rank() > 2) { + size_t product_of_dimensions = 1; + for (size_t r = 1; r < extents_type::rank(); r++) + product_of_dimensions *= layout.dimension[r]; + if (product_of_dimensions != layout.stride) + Kokkos::abort( + "Invalid conversion from LayoutRight to layout_right_padded"); + } else { + return MappingType{ + extents_type{ + dextents{ + layout.dimension[Idx]...}}, + layout.stride}; + } + } + } +} +template +KOKKOS_INLINE_FUNCTION auto mapping_from_array_layout_impl( + LayoutStride layout, std::index_sequence) { + static_assert( + std::is_same_v); + using index_type = typename MappingType::index_type; + index_type strides[MappingType::extents_type::rank()] = { + layout.stride[Idx]...}; + return MappingType{ + mdspan_non_standard_tag(), + static_cast( + dextents{ + layout.dimension[Idx]...}), + strides}; +} + +// specialization for rank 0 to avoid empty array +template +KOKKOS_INLINE_FUNCTION auto mapping_from_array_layout_impl( + LayoutStride, std::index_sequence<>) { + return MappingType{}; +} + +template +KOKKOS_INLINE_FUNCTION auto mapping_from_array_layout(ArrayLayout layout) { + return mapping_from_array_layout_impl( + layout, std::make_index_sequence()); +} + template KOKKOS_INLINE_FUNCTION auto mapping_from_view_mapping(const VM &view_mapping) { using mapping_type = typename MDSpanType::mapping_type; diff --git a/lib/kokkos/core/src/decl/Kokkos_Declare_CUDA.hpp b/lib/kokkos/core/src/decl/Kokkos_Declare_CUDA.hpp index ebdf2c8211..79c137bfdd 100644 --- a/lib/kokkos/core/src/decl/Kokkos_Declare_CUDA.hpp +++ b/lib/kokkos/core/src/decl/Kokkos_Declare_CUDA.hpp @@ -28,7 +28,9 @@ #include #include #include +#ifdef KOKKOS_ENABLE_DEPRECATED_CODE_4 #include +#endif #include #include #include diff --git a/lib/kokkos/core/src/decl/Kokkos_Declare_SYCL.hpp b/lib/kokkos/core/src/decl/Kokkos_Declare_SYCL.hpp index d13c90825c..3570ed2b6e 100644 --- a/lib/kokkos/core/src/decl/Kokkos_Declare_SYCL.hpp +++ b/lib/kokkos/core/src/decl/Kokkos_Declare_SYCL.hpp @@ -35,6 +35,16 @@ #include #include #include + +namespace Kokkos { +namespace Experimental { +using SYCLDeviceUSMSpace = ::Kokkos::SYCLDeviceUSMSpace; +using SYCLHostUSMSpace = ::Kokkos::SYCLHostUSMSpace; +using SYCLSharedUSMSpace = ::Kokkos::SYCLSharedUSMSpace; +using SYCL = ::Kokkos::SYCL; +} // namespace Experimental +} // namespace Kokkos + #endif #endif diff --git a/lib/kokkos/core/src/fwd/Kokkos_Fwd_SYCL.hpp b/lib/kokkos/core/src/fwd/Kokkos_Fwd_SYCL.hpp index 400794f865..399b986041 100644 --- a/lib/kokkos/core/src/fwd/Kokkos_Fwd_SYCL.hpp +++ b/lib/kokkos/core/src/fwd/Kokkos_Fwd_SYCL.hpp @@ -19,7 +19,6 @@ #if defined(KOKKOS_ENABLE_SYCL) namespace Kokkos { -namespace Experimental { class SYCLDeviceUSMSpace; ///< Memory space on SYCL device, not accessible from ///< the host class SYCLSharedUSMSpace; ///< Memory space accessible from both the SYCL @@ -27,7 +26,6 @@ class SYCLSharedUSMSpace; ///< Memory space accessible from both the SYCL class SYCLHostUSMSpace; ///< Memory space accessible from both the SYCL ///< device and the host (host pinned) class SYCL; ///< Execution space for SYCL -} // namespace Experimental } // namespace Kokkos #endif #endif diff --git a/lib/kokkos/core/src/impl/KokkosExp_Host_IterateTile.hpp b/lib/kokkos/core/src/impl/KokkosExp_Host_IterateTile.hpp index a44ffefa6b..a9db2c4cf4 100644 --- a/lib/kokkos/core/src/impl/KokkosExp_Host_IterateTile.hpp +++ b/lib/kokkos/core/src/impl/KokkosExp_Host_IterateTile.hpp @@ -1458,7 +1458,7 @@ struct Tile_Loop_Type<8, IsLeft, IType, void, void> { template struct Tile_Loop_Type<1, IsLeft, IType, Tagged, - std::enable_if_t::value>> { + std::enable_if_t>> { template static void apply(Func const& func, bool cond, Offset const& offset, ExtentA const& a, ExtentB const& b) { @@ -1477,7 +1477,7 @@ struct Tile_Loop_Type<1, IsLeft, IType, Tagged, template struct Tile_Loop_Type<2, IsLeft, IType, Tagged, - std::enable_if_t::value>> { + std::enable_if_t>> { template static void apply(Func const& func, bool cond, Offset const& offset, ExtentA const& a, ExtentB const& b) { @@ -1496,7 +1496,7 @@ struct Tile_Loop_Type<2, IsLeft, IType, Tagged, template struct Tile_Loop_Type<3, IsLeft, IType, Tagged, - std::enable_if_t::value>> { + std::enable_if_t>> { template static void apply(Func const& func, bool cond, Offset const& offset, ExtentA const& a, ExtentB const& b) { @@ -1515,7 +1515,7 @@ struct Tile_Loop_Type<3, IsLeft, IType, Tagged, template struct Tile_Loop_Type<4, IsLeft, IType, Tagged, - std::enable_if_t::value>> { + std::enable_if_t>> { template static void apply(Func const& func, bool cond, Offset const& offset, ExtentA const& a, ExtentB const& b) { @@ -1534,7 +1534,7 @@ struct Tile_Loop_Type<4, IsLeft, IType, Tagged, template struct Tile_Loop_Type<5, IsLeft, IType, Tagged, - std::enable_if_t::value>> { + std::enable_if_t>> { template static void apply(Func const& func, bool cond, Offset const& offset, ExtentA const& a, ExtentB const& b) { @@ -1553,7 +1553,7 @@ struct Tile_Loop_Type<5, IsLeft, IType, Tagged, template struct Tile_Loop_Type<6, IsLeft, IType, Tagged, - std::enable_if_t::value>> { + std::enable_if_t>> { template static void apply(Func const& func, bool cond, Offset const& offset, ExtentA const& a, ExtentB const& b) { @@ -1572,7 +1572,7 @@ struct Tile_Loop_Type<6, IsLeft, IType, Tagged, template struct Tile_Loop_Type<7, IsLeft, IType, Tagged, - std::enable_if_t::value>> { + std::enable_if_t>> { template static void apply(Func const& func, bool cond, Offset const& offset, ExtentA const& a, ExtentB const& b) { @@ -1591,7 +1591,7 @@ struct Tile_Loop_Type<7, IsLeft, IType, Tagged, template struct Tile_Loop_Type<8, IsLeft, IType, Tagged, - std::enable_if_t::value>> { + std::enable_if_t>> { template static void apply(Func const& func, bool cond, Offset const& offset, ExtentA const& a, ExtentB const& b) { @@ -1616,7 +1616,7 @@ struct HostIterateTile; // For ParallelFor template struct HostIterateTile::value>> { + std::enable_if_t>> { using index_type = typename RP::index_type; using point_type = typename RP::point_type; @@ -1635,12 +1635,11 @@ struct HostIterateTile 0 - ? (m_rp.m_upper[i] - offset[i]) - : (m_rp.m_upper[i] - - m_rp.m_lower[i]); // when single tile encloses range + (m_rp.m_upper[i] - 1 - offset[i]) == 0 ? 1 + : (m_rp.m_upper[i] - m_rp.m_tile[i]) > 0 + ? (m_rp.m_upper[i] - offset[i]) + : (m_rp.m_upper[i] - + m_rp.m_lower[i]); // when single tile encloses range } } @@ -2000,30 +1999,28 @@ struct HostIterateTile - std::enable_if_t<(sizeof...(Args) == RP::rank && std::is_void::value), - void> + std::enable_if_t<(sizeof...(Args) == RP::rank && std::is_void_v), void> apply(Args&&... args) const { m_func(args...); } template - std::enable_if_t<(sizeof...(Args) == RP::rank && !std::is_void::value), - void> + std::enable_if_t<(sizeof...(Args) == RP::rank && !std::is_void_v), void> apply(Args&&... args) const { m_func(m_tag, args...); } RP const m_rp; Functor const m_func; - std::conditional_t::value, int, Tag> m_tag; + std::conditional_t, int, Tag> m_tag; }; // For ParallelReduce // ValueType - scalar: For reductions template struct HostIterateTile::value && - !std::is_array::value>> { + std::enable_if_t && + !std::is_array_v>> { using index_type = typename RP::index_type; using point_type = typename RP::point_type; @@ -2050,12 +2047,11 @@ struct HostIterateTile 0 - ? (m_rp.m_upper[i] - offset[i]) - : (m_rp.m_upper[i] - - m_rp.m_lower[i]); // when single tile encloses range + (m_rp.m_upper[i] - 1 - offset[i]) == 0 ? 1 + : (m_rp.m_upper[i] - m_rp.m_tile[i]) > 0 + ? (m_rp.m_upper[i] - offset[i]) + : (m_rp.m_upper[i] - + m_rp.m_lower[i]); // when single tile encloses range } } @@ -2430,7 +2426,7 @@ struct HostIterateTile::value, int, Tag> m_tag; + std::conditional_t, int, Tag> m_tag; }; // For ParallelReduce @@ -2438,8 +2434,8 @@ struct HostIterateTile struct HostIterateTile::value && - std::is_array::value>> { + std::enable_if_t && + std::is_array_v>> { using index_type = typename RP::index_type; using point_type = typename RP::point_type; @@ -2463,12 +2459,11 @@ struct HostIterateTile 0 - ? (m_rp.m_upper[i] - offset[i]) - : (m_rp.m_upper[i] - - m_rp.m_lower[i]); // when single tile encloses range + (m_rp.m_upper[i] - 1 - offset[i]) == 0 ? 1 + : (m_rp.m_upper[i] - m_rp.m_tile[i]) > 0 + ? (m_rp.m_upper[i] - offset[i]) + : (m_rp.m_upper[i] - + m_rp.m_lower[i]); // when single tile encloses range } } @@ -2842,7 +2837,7 @@ struct HostIterateTile::value, int, Tag> m_tag; + std::conditional_t, int, Tag> m_tag; }; // ------------------------------------------------------------------ // diff --git a/lib/kokkos/core/src/impl/KokkosExp_IterateTileGPU.hpp b/lib/kokkos/core/src/impl/KokkosExp_IterateTileGPU.hpp index e1273ab9e3..e6b2fcbef4 100644 --- a/lib/kokkos/core/src/impl/KokkosExp_IterateTileGPU.hpp +++ b/lib/kokkos/core/src/impl/KokkosExp_IterateTileGPU.hpp @@ -41,13 +41,13 @@ struct EmulateCUDADim3 { template KOKKOS_IMPL_FORCEINLINE_FUNCTION std::enable_if_t::value> _tag_invoke(Functor const& f, Args&&... args) { - f((Args &&) args...); + f((Args&&)args...); } template KOKKOS_IMPL_FORCEINLINE_FUNCTION std::enable_if_t::value> _tag_invoke(Functor const& f, Args&&... args) { - f(Tag{}, (Args &&) args...); + f(Tag{}, (Args&&)args...); } template , Args&&... args) { - _tag_invoke(f, vals[Idxs]..., (Args &&) args...); + _tag_invoke(f, vals[Idxs]..., (Args&&)args...); } template @@ -63,7 +63,7 @@ KOKKOS_IMPL_FORCEINLINE_FUNCTION void _tag_invoke_array(Functor const& f, T (&vals)[N], Args&&... args) { _tag_invoke_array_helper(f, vals, std::make_index_sequence{}, - (Args &&) args...); + (Args&&)args...); } // ------------------------------------------------------------------ // diff --git a/lib/kokkos/core/src/impl/Kokkos_AnalyzePolicy.hpp b/lib/kokkos/core/src/impl/Kokkos_AnalyzePolicy.hpp index d77ec0c753..b483653021 100644 --- a/lib/kokkos/core/src/impl/Kokkos_AnalyzePolicy.hpp +++ b/lib/kokkos/core/src/impl/Kokkos_AnalyzePolicy.hpp @@ -143,7 +143,7 @@ struct AnalyzeExecPolicyUseMatcher, Trait, Traits...> { static constexpr auto trigger_error_message = show_name_of_invalid_execution_policy_trait{}; static_assert( - /* always false: */ std::is_void::value, + /* always false: */ std::is_void_v, "Unknown execution policy trait. Search compiler output for " "'show_name_of_invalid_execution_policy_trait' to see the type of the " "invalid trait."); diff --git a/lib/kokkos/core/src/impl/Kokkos_ChaseLev.hpp b/lib/kokkos/core/src/impl/Kokkos_ChaseLev.hpp index d8ab77b205..4ea0b8d343 100644 --- a/lib/kokkos/core/src/impl/Kokkos_ChaseLev.hpp +++ b/lib/kokkos/core/src/impl/Kokkos_ChaseLev.hpp @@ -95,12 +95,12 @@ struct non_owning_variable_size_circular_buffer { non_owning_variable_size_circular_buffer( non_owning_variable_size_circular_buffer const&) = delete; non_owning_variable_size_circular_buffer( - non_owning_variable_size_circular_buffer&&) = default; - non_owning_variable_size_circular_buffer& operator =( - non_owning_variable_size_circular_buffer const&) = delete; - non_owning_variable_size_circular_buffer& operator =( non_owning_variable_size_circular_buffer&&) = default; - ~non_owning_variable_size_circular_buffer() = default; + non_owning_variable_size_circular_buffer& operator=( + non_owning_variable_size_circular_buffer const&) = delete; + non_owning_variable_size_circular_buffer& operator=( + non_owning_variable_size_circular_buffer&&) = default; + ~non_owning_variable_size_circular_buffer() = default; KOKKOS_FORCEINLINE_FUNCTION constexpr size_type size() const noexcept { return m_size; } @@ -138,7 +138,7 @@ struct ChaseLevDeque { public: template ::value>> + std::is_default_constructible_v>> ChaseLevDeque() : m_array() {} explicit ChaseLevDeque(CircularBufferT buffer) : m_array(std::move(buffer)) {} @@ -165,7 +165,7 @@ struct ChaseLevDeque { #ifdef _WIN32 Kokkos::memory_fence(); bool const success = - Kokkos::atomic_compare_exchange_strong(&m_top, t, t + 1); + (t == Kokkos::atomic_compare_exchange(&m_top, t, t + 1)); Kokkos::memory_fence(); if (!success) { return_value = nullptr; @@ -226,7 +226,7 @@ struct ChaseLevDeque { #ifdef _WIN32 Kokkos::memory_fence(); bool const success = - Kokkos::atomic_compare_exchange_strong(&m_top, t, t + 1); + (t == Kokkos::atomic_compare_exchange(&m_top, t, t + 1)); Kokkos::memory_fence(); if (!success) { return_value = nullptr; diff --git a/lib/kokkos/core/src/impl/Kokkos_ClockTic.hpp b/lib/kokkos/core/src/impl/Kokkos_ClockTic.hpp index 6e3d99ebd6..ee53fd8bc6 100644 --- a/lib/kokkos/core/src/impl/Kokkos_ClockTic.hpp +++ b/lib/kokkos/core/src/impl/Kokkos_ClockTic.hpp @@ -27,8 +27,9 @@ // To use OpenCL(TM) built-in intrinsics inside kernels, we have to // forward-declare their prototype, also see // https://github.com/intel/pti-gpu/blob/master/chapters/binary_instrumentation/OpenCLBuiltIn.md -#if defined(KOKKOS_ENABLE_SYCL) && defined(KOKKOS_ARCH_INTEL_GPU) && \ - defined(__SYCL_DEVICE_ONLY__) +#if defined(KOKKOS_ENABLE_SYCL) && \ + defined(KOKKOS_ENABLE_SYCL_RELOCATABLE_DEVICE_CODE) && \ + defined(KOKKOS_ARCH_INTEL_GPU) && defined(__SYCL_DEVICE_ONLY__) extern SYCL_EXTERNAL unsigned long __attribute__((overloadable)) intel_get_cycle_counter(); #endif @@ -55,8 +56,10 @@ KOKKOS_IMPL_DEVICE_FUNCTION inline uint64_t clock_tic_device() noexcept { // Return value of 64-bit hi-res clock register. return clock64(); -#elif defined(KOKKOS_ENABLE_SYCL) && defined(KOKKOS_ARCH_INTEL_GPU) && \ - defined(__SYCL_DEVICE_ONLY__) +// FIXME_SYCL We can only return something useful for Intel GPUs and with RDC +#elif defined(KOKKOS_ENABLE_SYCL) && \ + defined(KOKKOS_ENABLE_SYCL_RELOCATABLE_DEVICE_CODE) && \ + defined(KOKKOS_ARCH_INTEL_GPU) && defined(__SYCL_DEVICE_ONLY__) return intel_get_cycle_counter(); diff --git a/lib/kokkos/core/src/impl/Kokkos_Combined_Reducer.hpp b/lib/kokkos/core/src/impl/Kokkos_Combined_Reducer.hpp index e6dd3c6339..d7319e80c8 100644 --- a/lib/kokkos/core/src/impl/Kokkos_Combined_Reducer.hpp +++ b/lib/kokkos/core/src/impl/Kokkos_Combined_Reducer.hpp @@ -93,7 +93,7 @@ struct CombinedReducerValueImpl, std::move(arg_values))... {} template - KOKKOS_INLINE_FUNCTION ValueType& get() & noexcept { + KOKKOS_INLINE_FUNCTION ValueType& get() & noexcept { return this->CombinedReducerValueItemImpl::ref(); } template @@ -181,7 +181,7 @@ struct CombinedReducerImpl, Space, KOKKOS_DEFAULTED_FUNCTION constexpr CombinedReducerImpl( CombinedReducerImpl const&) = default; KOKKOS_DEFAULTED_FUNCTION constexpr CombinedReducerImpl( - CombinedReducerImpl&&) = default; + CombinedReducerImpl&&) = default; KOKKOS_DEFAULTED_FUNCTION constexpr CombinedReducerImpl& operator=( CombinedReducerImpl const&) = default; KOKKOS_DEFAULTED_FUNCTION constexpr CombinedReducerImpl& operator=( @@ -192,8 +192,8 @@ struct CombinedReducerImpl, Space, template KOKKOS_FUNCTION constexpr explicit CombinedReducerImpl( value_type& value, ReducersDeduced&&... reducers) noexcept - : CombinedReducerStorageImpl((ReducersDeduced &&) - reducers)..., + : CombinedReducerStorageImpl( + (ReducersDeduced&&)reducers)..., m_value_view(&value) {} KOKKOS_FUNCTION constexpr void join(value_type& dest, @@ -348,8 +348,8 @@ struct CombinedReductionFunctorWrapperImpl< IndexOrMemberOrTagType1&& arg_first, IndexOrMemberTypesThenValueType&&... args) const { this->template _call_op_impl( - (IndexOrMemberOrTagType1 &&) arg_first, - (IndexOrMemberTypesThenValueType &&) args...); + (IndexOrMemberOrTagType1&&)arg_first, + (IndexOrMemberTypesThenValueType&&)args...); } // end call operator }}}2 @@ -369,19 +369,19 @@ struct CombinedReductionFunctorWrapperImpl< template KOKKOS_FORCEINLINE_FUNCTION std::enable_if_t< - !std::is_same, value_type>::value> + !std::is_same_v, value_type>> _call_op_impl(IdxOrMemberTypes&&... idxs, IdxOrMemberType1&& idx, IdxOrMemberTypesThenValueType&&... args) const { this->template _call_op_impl( - (IdxOrMemberTypes &&) idxs..., (IdxOrMemberType1 &&) idx, - (IdxOrMemberTypesThenValueType &&) args...); + (IdxOrMemberTypes&&)idxs..., (IdxOrMemberType1&&)idx, + (IdxOrMemberTypesThenValueType&&)args...); } // base case template KOKKOS_FORCEINLINE_FUNCTION void _call_op_impl(IdxOrMemberTypes&&... idxs, value_type& out) const { - m_functor((IdxOrMemberTypes &&) idxs..., + m_functor((IdxOrMemberTypes&&)idxs..., out.template get()...); } }; @@ -464,8 +464,8 @@ KOKKOS_INLINE_FUNCTION constexpr auto make_combined_reducer_value( typename _reducer_from_arg_t::value_type...>{ // This helper function is now poorly named after refactoring. - _get_value_from_combined_reducer_ctor_arg((ReferencesOrViewsOrReducers &&) - args)...}; + _get_value_from_combined_reducer_ctor_arg( + (ReferencesOrViewsOrReducers&&)args)...}; //---------------------------------------- } @@ -480,7 +480,7 @@ KOKKOS_INLINE_FUNCTION constexpr auto make_combined_reducer( Space, _reducer_from_arg_t...>; return reducer_type(value, _reducer_from_arg_t{ - (ReferencesOrViewsOrReducers &&) args}...); + (ReferencesOrViewsOrReducers&&)args}...); //---------------------------------------- } diff --git a/lib/kokkos/core/src/impl/Kokkos_ConcurrentBitset.hpp b/lib/kokkos/core/src/impl/Kokkos_ConcurrentBitset.hpp index ca4edce5c3..9bde2f72a3 100644 --- a/lib/kokkos/core/src/impl/Kokkos_ConcurrentBitset.hpp +++ b/lib/kokkos/core/src/impl/Kokkos_ConcurrentBitset.hpp @@ -110,15 +110,15 @@ struct concurrent_bitset { // when is full at the atomic_fetch_add(+1) // then a release occurs before the atomic_fetch_add(-1). - const uint32_t state = (uint32_t)Kokkos::atomic_fetch_add( - reinterpret_cast(buffer), 1); + const uint32_t state = + Kokkos::atomic_fetch_add(const_cast(buffer), 1); const uint32_t state_error = state_header != (state & state_header_mask); const uint32_t state_bit_used = state & state_used_mask; if (state_error || (bit_bound <= state_bit_used)) { - Kokkos::atomic_fetch_add(reinterpret_cast(buffer), -1); + Kokkos::atomic_fetch_sub(const_cast(buffer), 1); return state_error ? type(-2, -2) : type(-1, -1); } @@ -132,7 +132,8 @@ struct concurrent_bitset { while (1) { const uint32_t word = bit >> bits_per_int_lg2; const uint32_t mask = 1u << (bit & bits_per_int_mask); - const uint32_t prev = Kokkos::atomic_fetch_or(buffer + word + 1, mask); + const uint32_t prev = Kokkos::atomic_fetch_or( + const_cast(buffer) + word + 1, mask); if (!(prev & mask)) { // Successfully claimed 'result.first' by @@ -194,15 +195,15 @@ struct concurrent_bitset { // when is full at the atomic_fetch_add(+1) // then a release occurs before the atomic_fetch_add(-1). - const uint32_t state = (uint32_t)Kokkos::atomic_fetch_add( - reinterpret_cast(buffer), 1); + const uint32_t state = + Kokkos::atomic_fetch_add(const_cast(buffer), 1); const uint32_t state_error = state_header != (state & state_header_mask); const uint32_t state_bit_used = state & state_used_mask; if (state_error || (bit_bound <= state_bit_used)) { - Kokkos::atomic_fetch_add(reinterpret_cast(buffer), -1); + Kokkos::atomic_fetch_sub(const_cast(buffer), 1); return state_error ? type(-2, -2) : type(-1, -1); } @@ -216,7 +217,8 @@ struct concurrent_bitset { while (1) { const uint32_t word = bit >> bits_per_int_lg2; const uint32_t mask = 1u << (bit & bits_per_int_mask); - const uint32_t prev = Kokkos::atomic_fetch_or(buffer + word + 1, mask); + const uint32_t prev = Kokkos::atomic_fetch_or( + const_cast(buffer) + word + 1, mask); if (!(prev & mask)) { // Successfully claimed 'result.first' by @@ -262,8 +264,8 @@ struct concurrent_bitset { } const uint32_t mask = 1u << (bit & bits_per_int_mask); - const uint32_t prev = - Kokkos::atomic_fetch_and(buffer + (bit >> bits_per_int_lg2) + 1, ~mask); + const uint32_t prev = Kokkos::atomic_fetch_and( + const_cast(buffer) + (bit >> bits_per_int_lg2) + 1, ~mask); if (!(prev & mask)) { return -1; @@ -273,7 +275,7 @@ struct concurrent_bitset { Kokkos::memory_fence(); const int count = - Kokkos::atomic_fetch_add(reinterpret_cast(buffer), -1); + Kokkos::atomic_fetch_sub(const_cast(buffer), 1); // Flush the store-release Kokkos::memory_fence(); @@ -299,8 +301,8 @@ struct concurrent_bitset { } const uint32_t mask = 1u << (bit & bits_per_int_mask); - const uint32_t prev = - Kokkos::atomic_fetch_or(buffer + (bit >> bits_per_int_lg2) + 1, mask); + const uint32_t prev = Kokkos::atomic_fetch_or( + const_cast(buffer) + (bit >> bits_per_int_lg2) + 1, mask); if (!(prev & mask)) { return -1; @@ -310,7 +312,7 @@ struct concurrent_bitset { Kokkos::memory_fence(); const int count = - Kokkos::atomic_fetch_add(reinterpret_cast(buffer), -1); + Kokkos::atomic_fetch_sub(const_cast(buffer), 1); return (count & state_used_mask) - 1; } diff --git a/lib/kokkos/core/src/impl/Kokkos_Core.cpp b/lib/kokkos/core/src/impl/Kokkos_Core.cpp index 532709aa98..72f33ffaab 100644 --- a/lib/kokkos/core/src/impl/Kokkos_Core.cpp +++ b/lib/kokkos/core/src/impl/Kokkos_Core.cpp @@ -138,7 +138,7 @@ int get_device_count() { KOKKOS_IMPL_HIP_SAFE_CALL(hipGetDeviceCount(&count)); return count; #elif defined(KOKKOS_ENABLE_SYCL) - return Kokkos::Experimental::Impl::get_sycl_devices().size(); + return Kokkos::Impl::get_sycl_devices().size(); #elif defined(KOKKOS_ENABLE_OPENACC) return acc_get_num_devices( Kokkos::Experimental::Impl::OpenACC_Traits::dev_type); @@ -183,7 +183,7 @@ std::vector const& Kokkos::Impl::get_visible_devices() { #elif defined(KOKKOS_ENABLE_OPENMPTARGET) int device = omp_get_default_device(); // FIXME_OPENMPTARGET #elif defined(KOKKOS_ENABLE_SYCL) - int device = Experimental::Impl::SYCLInternal::m_syclDev; + int device = Impl::SYCLInternal::m_syclDev; #else int device = -1; return device; @@ -271,7 +271,7 @@ int Kokkos::Impl::get_ctest_gpu(int local_rank) { ss << "Error: local rank " << local_rank << " is outside the bounds of resource groups provided by CTest. Raised" << " by Kokkos::Impl::get_ctest_gpu()."; - throw_runtime_exception(ss.str()); + abort(ss.str().c_str()); } // Get the resource types allocated to this resource group @@ -284,7 +284,7 @@ int Kokkos::Impl::get_ctest_gpu(int local_rank) { std::ostringstream ss; ss << "Error: " << ctest_resource_group_name << " is not specified. Raised" << " by Kokkos::Impl::get_ctest_gpu()."; - throw_runtime_exception(ss.str()); + abort(ss.str().c_str()); } // Look for the device type specified in CTEST_KOKKOS_DEVICE_TYPE @@ -308,7 +308,7 @@ int Kokkos::Impl::get_ctest_gpu(int local_rank) { ss << "Error: device type '" << ctest_kokkos_device_type << "' not included in " << ctest_resource_group_name << ". Raised by Kokkos::Impl::get_ctest_gpu()."; - throw_runtime_exception(ss.str()); + abort(ss.str().c_str()); } // Get the device ID @@ -324,7 +324,7 @@ int Kokkos::Impl::get_ctest_gpu(int local_rank) { std::ostringstream ss; ss << "Error: " << ctest_resource_group_id_name << " is not specified. Raised by Kokkos::Impl::get_ctest_gpu()."; - throw_runtime_exception(ss.str()); + abort(ss.str().c_str()); } auto const* comma = std::strchr(resource_str, ','); @@ -332,7 +332,7 @@ int Kokkos::Impl::get_ctest_gpu(int local_rank) { std::ostringstream ss; ss << "Error: invalid value of " << ctest_resource_group_id_name << ": '" << resource_str << "'. Raised by Kokkos::Impl::get_ctest_gpu()."; - throw_runtime_exception(ss.str()); + abort(ss.str().c_str()); } std::string id(resource_str + 3, comma - resource_str - 3); @@ -613,7 +613,7 @@ void pre_initialize_internal(const Kokkos::InitializationSettings& settings) { #endif declare_configuration_metadata("architecture", "Default Device", - typeid(Kokkos::DefaultExecutionSpace).name()); + Kokkos::DefaultExecutionSpace::name()); #if defined(KOKKOS_ARCH_A64FX) declare_configuration_metadata("architecture", "CPU architecture", "A64FX"); @@ -666,6 +666,9 @@ void pre_initialize_internal(const Kokkos::InitializationSettings& settings) { #elif defined(KOKKOS_ARCH_RISCV_SG2042) declare_configuration_metadata("architecture", "CPU architecture", "SG2042 (RISC-V)") +#elif defined(KOKKOS_ARCH_RISCV_RVA22V) + declare_configuration_metadata("architecture", "CPU architecture", + "RVA22V (RISC-V)") #else declare_configuration_metadata("architecture", "CPU architecture", "none"); #endif @@ -738,8 +741,8 @@ void pre_initialize_internal(const Kokkos::InitializationSettings& settings) { declare_configuration_metadata("architecture", "GPU architecture", "HOPPER90"); #elif defined(KOKKOS_ARCH_AMD_GFX906) - declare_configuration_metadata("architecture", "GPU architecture", - "AMD_GFX906"); + declare_configuration_metadata("architecture", "GPU architecture", + "AMD_GFX906"); #elif defined(KOKKOS_ARCH_AMD_GFX908) declare_configuration_metadata("architecture", "GPU architecture", "AMD_GFX908"); @@ -976,7 +979,7 @@ void Kokkos::Impl::parse_environment_variables( Tools::Impl::parse_environment_variables(tools_init_arguments); if (init_result.result == Tools::Impl::InitializationStatus::environment_argument_mismatch) { - Impl::throw_runtime_exception(init_result.error_message); + Kokkos::abort(init_result.error_message.c_str()); } combine(settings, tools_init_arguments); diff --git a/lib/kokkos/core/src/impl/Kokkos_Default_GraphNodeKernel.hpp b/lib/kokkos/core/src/impl/Kokkos_Default_GraphNodeKernel.hpp index c71c21d2ac..cd00fdadeb 100644 --- a/lib/kokkos/core/src/impl/Kokkos_Default_GraphNodeKernel.hpp +++ b/lib/kokkos/core/src/impl/Kokkos_Default_GraphNodeKernel.hpp @@ -36,15 +36,22 @@ struct GraphNodeKernelDefaultImpl { // TODO @graphs decide if this should use vtable or intrusive erasure via // function pointers like in the rest of the graph interface virtual void execute_kernel() = 0; + + GraphNodeKernelDefaultImpl() = default; + + explicit GraphNodeKernelDefaultImpl(ExecutionSpace exec) + : m_execution_space(std::move(exec)) {} + + ExecutionSpace m_execution_space; }; // TODO Indicate that this kernel specialization is only for the Host somehow? template class GraphNodeKernelImpl - : public PatternImplSpecializationFromTag::type, - public GraphNodeKernelDefaultImpl { + : public GraphNodeKernelDefaultImpl, + public PatternImplSpecializationFromTag::type { public: using base_t = typename PatternImplSpecializationFromTag - GraphNodeKernelImpl(std::string const&, ExecutionSpace const&, - Functor arg_functor, PolicyDeduced&& arg_policy, - ArgsDeduced&&... args) - : base_t(std::move(arg_functor), (PolicyDeduced &&) arg_policy, - (ArgsDeduced &&) args...), - execute_kernel_vtable_base_t() {} + GraphNodeKernelImpl(std::string const &, ExecutionSpace const &, + Functor arg_functor, PolicyDeduced &&arg_policy, + ArgsDeduced &&...args) + : execute_kernel_vtable_base_t(arg_policy.space()), + base_t(std::move(arg_functor), (PolicyDeduced &&)arg_policy, + (ArgsDeduced &&)args...) {} // FIXME @graph Forward through the instance once that works in the backends template - GraphNodeKernelImpl(ExecutionSpace const& ex, Functor arg_functor, - PolicyDeduced&& arg_policy, ArgsDeduced&&... args) + GraphNodeKernelImpl(ExecutionSpace const &ex, Functor arg_functor, + PolicyDeduced &&arg_policy, ArgsDeduced &&...args) : GraphNodeKernelImpl("", ex, std::move(arg_functor), - (PolicyDeduced &&) arg_policy, - (ArgsDeduced &&) args...) {} + (PolicyDeduced &&)arg_policy, + (ArgsDeduced &&)args...) { + // FIXME This constructor seem unused. + } - void execute_kernel() final { this->base_t::execute(); } + void execute_kernel() override final { this->base_t::execute(); } }; // end GraphNodeKernelImpl }}}1 @@ -88,7 +97,7 @@ struct GraphNodeAggregateKernelDefaultImpl using is_graph_kernel = std::true_type; }; using graph_kernel = GraphNodeAggregateKernelDefaultImpl; - void execute_kernel() final {} + void execute_kernel() override final {} }; } // end namespace Impl diff --git a/lib/kokkos/core/src/impl/Kokkos_Default_GraphNode_Impl.hpp b/lib/kokkos/core/src/impl/Kokkos_Default_GraphNode_Impl.hpp index 223ae391ab..31d147ea89 100644 --- a/lib/kokkos/core/src/impl/Kokkos_Default_GraphNode_Impl.hpp +++ b/lib/kokkos/core/src/impl/Kokkos_Default_GraphNode_Impl.hpp @@ -69,10 +69,10 @@ struct GraphNodeBackendSpecificDetails { GraphNodeBackendSpecificDetails(GraphNodeBackendSpecificDetails&&) noexcept = delete; - GraphNodeBackendSpecificDetails& operator =( + GraphNodeBackendSpecificDetails& operator=( GraphNodeBackendSpecificDetails const&) = delete; - GraphNodeBackendSpecificDetails& operator =( + GraphNodeBackendSpecificDetails& operator=( GraphNodeBackendSpecificDetails&&) noexcept = delete; ~GraphNodeBackendSpecificDetails() = default; @@ -92,6 +92,18 @@ struct GraphNodeBackendSpecificDetails { m_is_aggregate = true; } + // A node is awaitable if it can execute a kernel. + // A root node or an aggregate node cannot be waited for, because it does + // not launch anything. + bool awaitable() const { return (!m_is_root) && (!m_is_aggregate); } + + // Retrieve the execution space instance that has been passed to + // the kernel at construction phase. + const ExecutionSpace& get_execution_space() const { + KOKKOS_EXPECTS(m_kernel_ptr != nullptr) + return m_kernel_ptr->m_execution_space; + } + void set_predecessor( std::shared_ptr> arg_pred_impl) { @@ -104,7 +116,7 @@ struct GraphNodeBackendSpecificDetails { m_predecessors.push_back(std::move(arg_pred_impl)); } - void execute_node() { + void execute_node(const ExecutionSpace& exec) { // This node could have already been executed as the predecessor of some // other KOKKOS_EXPECTS(bool(m_kernel_ptr) || m_has_executed) @@ -115,8 +127,18 @@ struct GraphNodeBackendSpecificDetails { // supported semantics, but instinct I have feels like it should be... m_has_executed = true; for (auto const& predecessor : m_predecessors) { - predecessor->execute_node(); + predecessor->execute_node(exec); } + + // Before executing the kernel, be sure to fence the execution space + // instance of predecessors. + for (const auto& predecessor : m_predecessors) { + if (predecessor->awaitable() && + predecessor->get_execution_space() != this->get_execution_space()) + predecessor->get_execution_space().fence( + "Kokkos::DefaultGraphNode::execute_node: sync with predecessors"); + } + m_kernel_ptr->execute_kernel(); } KOKKOS_ENSURES(m_has_executed) diff --git a/lib/kokkos/core/src/impl/Kokkos_Default_Graph_Impl.hpp b/lib/kokkos/core/src/impl/Kokkos_Default_Graph_Impl.hpp index 05d4854919..8dfa19a178 100644 --- a/lib/kokkos/core/src/impl/Kokkos_Default_Graph_Impl.hpp +++ b/lib/kokkos/core/src/impl/Kokkos_Default_Graph_Impl.hpp @@ -58,12 +58,12 @@ struct GraphImpl : private ExecutionSpaceInstanceStorage { // Not movable or copyable; it spends its whole live as a shared_ptr in the // Graph object - GraphImpl() = default; - GraphImpl(GraphImpl const&) = delete; - GraphImpl(GraphImpl&&) = delete; + GraphImpl() = default; + GraphImpl(GraphImpl const&) = delete; + GraphImpl(GraphImpl&&) = delete; GraphImpl& operator=(GraphImpl const&) = delete; - GraphImpl& operator=(GraphImpl&&) = delete; - ~GraphImpl() = default; + GraphImpl& operator=(GraphImpl&&) = delete; + ~GraphImpl() = default; explicit GraphImpl(ExecutionSpace arg_space) : execution_space_instance_storage_base_t(std::move(arg_space)) {} @@ -136,17 +136,40 @@ struct GraphImpl : private ExecutionSpaceInstanceStorage { return rv; } - void submit() { + void instantiate() { + KOKKOS_EXPECTS(!m_has_been_instantiated); + m_has_been_instantiated = true; + } + + void submit(const ExecutionSpace& exec) { + if (!m_has_been_instantiated) instantiate(); // This reset is gross, but for the purposes of our simple host // implementation... for (auto& sink : m_sinks) { sink->reset_has_executed(); } + + // We don't know where the nodes will execute, so we need to fence the given + // execution space instance before proceeding. This is the simplest way + // of guaranteeing that the kernels in the graph are correctly "enqueued". + exec.fence( + "Kokkos::DefaultGraph::submit: fencing before launching graph nodes"); + for (auto& sink : m_sinks) { - sink->execute_node(); + sink->execute_node(exec); + } + + // Once all sinks have been executed, we need to fence them. + for (const auto& sink : m_sinks) { + if (sink->awaitable() && sink->get_execution_space() != exec) + sink->get_execution_space().fence( + "Kokkos::DefaultGraph::submit: fencing before ending graph submit"); } } + private: + bool m_has_been_instantiated = false; + // end required customizations }}}2 //---------------------------------------------------------------------------- }; diff --git a/lib/kokkos/core/src/impl/Kokkos_EBO.hpp b/lib/kokkos/core/src/impl/Kokkos_EBO.hpp index 8ba94ba4cc..a8a4d6617b 100644 --- a/lib/kokkos/core/src/impl/Kokkos_EBO.hpp +++ b/lib/kokkos/core/src/impl/Kokkos_EBO.hpp @@ -52,16 +52,16 @@ struct EBOBaseImpl; template class CtorNotOnDevice> struct EBOBaseImpl { template ::value && - std::is_constructible::value && + std::enable_if_t && + std::is_constructible_v && !CtorNotOnDevice::value, int> = 0> KOKKOS_FORCEINLINE_FUNCTION constexpr explicit EBOBaseImpl( Args&&...) noexcept {} template ::value && - std::is_constructible::value && + std::enable_if_t && + std::is_constructible_v && CtorNotOnDevice::value, long> = 0> inline constexpr explicit EBOBaseImpl(Args&&...) noexcept {} @@ -110,18 +110,18 @@ struct EBOBaseImpl { T m_ebo_object; template ::value && + std::enable_if_t && !CTorsNotOnDevice::value && - std::is_constructible::value, + std::is_constructible_v, int> = 0> KOKKOS_FORCEINLINE_FUNCTION constexpr explicit EBOBaseImpl( Args&&... args) noexcept(noexcept(T(std::forward(args)...))) : m_ebo_object(std::forward(args)...) {} template ::value && + std::enable_if_t && CTorsNotOnDevice::value && - std::is_constructible::value, + std::is_constructible_v, long> = 0> inline constexpr explicit EBOBaseImpl(Args&&... args) noexcept( noexcept(T(std::forward(args)...))) @@ -167,9 +167,9 @@ struct EBOBaseImpl { template class CtorsNotOnDevice = NoCtorsNotOnDevice> struct StandardLayoutNoUniqueAddressMemberEmulation - : EBOBaseImpl::value, CtorsNotOnDevice> { + : EBOBaseImpl, CtorsNotOnDevice> { private: - using ebo_base_t = EBOBaseImpl::value, CtorsNotOnDevice>; + using ebo_base_t = EBOBaseImpl, CtorsNotOnDevice>; public: using ebo_base_t::ebo_base_t; diff --git a/lib/kokkos/core/src/impl/Kokkos_ExecPolicy.cpp b/lib/kokkos/core/src/impl/Kokkos_ExecPolicy.cpp index 04c5e0bd22..58a5de2aa6 100644 --- a/lib/kokkos/core/src/impl/Kokkos_ExecPolicy.cpp +++ b/lib/kokkos/core/src/impl/Kokkos_ExecPolicy.cpp @@ -41,7 +41,7 @@ void team_policy_check_valid_storage_level_argument(int level) { std::stringstream ss; ss << "TeamPolicy::set_scratch_size(/*level*/ " << level << ", ...) storage level argument must be 0 or 1 to be valid\n"; - Impl::throw_runtime_exception(ss.str()); + abort(ss.str().c_str()); } } diff --git a/lib/kokkos/core/src/impl/Kokkos_ExecSpaceManager.hpp b/lib/kokkos/core/src/impl/Kokkos_ExecSpaceManager.hpp index 58ed54275a..5805b78ee7 100644 --- a/lib/kokkos/core/src/impl/Kokkos_ExecSpaceManager.hpp +++ b/lib/kokkos/core/src/impl/Kokkos_ExecSpaceManager.hpp @@ -123,14 +123,14 @@ template struct ExecSpaceDerived : ExecSpaceBase { static_assert(check_valid_execution_space()); static_assert(check_is_regular()); - void initialize(InitializationSettings const& settings) final { + void initialize(InitializationSettings const& settings) override final { ExecutionSpace::impl_initialize(settings); } - void finalize() final { ExecutionSpace::impl_finalize(); } - void static_fence(std::string const& label) final { + void finalize() override final { ExecutionSpace::impl_finalize(); } + void static_fence(std::string const& label) override final { ExecutionSpace::impl_static_fence(label); } - void print_configuration(std::ostream& os, bool verbose) final { + void print_configuration(std::ostream& os, bool verbose) override final { ExecutionSpace().print_configuration(os, verbose); } }; diff --git a/lib/kokkos/core/src/impl/Kokkos_FixedBufferMemoryPool.hpp b/lib/kokkos/core/src/impl/Kokkos_FixedBufferMemoryPool.hpp deleted file mode 100644 index 4726a87b97..0000000000 --- a/lib/kokkos/core/src/impl/Kokkos_FixedBufferMemoryPool.hpp +++ /dev/null @@ -1,279 +0,0 @@ -//@HEADER -// ************************************************************************ -// -// Kokkos v. 4.0 -// Copyright (2022) National Technology & Engineering -// Solutions of Sandia, LLC (NTESS). -// -// Under the terms of Contract DE-NA0003525 with NTESS, -// the U.S. Government retains certain rights in this software. -// -// Part of Kokkos, under the Apache License v2.0 with LLVM Exceptions. -// See https://kokkos.org/LICENSE for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// -//@HEADER - -#ifndef KOKKOS_IMPL_KOKKOS_FIXEDBUFFERMEMORYPOOL_HPP -#define KOKKOS_IMPL_KOKKOS_FIXEDBUFFERMEMORYPOOL_HPP - -#include -#include - -#include -#include - -namespace Kokkos { -namespace Impl { - -template -class FixedBlockSizeMemoryPool - : private MemorySpaceInstanceStorage { - public: - using memory_space = typename DeviceType::memory_space; - using size_type = SizeType; - - private: - using memory_space_storage_base = - MemorySpaceInstanceStorage; - using tracker_type = Kokkos::Impl::SharedAllocationTracker; - using record_type = Kokkos::Impl::SharedAllocationRecord; - - struct alignas(Align) Block { - union { - char ignore; - char data[Size]; - }; - }; - - static constexpr auto actual_size = sizeof(Block); - - // TODO shared allocation tracker - // TODO @optimization put the index values on different cache lines (CPU) or - // pages (GPU)? - - tracker_type m_tracker = {}; - size_type m_num_blocks = 0; - size_type m_first_free_idx = 0; - size_type m_last_free_idx = 0; - Kokkos::OwningRawPtr m_first_block = nullptr; - Kokkos::OwningRawPtr m_free_indices = nullptr; - - enum : size_type { IndexInUse = ~size_type(0) }; - - public: - FixedBlockSizeMemoryPool(memory_space const& mem_space, size_type num_blocks) - : memory_space_storage_base(mem_space), - m_tracker(), - m_num_blocks(num_blocks), - m_first_free_idx(0), - m_last_free_idx(num_blocks) { - // TODO alignment? - auto block_record = record_type::allocate( - mem_space, "FixedBlockSizeMemPool_blocks", num_blocks * sizeof(Block)); - KOKKOS_ASSERT(intptr_t(block_record->data()) % Align == 0); - m_tracker.assign_allocated_record_to_uninitialized(block_record); - m_first_block = (Block*)block_record->data(); - - auto idx_record = - record_type::allocate(mem_space, "Kokkos::FixedBlockSizeMemPool_blocks", - num_blocks * sizeof(size_type)); - KOKKOS_ASSERT(intptr_t(idx_record->data()) % alignof(size_type) == 0); - m_tracker.assign_allocated_record_to_uninitialized(idx_record); - m_free_indices = (size_type*)idx_record->data(); - - for (size_type i = 0; i < num_blocks; ++i) { - m_free_indices[i] = i; - } - - Kokkos::memory_fence(); - } - - // For compatibility with MemoryPool<> - FixedBlockSizeMemoryPool(memory_space const& mem_space, - size_t mempool_capacity, unsigned, unsigned, - unsigned) - : FixedBlockSizeMemoryPool( - mem_space, mempool_capacity / - actual_size) { /* forwarding ctor, must be empty */ - } - - KOKKOS_DEFAULTED_FUNCTION FixedBlockSizeMemoryPool() = default; - KOKKOS_DEFAULTED_FUNCTION FixedBlockSizeMemoryPool( - FixedBlockSizeMemoryPool&&) = default; - KOKKOS_DEFAULTED_FUNCTION FixedBlockSizeMemoryPool( - FixedBlockSizeMemoryPool const&) = default; - KOKKOS_DEFAULTED_FUNCTION FixedBlockSizeMemoryPool& operator=( - FixedBlockSizeMemoryPool&&) = default; - KOKKOS_DEFAULTED_FUNCTION FixedBlockSizeMemoryPool& operator=( - FixedBlockSizeMemoryPool const&) = default; - - KOKKOS_INLINE_FUNCTION - void* allocate(size_type alloc_size) const noexcept { - (void)alloc_size; - KOKKOS_EXPECTS(alloc_size <= Size); - auto free_idx_counter = Kokkos::atomic_fetch_add( - (volatile size_type*)&m_first_free_idx, size_type(1)); - auto free_idx_idx = free_idx_counter % m_num_blocks; - - // We don't have exclusive access to m_free_indices[free_idx_idx] because - // the allocate counter might have lapped us since we incremented it - auto current_free_idx = m_free_indices[free_idx_idx]; - size_type free_idx = IndexInUse; - free_idx = Kokkos::atomic_compare_exchange(&m_free_indices[free_idx_idx], - current_free_idx, free_idx); - Kokkos::memory_fence(); - - // TODO figure out how to decrement here? - - if (free_idx == IndexInUse) { - return nullptr; - } else { - return (void*)&m_first_block[free_idx]; - } - } - - KOKKOS_INLINE_FUNCTION - void deallocate(void* ptr, size_type /*alloc_size*/) const noexcept { - // figure out which block we are - auto offset = intptr_t(ptr) - intptr_t(m_first_block); - - KOKKOS_EXPECTS(offset % actual_size == 0 && - offset / actual_size < m_num_blocks); - - Kokkos::memory_fence(); - auto last_idx_idx = Kokkos::atomic_fetch_add( - (volatile size_type*)&m_last_free_idx, size_type(1)); - last_idx_idx %= m_num_blocks; - m_free_indices[last_idx_idx] = offset / actual_size; - } -}; - -#if 0 -template < - class DeviceType, - size_t Size, - size_t Align=1, - class SizeType = typename DeviceType::execution_space::size_type -> -class FixedBlockSizeChaseLevMemoryPool - : private MemorySpaceInstanceStorage -{ -public: - - using memory_space = typename DeviceType::memory_space; - using size_type = SizeType; - -private: - - using memory_space_storage_base = MemorySpaceInstanceStorage; - using tracker_type = Kokkos::Impl::SharedAllocationTracker; - using record_type = Kokkos::Impl::SharedAllocationRecord; - - struct alignas(Align) Block { union { char ignore; char data[Size]; }; }; - - static constexpr auto actual_size = sizeof(Block); - - tracker_type m_tracker = { }; - size_type m_num_blocks = 0; - size_type m_first_free_idx = 0; - size_type m_last_free_idx = 0; - - - enum : size_type { IndexInUse = ~size_type(0) }; - -public: - - FixedBlockSizeMemoryPool( - memory_space const& mem_space, - size_type num_blocks - ) : memory_space_storage_base(mem_space), - m_tracker(), - m_num_blocks(num_blocks), - m_first_free_idx(0), - m_last_free_idx(num_blocks) - { - // TODO alignment? - auto block_record = record_type::allocate( - mem_space, "FixedBlockSizeMemPool_blocks", num_blocks * sizeof(Block) - ); - KOKKOS_ASSERT(intptr_t(block_record->data()) % Align == 0); - m_tracker.assign_allocated_record_to_uninitialized(block_record); - m_first_block = (Block*)block_record->data(); - - auto idx_record = record_type::allocate( - mem_space, "FixedBlockSizeMemPool_blocks", num_blocks * sizeof(size_type) - ); - KOKKOS_ASSERT(intptr_t(idx_record->data()) % alignof(size_type) == 0); - m_tracker.assign_allocated_record_to_uninitialized(idx_record); - m_free_indices = (size_type*)idx_record->data(); - - for(size_type i = 0; i < num_blocks; ++i) { - m_free_indices[i] = i; - } - - Kokkos::memory_fence(); - } - - // For compatibility with MemoryPool<> - FixedBlockSizeMemoryPool( - memory_space const& mem_space, - size_t mempool_capacity, - unsigned, unsigned, unsigned - ) : FixedBlockSizeMemoryPool(mem_space, mempool_capacity / actual_size) - { /* forwarding ctor, must be empty */ } - - KOKKOS_DEFAULTED_FUNCTION FixedBlockSizeMemoryPool() = default; - KOKKOS_DEFAULTED_FUNCTION FixedBlockSizeMemoryPool(FixedBlockSizeMemoryPool&&) = default; - KOKKOS_DEFAULTED_FUNCTION FixedBlockSizeMemoryPool(FixedBlockSizeMemoryPool const&) = default; - KOKKOS_DEFAULTED_FUNCTION FixedBlockSizeMemoryPool& operator=(FixedBlockSizeMemoryPool&&) = default; - KOKKOS_DEFAULTED_FUNCTION FixedBlockSizeMemoryPool& operator=(FixedBlockSizeMemoryPool const&) = default; - - - KOKKOS_INLINE_FUNCTION - void* allocate(size_type alloc_size) const noexcept - { - KOKKOS_EXPECTS(alloc_size <= Size); - auto free_idx_counter = Kokkos::atomic_fetch_add((volatile size_type*)&m_first_free_idx, size_type(1)); - auto free_idx_idx = free_idx_counter % m_num_blocks; - - // We don't have exclusive access to m_free_indices[free_idx_idx] because - // the allocate counter might have lapped us since we incremented it - auto current_free_idx = m_free_indices[free_idx_idx]; - size_type free_idx = IndexInUse; - free_idx = - Kokkos::atomic_compare_exchange(&m_free_indices[free_idx_idx], current_free_idx, free_idx); - Kokkos::memory_fence(); - - // TODO figure out how to decrement here? - - if(free_idx == IndexInUse) { - return nullptr; - } - else { - return (void*)&m_first_block[free_idx]; - } - } - - KOKKOS_INLINE_FUNCTION - void deallocate(void* ptr, size_type alloc_size) const noexcept - { - // figure out which block we are - auto offset = intptr_t(ptr) - intptr_t(m_first_block); - - KOKKOS_EXPECTS(offset % actual_size == 0 && offset/actual_size < m_num_blocks); - - Kokkos::memory_fence(); - auto last_idx_idx = Kokkos::atomic_fetch_add((volatile size_type*)&m_last_free_idx, size_type(1)); - last_idx_idx %= m_num_blocks; - m_free_indices[last_idx_idx] = offset / actual_size; - } - -}; -#endif - -} // end namespace Impl -} // end namespace Kokkos - -#endif // KOKKOS_IMPL_KOKKOS_FIXEDBUFFERMEMORYPOOL_HPP diff --git a/lib/kokkos/core/src/impl/Kokkos_FunctorAnalysis.hpp b/lib/kokkos/core/src/impl/Kokkos_FunctorAnalysis.hpp index e844a5295e..29a365e6e4 100644 --- a/lib/kokkos/core/src/impl/Kokkos_FunctorAnalysis.hpp +++ b/lib/kokkos/core/src/impl/Kokkos_FunctorAnalysis.hpp @@ -118,8 +118,8 @@ struct FunctorAnalysis { using functor_has_space = has_execution_space; static_assert(!policy_has_space::value || !functor_has_space::value || - std::is_same::value, + std::is_same_v, "Execution Policy and Functor execution space must match"); //---------------------------------------- @@ -136,9 +136,8 @@ struct FunctorAnalysis { typename std::is_void::type> { using type = typename F::value_type; - static_assert(!std::is_reference::value && - std::rank::value <= 1 && - std::extent::value == 0, + static_assert(!std::is_reference_v && std::rank_v <= 1 && + std::extent_v == 0, "Kokkos Functor::value_type is T or T[]"); }; @@ -149,7 +148,7 @@ struct FunctorAnalysis { template ::type, - bool T = std::is_void::value> + bool T = std::is_void_v> struct deduce_value_type { using type = V; }; @@ -290,8 +289,8 @@ struct FunctorAnalysis { using candidate_type = typename deduce_value_type::type; enum { - candidate_is_void = std::is_void::value, - candidate_is_array = std::rank::value == 1 + candidate_is_void = std::is_void_v, + candidate_is_array = std::rank_v == 1 }; //---------------------------------------- @@ -306,7 +305,7 @@ struct FunctorAnalysis { using value_type = std::remove_extent_t; - static_assert(!std::is_const::value, + static_assert(!std::is_const_v, "Kokkos functor operator reduce argument cannot be const"); private: @@ -614,21 +613,20 @@ struct FunctorAnalysis { }; template - struct DeduceJoinNoTag::value || - (!is_reducer::value && - std::is_void::value)) && - detected_join_no_tag::value>> + struct DeduceJoinNoTag< + F, std::enable_if_t<(is_reducer::value || + (!is_reducer::value && std::is_void_v)) && + detected_join_no_tag::value>> : public has_join_no_tag_function { enum : bool { value = true }; }; template struct DeduceJoinNoTag< - F, - std::enable_if_t<(is_reducer::value || - (!is_reducer::value && std::is_void::value)) && - (!detected_join_no_tag::value && - detected_volatile_join_no_tag::value)>> + F, std::enable_if_t<(is_reducer::value || + (!is_reducer::value && std::is_void_v)) && + (!detected_join_no_tag::value && + detected_volatile_join_no_tag::value)>> : public has_volatile_join_no_tag_function { enum : bool { value = true }; static_assert(Impl::dependent_false_v, @@ -735,8 +733,8 @@ struct FunctorAnalysis { template struct DeduceInitNoTag< - F, std::enable_if_t::value || (!is_reducer::value && - std::is_void::value), + F, std::enable_if_t::value || + (!is_reducer::value && std::is_void_v), decltype(has_init_no_tag_function::enable_if( &F::init))>> : public has_init_no_tag_function { @@ -835,8 +833,8 @@ struct FunctorAnalysis { template struct DeduceFinalNoTag< - F, std::enable_if_t::value || (!is_reducer::value && - std::is_void::value), + F, std::enable_if_t::value || + (!is_reducer::value && std::is_void_v), decltype(has_final_no_tag_function::enable_if( &F::final))>> : public has_final_no_tag_function { @@ -906,14 +904,14 @@ struct FunctorAnalysis { Functor m_functor; template - KOKKOS_INLINE_FUNCTION constexpr std::enable_if_t len() const - noexcept { + KOKKOS_INLINE_FUNCTION constexpr std::enable_if_t len() + const noexcept { return m_functor.value_count; } template - KOKKOS_INLINE_FUNCTION constexpr std::enable_if_t len() const - noexcept { + KOKKOS_INLINE_FUNCTION constexpr std::enable_if_t len() + const noexcept { return candidate_is_void ? 0 : 1; } @@ -973,8 +971,8 @@ struct FunctorAnalysis { DeduceJoin<>::join(&m_functor, dst, src); } - KOKKOS_INLINE_FUNCTION reference_type init(ValueType* const dst) const - noexcept { + KOKKOS_INLINE_FUNCTION reference_type + init(ValueType* const dst) const noexcept { DeduceInit<>::init(&m_functor, dst); return reference(dst); } @@ -987,11 +985,11 @@ struct FunctorAnalysis { KOKKOS_INLINE_FUNCTION const Functor& get_functor() const { return m_functor; } - Reducer(Reducer const&) = default; - Reducer(Reducer&&) = default; + Reducer(Reducer const&) = default; + Reducer(Reducer&&) = default; Reducer& operator=(Reducer const&) = delete; - Reducer& operator=(Reducer&&) = delete; - ~Reducer() = default; + Reducer& operator=(Reducer&&) = delete; + ~Reducer() = default; KOKKOS_INLINE_FUNCTION explicit constexpr Reducer( Functor const& arg_functor) noexcept diff --git a/lib/kokkos/core/src/impl/Kokkos_GraphImpl.hpp b/lib/kokkos/core/src/impl/Kokkos_GraphImpl.hpp index 56f95c814d..6d3ebf64be 100644 --- a/lib/kokkos/core/src/impl/Kokkos_GraphImpl.hpp +++ b/lib/kokkos/core/src/impl/Kokkos_GraphImpl.hpp @@ -56,7 +56,7 @@ struct GraphAccess { static_assert( Kokkos::Impl::is_specialization_of::value, "Kokkos Internal Error in graph interface"); - return std::make_shared((Args &&) args...); + return std::make_shared((Args&&)args...); } template ::value, "Kokkos Internal Implementation error (bad argument to " "`GraphAccess::get_node_ptr()`)"); - return ((NodeRef &&) node_ref).get_node_ptr(); + return ((NodeRef&&)node_ref).get_node_ptr(); } template @@ -93,7 +93,7 @@ struct GraphAccess { Kokkos::Experimental::GraphNodeRef>::value, "Kokkos Internal Implementation error (bad argument to " "`GraphAccess::get_graph_weak_ptr()`)"); - return ((NodeRef &&) node_ref).get_graph_weak_ptr(); + return ((NodeRef&&)node_ref).get_graph_weak_ptr(); } // end accessors for private members of public interface }}}2 diff --git a/lib/kokkos/core/src/impl/Kokkos_GraphImpl_Utilities.hpp b/lib/kokkos/core/src/impl/Kokkos_GraphImpl_Utilities.hpp index 2ab05cb8e4..b02a265472 100644 --- a/lib/kokkos/core/src/impl/Kokkos_GraphImpl_Utilities.hpp +++ b/lib/kokkos/core/src/impl/Kokkos_GraphImpl_Utilities.hpp @@ -54,9 +54,9 @@ template